diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java index d35da257cd0..38ea8ee9b66 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java @@ -30,6 +30,7 @@ import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; +import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -51,6 +52,7 @@ import org.apache.hadoop.ozone.audit.S3GAction; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneKey; +import org.apache.hadoop.ozone.client.OzoneKeyDetails; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes; import org.apache.hadoop.ozone.om.helpers.ErrorInfo; @@ -349,25 +351,35 @@ public MultiDeleteResponse multiDelete( if (request.getObjects() != null) { Map undeletedKeyResultMap; - for (DeleteObject keyToDelete : request.getObjects()) { - deleteKeys.add(keyToDelete.getKey()); - } + boolean hasConditionalDeletes = request.getObjects().stream() + .anyMatch(d -> StringUtils.isNotBlank(d.getIfMatch())); long startNanos = Time.monotonicNowNanos(); try { S3Owner.verifyBucketOwnerCondition(getHeaders(), bucketName, bucket.getOwner()); - undeletedKeyResultMap = bucket.deleteKeys(deleteKeys, true); - for (DeleteObject d : request.getObjects()) { - ErrorInfo error = undeletedKeyResultMap.get(d.getKey()); - boolean deleted = error == null || - // if the key is not found, it is assumed to be successfully deleted - ResultCodes.KEY_NOT_FOUND.name().equals(error.getCode()); - if (deleted) { - deleteKeys.remove(d.getKey()); - if (!request.isQuiet()) { - result.addDeleted(new DeletedObject(d.getKey())); + if (hasConditionalDeletes) { + for (DeleteObject keyToDelete : request.getObjects()) { + deleteKeys.add(keyToDelete.getKey()); + deleteSingleObject(bucket, keyToDelete, result, request.isQuiet(), + deleteKeys); + } + } else { + for (DeleteObject keyToDelete : request.getObjects()) { + deleteKeys.add(keyToDelete.getKey()); + } + undeletedKeyResultMap = bucket.deleteKeys(deleteKeys, true); + for (DeleteObject d : request.getObjects()) { + ErrorInfo error = undeletedKeyResultMap.get(d.getKey()); + boolean deleted = error == null || + // if the key is not found, it is assumed to be successfully deleted + ResultCodes.KEY_NOT_FOUND.name().equals(error.getCode()); + if (deleted) { + deleteKeys.remove(d.getKey()); + if (!request.isQuiet()) { + result.addDeleted(new DeletedObject(d.getKey())); + } + } else { + result.addError(new Error(d.getKey(), error.getCode(), error.getMessage())); } - } else { - result.addError(new Error(d.getKey(), error.getCode(), error.getMessage())); } } getMetrics().updateDeleteKeySuccessStats(startNanos); @@ -393,6 +405,48 @@ public MultiDeleteResponse multiDelete( return result; } + private void deleteSingleObject(OzoneBucket bucket, DeleteObject deleteObject, + MultiDeleteResponse result, boolean quiet, List failedDeletes) + throws IOException { + String key = deleteObject.getKey(); + if (StringUtils.isNotBlank(deleteObject.getIfMatch())) { + try { + OzoneKeyDetails keyDetails = bucket.getKey(key); + String currentETag = keyDetails.getMetadata().get(ETAG); + if (!S3ConditionalRequest.matchesETag(deleteObject.getIfMatch(), currentETag)) { + addPreconditionFailedError(result, key); + return; + } + } catch (OMException ex) { + if (ex.getResult() == ResultCodes.KEY_NOT_FOUND) { + addPreconditionFailedError(result, key); + return; + } + throw ex; + } + } + + Map undeletedKeyResultMap = + bucket.deleteKeys(Collections.singletonList(key), true); + ErrorInfo error = undeletedKeyResultMap.get(key); + boolean deleted = error == null || + ResultCodes.KEY_NOT_FOUND.name().equals(error.getCode()); + if (deleted) { + failedDeletes.remove(key); + if (!quiet) { + result.addDeleted(new DeletedObject(key)); + } + } else { + result.addError(new Error(key, error.getCode(), error.getMessage())); + } + } + + private static void addPreconditionFailedError(MultiDeleteResponse result, + String key) { + result.addError(new Error(key, S3ErrorTable.PRECOND_FAILED.getCode(), + S3ErrorTable.PRECOND_FAILED.getErrorMessage())); + } + private void addKey(ListObjectResponse response, OzoneKey next) { KeyMetadata keyMetadata = new KeyMetadata(); keyMetadata.setKey(EncodingTypeObject.createNullable(next.getName(), diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/MultiDeleteRequest.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/MultiDeleteRequest.java index 10ba02456a9..6f1babd03da 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/MultiDeleteRequest.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/MultiDeleteRequest.java @@ -69,6 +69,9 @@ public static class DeleteObject { @XmlElement(name = "VersionId") private String versionId; + @XmlElement(name = S3Consts.IF_MATCH_HEADER) + private String ifMatch; + public DeleteObject() { } @@ -76,6 +79,11 @@ public DeleteObject(String key) { this.key = key; } + public DeleteObject(String key, String ifMatch) { + this.key = key; + this.ifMatch = ifMatch; + } + public String getKey() { return key; } @@ -91,5 +99,13 @@ public String getVersionId() { public void setVersionId(String versionId) { this.versionId = versionId; } + + public String getIfMatch() { + return ifMatch; + } + + public void setIfMatch(String ifMatch) { + this.ifMatch = ifMatch; + } } } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3ConditionalRequest.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3ConditionalRequest.java index 259abd7d4ed..d1c7a77a7b9 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3ConditionalRequest.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3ConditionalRequest.java @@ -196,6 +196,10 @@ private static Response buildNotModifiedResponse(OzoneKey key) { return responseBuilder.build(); } + static boolean matchesETag(String headerValue, String currentETag) { + return eTagMatches(headerValue, currentETag); + } + private static boolean eTagMatches(String headerValue, String currentETag) { if (headerValue == null) { return false; diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultiDeleteRequestUnmarshaller.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultiDeleteRequestUnmarshaller.java index 73179afc896..bf98e7f6a4f 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultiDeleteRequestUnmarshaller.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultiDeleteRequestUnmarshaller.java @@ -66,6 +66,25 @@ public void fromStreamWithoutNamespace() throws IOException { assertEquals(3, multiDeleteRequest.getObjects().size()); } + @Test + public void fromStreamWithIfMatch() throws IOException { + ByteArrayInputStream inputBody = + new ByteArrayInputStream( + ("" + + "key1\"abc\"" + + "key2*" + + "") + .getBytes(UTF_8)); + + MultiDeleteRequest multiDeleteRequest = unmarshall(inputBody); + + assertEquals(2, multiDeleteRequest.getObjects().size()); + assertEquals("key1", multiDeleteRequest.getObjects().get(0).getKey()); + assertEquals("\"abc\"", multiDeleteRequest.getObjects().get(0).getIfMatch()); + assertEquals("key2", multiDeleteRequest.getObjects().get(1).getKey()); + assertEquals("*", multiDeleteRequest.getObjects().get(1).getIfMatch()); + } + private MultiDeleteRequest unmarshall(ByteArrayInputStream inputBody) throws IOException { return new MultiDeleteRequestUnmarshaller() diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectMultiDelete.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectMultiDelete.java index 1d657460346..52376977fea 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectMultiDelete.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectMultiDelete.java @@ -17,14 +17,21 @@ package org.apache.hadoop.ozone.s3.endpoint; +import static java.util.Collections.emptyMap; import static java.util.Collections.singleton; +import static org.apache.hadoop.ozone.OzoneConsts.ETAG; import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertErrorResponse; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.MALFORMED_XML; +import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.PRECOND_FAILED; +import static org.apache.hadoop.ozone.s3.util.S3Utils.parseETag; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import com.google.common.collect.Sets; import java.io.IOException; +import java.util.HashMap; import java.util.HashSet; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import javax.xml.bind.JAXBException; @@ -32,6 +39,7 @@ import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientStub; import org.apache.hadoop.ozone.client.OzoneKey; +import org.apache.hadoop.ozone.client.io.OzoneOutputStream; import org.apache.hadoop.ozone.s3.endpoint.MultiDeleteRequest.DeleteObject; import org.apache.hadoop.ozone.s3.exception.OS3Exception; import org.apache.hadoop.ozone.s3.util.S3Consts; @@ -140,6 +148,110 @@ public void multiDeleteAllowsMaxKeysPerRequest() throws Exception { assertEquals(3, Sets.newHashSet(bucket.listKeys("")).size()); } + @Test + public void conditionalDeleteMatchingIfMatch() throws Exception { + OzoneClient client = new OzoneClientStub(); + OzoneBucket bucket = initConditionalDeleteTestData(client); + + BucketEndpoint rest = EndpointBuilder.newBucketEndpointBuilder() + .setClient(client) + .build(); + + MultiDeleteRequest mdr = new MultiDeleteRequest(); + mdr.getObjects().add(new DeleteObject("key1", "\"match-1\"")); + mdr.getObjects().add(new DeleteObject("key2", "match-2")); + + MultiDeleteResponse response = rest.multiDelete("b1", "", mdr); + + assertEquals(2, response.getDeletedObjects().size()); + assertEquals(0, response.getErrors().size()); + assertEquals(singleton("key3"), listKeyNames(bucket)); + } + + @Test + public void conditionalDeleteMismatchedIfMatch() throws Exception { + OzoneClient client = new OzoneClientStub(); + initConditionalDeleteTestData(client); + + BucketEndpoint rest = EndpointBuilder.newBucketEndpointBuilder() + .setClient(client) + .build(); + + MultiDeleteRequest mdr = new MultiDeleteRequest(); + mdr.getObjects().add(new DeleteObject("key1", "\"wrong-match\"")); + + MultiDeleteResponse response = rest.multiDelete("b1", "", mdr); + + assertEquals(0, response.getDeletedObjects().size()); + assertEquals(1, response.getErrors().size()); + assertEquals(PRECOND_FAILED.getCode(), response.getErrors().get(0).getCode()); + assertEquals("key1", response.getErrors().get(0).getKey()); + } + + @Test + public void conditionalDeleteMissingKeyFails() throws Exception { + OzoneClient client = new OzoneClientStub(); + initConditionalDeleteTestData(client); + + BucketEndpoint rest = EndpointBuilder.newBucketEndpointBuilder() + .setClient(client) + .build(); + + MultiDeleteRequest mdr = new MultiDeleteRequest(); + mdr.getObjects().add(new DeleteObject("missing-key", "\"match-1\"")); + + MultiDeleteResponse response = rest.multiDelete("b1", "", mdr); + + assertEquals(0, response.getDeletedObjects().size()); + assertEquals(1, response.getErrors().size()); + assertEquals(PRECOND_FAILED.getCode(), response.getErrors().get(0).getCode()); + } + + @Test + public void conditionalDeleteWildcardIfMatch() throws Exception { + OzoneClient client = new OzoneClientStub(); + OzoneBucket bucket = initConditionalDeleteTestData(client); + + BucketEndpoint rest = EndpointBuilder.newBucketEndpointBuilder() + .setClient(client) + .build(); + + MultiDeleteRequest mdr = new MultiDeleteRequest(); + mdr.getObjects().add(new DeleteObject("key1", "*")); + mdr.getObjects().add(new DeleteObject("missing-key", "*")); + + MultiDeleteResponse response = rest.multiDelete("b1", "", mdr); + + assertEquals(1, response.getDeletedObjects().size()); + assertEquals("key1", response.getDeletedObjects().get(0).getKey()); + assertEquals(1, response.getErrors().size()); + assertEquals("missing-key", response.getErrors().get(0).getKey()); + assertEquals(PRECOND_FAILED.getCode(), response.getErrors().get(0).getCode()); + assertTrue(listKeyNames(bucket).contains("key2")); + assertTrue(listKeyNames(bucket).contains("key3")); + } + + @Test + public void conditionalDeleteMixedWithUnconditional() throws Exception { + OzoneClient client = new OzoneClientStub(); + OzoneBucket bucket = initConditionalDeleteTestData(client); + + BucketEndpoint rest = EndpointBuilder.newBucketEndpointBuilder() + .setClient(client) + .build(); + + MultiDeleteRequest mdr = new MultiDeleteRequest(); + mdr.getObjects().add(new DeleteObject("key1", "\"match-1\"")); + mdr.getObjects().add(new DeleteObject("key2")); + mdr.getObjects().add(new DeleteObject("missing-unconditional")); + + MultiDeleteResponse response = rest.multiDelete("b1", "", mdr); + + assertEquals(3, response.getDeletedObjects().size()); + assertEquals(0, response.getErrors().size()); + assertEquals(singleton("key3"), listKeyNames(bucket)); + } + private OzoneBucket initTestData(OzoneClient client) throws IOException { client.getObjectStore().createS3Bucket("b1"); @@ -151,4 +263,30 @@ private OzoneBucket initTestData(OzoneClient client) throws IOException { bucket.createKey("key3", 0).close(); return bucket; } + + private OzoneBucket initConditionalDeleteTestData(OzoneClient client) throws IOException { + client.getObjectStore().createS3Bucket("b1"); + + OzoneBucket bucket = client.getObjectStore().getS3Bucket("b1"); + createKeyWithIfMatchTarget(bucket, "key1", "\"match-1\""); + createKeyWithIfMatchTarget(bucket, "key2", "match-2"); + createKeyWithIfMatchTarget(bucket, "key3", "match-3"); + return bucket; + } + + private static void createKeyWithIfMatchTarget(OzoneBucket bucket, String keyName, + String ifMatch) throws IOException { + Map metadata = new HashMap<>(); + metadata.put(ETAG, parseETag(ifMatch)); + try (OzoneOutputStream out = bucket.createKey(keyName, 1, + bucket.getReplicationConfig(), metadata, emptyMap())) { + out.write('x'); + } + } + + private static Set listKeyNames(OzoneBucket bucket) throws IOException { + return Sets.newHashSet(bucket.listKeys("")).stream() + .map(OzoneKey::getName) + .collect(Collectors.toSet()); + } }