diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java index 58183e87705..de17e773c65 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java @@ -631,6 +631,42 @@ public void testListMultipartUploadParts() throws Exception { assertFalse(ozoneMultipartUploadPartListParts.isTruncated()); } + @Test + public void testAbortMultipartUploadSuccessWithMissingParentDirectories() throws Exception { + String parentDir = "parentDirToDelete/"; + keyName = parentDir + UUID.randomUUID(); + + OzoneManager ozoneManager = cluster.getOzoneManager(); + String buckKey = ozoneManager.getMetadataManager() + .getBucketKey(volume.getName(), bucket.getName()); + OmBucketInfo buckInfo = + ozoneManager.getMetadataManager().getBucketTable().get(buckKey); + BucketLayout bucketLayout = buckInfo.getBucketLayout(); + + String uploadID = initiateMultipartUpload(bucket, keyName, RATIS, + ONE); + + OMMetadataManager metadataMgr = + cluster.getOzoneManager().getMetadataManager(); + String multipartOpenKey = + metadataMgr.getMultipartKeyFSO(volumeName, bucketName, keyName, uploadID); + String multipartKey = metadataMgr.getMultipartKey(volumeName, bucketName, + keyName, uploadID); + + // Delete parent directory + ozClient.getProxy().deleteKey(volumeName, bucketName, parentDir, false); + + // Abort multipart upload with missing parent directory + bucket.abortMultipartUpload(keyName, uploadID); + + OmKeyInfo omKeyInfo = + metadataMgr.getOpenKeyTable(bucketLayout).get(multipartOpenKey); + OmMultipartKeyInfo omMultipartKeyInfo = + metadataMgr.getMultipartInfoTable().get(multipartKey); + assertNull(omKeyInfo); + assertNull(omMultipartKeyInfo); + } + private void verifyPartNamesInDB(Map partsMap, OzoneMultipartUploadPartListParts ozoneMultipartUploadPartListParts, String uploadID) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 8f4c070b76c..06bcb105842 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -115,6 +115,7 @@ import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.BUCKET_NOT_FOUND; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.FILE_NOT_FOUND; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.VOLUME_NOT_FOUND; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NO_SUCH_MULTIPART_UPLOAD_ERROR; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DIR; import static org.apache.hadoop.ozone.om.service.SnapshotDeletingService.isBlockLocationInfoSame; import static org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.checkSnapshotDirExist; @@ -899,11 +900,26 @@ public String getMultipartKeyFSO(String volume, String bucket, String key, Strin final long volumeId = getVolumeId(volume); final long bucketId = getBucketId(volume, bucket); - long parentId = - OMFileRequest.getParentID(volumeId, bucketId, key, this); - - String fileName = OzoneFSUtils.getFileName(key); + long parentId; + try { + parentId = OMFileRequest.getParentID(volumeId, bucketId, key, this); + } catch (final Exception e) { + // It is possible we miss directories and exception is thrown. + // see https://issues.apache.org/jira/browse/HDDS-11784 + LOG.warn("Got exception when finding parent id for {}/{}/{}. Use another way to get it", + volumeId, bucketId, key, e); + final String multipartKey = + getMultipartKey(volume, bucket, key, uploadId); + final OmMultipartKeyInfo multipartKeyInfo = + getMultipartInfoTable().get(multipartKey); + if (multipartKeyInfo == null) { + LOG.error("Could not find multipartKeyInfo for {}", multipartKey); + throw new OMException(NO_SUCH_MULTIPART_UPLOAD_ERROR); + } + parentId = multipartKeyInfo.getParentID(); + } + final String fileName = OzoneFSUtils.getFileName(key); return getMultipartKey(volumeId, bucketId, parentId, fileName, uploadId); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponseWithFSO.java index 4d1a6ce09bc..918f8af4e00 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponseWithFSO.java @@ -37,6 +37,7 @@ import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.FILE_TABLE; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.MULTIPARTINFO_TABLE; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.OPEN_FILE_TABLE; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DIRECTORY_TABLE; /** * Response for Multipart Upload Complete request. @@ -47,7 +48,7 @@ * 3) Delete unused parts. */ @CleanupTableInfo(cleanupTables = {OPEN_FILE_TABLE, FILE_TABLE, DELETED_TABLE, - MULTIPARTINFO_TABLE}) + MULTIPARTINFO_TABLE, DIRECTORY_TABLE}) public class S3MultipartUploadCompleteResponseWithFSO extends S3MultipartUploadCompleteResponse {