Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/w/8.6/bugfix/CLDSRV-518/duplicat…
Browse files Browse the repository at this point in the history 
…ion' into w/8.7/bugfix/CLDSRV-518/duplication
  • Loading branch information
@nicolas2bert
nicolas2bert committed Mar 19, 2024
2 parents 735fcd0 + c6724eb commit 41975d5
Show file tree
Hide file tree
Showing 3 changed files with 181 additions and 58 deletions.
5 changes: 5 additions & 0 deletions lib/routes/routeBackbeat.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -519,6 +519,11 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) {
if (!nullVersionCompatMode) {
omVal.isNull2 = true;
}
// Delete the version id from the version metadata payload to prevent issues
// with creating a non-version object (versioning set to false) that includes a version id.
// For example, this version ID might come from a null version of a suspended bucket being
// replicated to this bucket.
delete omVal.versionId;
if (versioning) {
// If the null version does not have a version id, it is a current null version.
// To update the metadata of a current version, versioning is set to false.
Expand Down
173 changes: 115 additions & 58 deletions tests/multipleBackend/routes/routeBackbeat.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,8 @@ const AWS = require('aws-sdk');
const async = require('async');
const crypto = require('crypto');
const { v4: uuidv4 } = require('uuid');
const { models, versioning } = require('arsenal');
const { versioning } = require('arsenal');
const versionIdUtils = versioning.VersionID;
const { ObjectMD } = models;

const { makeid } = require('../../unit/helpers');
const { makeRequest, makeBackbeatRequest } = require('../../functional/raw-node/utils/makeRequest');
Expand All @@ -29,7 +28,6 @@ const azureClient = getAzureClient();
const containerName = getAzureContainerName(azureLocation);

const ipAddress = process.env.IP ? process.env.IP : '127.0.0.1';
// const describeSkipIfAWS = process.env.AWS_ON_AIR ? describe.skip : describe;

const backbeatAuthCredentials = {
accessKey: 'accessKey1',
Expand Down Expand Up @@ -146,21 +144,6 @@ function checkVersionData(s3, bucket, objectKey, versionId, dataValue, done) {
});
}

function updateStorageClass(data, storageClass) {
let parsedBody;
try {
parsedBody = JSON.parse(data.body);
} catch (err) {
return { error: err };
}
const { result, error } = ObjectMD.createFromBlob(parsedBody.Body);
if (error) {
return { error };
}
result.setAmzStorageClass(storageClass);
return { result };
}

describe.skip('backbeat DELETE routes', () => {
it('abort MPU', done => {
const awsKey = 'backbeat-mpu-test';
Expand Down Expand Up @@ -207,6 +190,19 @@ describe.skip('backbeat DELETE routes', () => {
});
});

function updateStorageClass(data, storageClass) {
let result;
try {
const parsedBody = JSON.parse(JSON.parse(data.body).Body);
parsedBody['x-amz-storage-class'] = storageClass;
result = JSON.stringify(parsedBody);
} catch (err) {
return { error: err };
}

return { result };
}

function getMetadataToPut(putDataResponse) {
const mdToPut = Object.assign({}, testMd);
// Reproduce what backbeat does to update target metadata
Expand Down Expand Up @@ -306,11 +302,11 @@ describe('backbeat routes', () => {

it('should update metadata of a current null version', done => {
let objMD;
return async.series([
next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next),
next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } },
next),
next => makeBackbeatRequest({
return async.series({
putObject: next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next),
enableVersioningSource: next => s3.putBucketVersioning(
{ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } }, next),
getMetadata: next => makeBackbeatRequest({
method: 'GET',
resourceType: 'metadata',
bucket,
Expand All @@ -330,7 +326,7 @@ describe('backbeat routes', () => {
objMD = result;
return next();
}),
next => makeBackbeatRequest({
putMetadata: next => makeBackbeatRequest({
method: 'PUT',
resourceType: 'metadata',
bucket,
Expand All @@ -339,19 +335,37 @@ describe('backbeat routes', () => {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(),
requestBody: objMD,
}, next),
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
next => s3.listObjectVersions({ Bucket: bucket }, next),
], (err, data) => {
headObject: next => s3.headObject(
{ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
getMetadataAfter: next => makeBackbeatRequest({
method: 'GET',
resourceType: 'metadata',
bucket,
objectKey: keyName,
queryObj: {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
}, next),
listObjectVersions: next => s3.listObjectVersions({ Bucket: bucket }, next),
}, (err, results) => {
if (err) {
return done(err);
}
const headObjectRes = data[4];

const headObjectRes = results.headObject;
assert.strictEqual(headObjectRes.VersionId, 'null');
assert.strictEqual(headObjectRes.StorageClass, storageClass);

const listObjectVersionsRes = data[5];
const getMetadataAfterRes = results.getMetadataAfter;
const objMDAfter = JSON.parse(getMetadataAfterRes.body).Body;
const expectedMd = JSON.parse(objMD);
expectedMd.isNull = true; // TODO remove the line once CLDSRV-509 is fixed
assert.deepStrictEqual(JSON.parse(objMDAfter), expectedMd);

const listObjectVersionsRes = results.listObjectVersions;
const { Versions } = listObjectVersionsRes;

assert.strictEqual(Versions.length, 1);
Expand All @@ -365,18 +379,20 @@ describe('backbeat routes', () => {
it('should update metadata of a non-current null version', done => {
let objMD;
let expectedVersionId;
return async.series([
next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next),
next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } },
next),
next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, (err, data) => {
return async.series({
putObjectInitial: next => s3.putObject(
{ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next),
enableVersioning: next => s3.putBucketVersioning(
{ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } }, next),
putObjectAgain: next => s3.putObject(
{ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, (err, data) => {
if (err) {
return next(err);
}
expectedVersionId = data.VersionId;
return next();
}),
next => makeBackbeatRequest({
getMetadata: next => makeBackbeatRequest({
method: 'GET',
resourceType: 'metadata',
bucket,
Expand All @@ -396,7 +412,7 @@ describe('backbeat routes', () => {
objMD = result;
return next();
}),
next => makeBackbeatRequest({
putMetadata: next => makeBackbeatRequest({
method: 'PUT',
resourceType: 'metadata',
bucket,
Expand All @@ -405,23 +421,36 @@ describe('backbeat routes', () => {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(),
requestBody: objMD,
}, next),
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
next => s3.listObjectVersions({ Bucket: bucket }, next),
], (err, data) => {
headObject: next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
getMetadataAfter: next => makeBackbeatRequest({
method: 'GET',
resourceType: 'metadata',
bucket,
objectKey: keyName,
queryObj: {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
}, next),
listObjectVersions: next => s3.listObjectVersions({ Bucket: bucket }, next),
}, (err, results) => {
if (err) {
return done(err);
}
const headObjectRes = data[5];
const headObjectRes = results.headObject;
assert.strictEqual(headObjectRes.VersionId, 'null');
assert.strictEqual(headObjectRes.StorageClass, storageClass);

const listObjectVersionsRes = data[6];
const getMetadataAfterRes = results.getMetadataAfter;
const objMDAfter = JSON.parse(getMetadataAfterRes.body).Body;
assert.deepStrictEqual(JSON.parse(objMDAfter), JSON.parse(objMD));

const listObjectVersionsRes = results.listObjectVersions;
const { Versions } = listObjectVersionsRes;

assert.strictEqual(Versions.length, 2);

const currentVersion = Versions.find(v => v.IsLatest);
assertVersionHasNotBeenUpdated(currentVersion, expectedVersionId);

Expand Down Expand Up @@ -469,9 +498,19 @@ describe('backbeat routes', () => {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(),
requestBody: objMD,
}, next),
headObject: next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
getMetadataAfter: next => makeBackbeatRequest({
method: 'GET',
resourceType: 'metadata',
bucket,
objectKey: keyName,
queryObj: {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
}, next),
listObjectVersions: next => s3.listObjectVersions({ Bucket: bucket }, next),
}, (err, results) => {
if (err) {
Expand All @@ -481,6 +520,10 @@ describe('backbeat routes', () => {
assert.strictEqual(headObjectRes.VersionId, 'null');
assert.strictEqual(headObjectRes.StorageClass, storageClass);

const getMetadataAfterRes = results.getMetadataAfter;
const objMDAfter = JSON.parse(getMetadataAfterRes.body).Body;
assert.deepStrictEqual(JSON.parse(objMDAfter), JSON.parse(objMD));

const listObjectVersionsRes = results.listObjectVersions;
const { Versions } = listObjectVersionsRes;

Expand Down Expand Up @@ -534,9 +577,19 @@ describe('backbeat routes', () => {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(),
requestBody: objMD,
}, next),
headObject: next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
getMetadataAfter: next => makeBackbeatRequest({
method: 'GET',
resourceType: 'metadata',
bucket,
objectKey: keyName,
queryObj: {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
}, next),
listObjectVersions: next => s3.listObjectVersions({ Bucket: bucket }, next),
}, (err, results) => {
if (err) {
Expand All @@ -546,6 +599,10 @@ describe('backbeat routes', () => {
assert.strictEqual(headObjectRes.VersionId, 'null');
assert.strictEqual(headObjectRes.StorageClass, storageClass);

const getMetadataAfterRes = results.getMetadataAfter;
const objMDAfter = JSON.parse(getMetadataAfterRes.body).Body;
assert.deepStrictEqual(JSON.parse(objMDAfter), JSON.parse(objMD));

const listObjectVersionsRes = results.listObjectVersions;
const { Versions } = listObjectVersionsRes;

Expand Down Expand Up @@ -590,7 +647,7 @@ describe('backbeat routes', () => {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(),
requestBody: objMD,
}, next),
next => s3.headObject({ Bucket: bucket, Key: keyName }, next),
next => s3.listObjectVersions({ Bucket: bucket }, next),
Expand Down Expand Up @@ -652,7 +709,7 @@ describe('backbeat routes', () => {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(),
requestBody: objMD,
}, next),
next => s3.headObject({ Bucket: bucket, Key: keyName }, next),
next => s3.listObjectVersions({ Bucket: bucket }, next),
Expand Down Expand Up @@ -715,7 +772,7 @@ describe('backbeat routes', () => {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(),
requestBody: objMD,
}, next),
next => s3.headObject({ Bucket: bucket, Key: keyName }, next),
next => s3.listObjectVersions({ Bucket: bucket }, next),
Expand Down Expand Up @@ -786,7 +843,7 @@ describe('backbeat routes', () => {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(),
requestBody: objMD,
}, next),
next => s3.headObject({ Bucket: bucket, Key: keyName }, next),
next => s3.listObjectVersions({ Bucket: bucket }, next),
Expand Down Expand Up @@ -851,7 +908,7 @@ describe('backbeat routes', () => {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(),
requestBody: objMD,
}, next),
next => s3.headObject({ Bucket: bucket, Key: keyName }, next),
next => s3.listObjectVersions({ Bucket: bucket }, next),
Expand Down Expand Up @@ -911,7 +968,7 @@ describe('backbeat routes', () => {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(),
requestBody: objMD,
}, next),
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
next => s3.listObjectVersions({ Bucket: bucket }, next),
Expand Down Expand Up @@ -972,7 +1029,7 @@ describe('backbeat routes', () => {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(),
requestBody: objMD,
}, next),
next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next),
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
Expand Down Expand Up @@ -1035,7 +1092,7 @@ describe('backbeat routes', () => {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(),
requestBody: objMD,
}, next),
next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } },
next),
Expand Down Expand Up @@ -1115,7 +1172,7 @@ describe('backbeat routes', () => {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(),
requestBody: objMD,
}, next),
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
next => s3.listObjectVersions({ Bucket: bucket }, next),
Expand Down Expand Up @@ -1190,7 +1247,7 @@ describe('backbeat routes', () => {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(),
requestBody: objMD,
}, next),
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
next => s3.listObjectVersions({ Bucket: bucket }, next),
Expand Down Expand Up @@ -1262,7 +1319,7 @@ describe('backbeat routes', () => {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(),
requestBody: objMD,
}, next),
next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next),
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
Expand Down Expand Up @@ -1336,7 +1393,7 @@ describe('backbeat routes', () => {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(),
requestBody: objMD,
}, next),
next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } },
next),
Expand Down
Loading

0 comments on commit 41975d5

Please sign in to comment.