diff --git a/tests/functional/raw-node/test/GCP/bucket/bucket.js b/tests/functional/raw-node/test/GCP/bucket/bucket.js new file mode 100644 index 0000000000..f155a0e0b8 --- /dev/null +++ b/tests/functional/raw-node/test/GCP/bucket/bucket.js @@ -0,0 +1,169 @@ +const assert = require('assert'); +const async = require('async'); +const util = require('util'); +const arsenal = require('arsenal'); +const { + HeadBucketCommand, + ListObjectsCommand, + CreateBucketCommand, + DeleteBucketCommand, + PutObjectCommand, + DeleteObjectCommand, +} = require('@aws-sdk/client-s3'); +const { GCP } = arsenal.storage.data.external.GCP; +const { genUniqID, genBucketName, gcpRetry } = require('../../../utils/gcpUtils'); +const { getRealAwsConfig } = + require('../../../../aws-node-sdk/test/support/awsConfig'); +const { listingHardLimit } = require('../../../../../../constants'); + +const credentialOne = 'gcpbackend'; +const config = getRealAwsConfig(credentialOne); +const gcpClient = new GCP(config); + +describe('GCP: Bucket', function testSuite() { + this.timeout(180000); + + const bucketName = genBucketName('bucket'); + + before(async () => { + process.stdout.write(`Creating test bucket ${bucketName}\n`); + await gcpRetry(gcpClient, new CreateBucketCommand({ Bucket: bucketName })); + }); + + after(async () => { + await gcpRetry(gcpClient, new DeleteBucketCommand({ Bucket: bucketName })); + }); + + describe('HEAD Bucket', () => { + it('should return 404 for non-existing bucket', async () => { + const badBucketName = `cldsrvci-bucket-${genUniqID()}`; + try { + await gcpClient.send(new HeadBucketCommand({ Bucket: badBucketName })); + assert.fail('Expected 404 error, but got success'); + } catch (err) { + assert(err); + assert.strictEqual(err.$metadata?.httpStatusCode, 404); + const errorName = err.name === 'NotFound' ? 'NoSuchBucket' : err.name; + assert.strictEqual(errorName, 'NoSuchBucket'); + } + }); + + it('should return 200 and bucket metadata', async () => { + // Need to use the helper headBucket function for middleware with MetaVersionId + const res = await util.promisify(gcpClient.headBucket.bind(gcpClient))({ Bucket: bucketName }); + const { $metadata, ...data } = res; + assert.strictEqual($metadata?.httpStatusCode, 200); + // Ensure MetaVersionId is present and non-empty + assert.ok( + typeof data.MetaVersionId === 'string' + && data.MetaVersionId.length > 0 + ); + }); + }); + + describe('GET Bucket (List Objects)', () => { + const smallSize = 20; + const bigSize = listingHardLimit + 1; + + function populateBucket(createdObjects, callback) { + process.stdout.write( + `Putting ${createdObjects.length} objects into bucket\n`); + async.mapLimit( + createdObjects, + 10, + async object => gcpClient.send(new PutObjectCommand({ + Bucket: bucketName, + Key: object, + })), + err => { + if (err) { + process.stdout.write(`err putting objects ${err}\n`); + } + return callback(err); + } + ); + } + + function removeObjects(createdObjects, callback) { + process.stdout.write( + `Deleting ${createdObjects.length} objects from bucket\n`); + async.mapLimit( + createdObjects, + 10, + async object => gcpClient.send(new DeleteObjectCommand({ + Bucket: bucketName, + Key: object, + })), + err => { + if (err) { + process.stdout.write(`err deleting objects ${err}\n`); + } + return callback(err); + } + ); + } + + it('should return 200', async () => { + const res = await gcpClient.send( + new ListObjectsCommand({ Bucket: bucketName })); + assert.strictEqual(res.$metadata?.httpStatusCode, 200); + }); + + describe('with less than listingHardLimit number of objects', () => { + const createdObjects = Array.from( + Array(smallSize).keys()).map(i => `someObject-${i}`); + + before(done => populateBucket(createdObjects, done)); + after(done => removeObjects(createdObjects, done)); + + it(`should list all ${smallSize} created objects`, async () => { + const res = await gcpClient.send( + new ListObjectsCommand({ Bucket: bucketName })); + assert.strictEqual(res.Contents.length, smallSize); + }); + + it('should list MaxKeys number of objects with MaxKeys at 10', async () => { + const res = await gcpClient.send(new ListObjectsCommand({ + Bucket: bucketName, + MaxKeys: 10, + })); + assert.strictEqual(res.Contents.length, 10); + }); + }); + + describe('with more than listingHardLimit number of objects', () => { + const createdObjects = Array.from( + Array(bigSize).keys()).map(i => `someObject-${i}`); + + before(done => populateBucket(createdObjects, done)); + after(done => removeObjects(createdObjects, done)); + + it('should list at max 1000 of objects created', async () => { + const res = await gcpClient.send( + new ListObjectsCommand({ Bucket: bucketName })); + assert.strictEqual(res.Contents.length, listingHardLimit); + }); + + describe('with MaxKeys at 1001', () => { + // TODO: S3C-5445 + // Note: this test is testing GCP behaviour, not the Cloudserver one. + // It tests that GET https://.storage.googleapis.com/?max-keys=1001 + // returns only the first 1000 objects. + // + // Expected behavior: the GCP XML API should not return a list longer + // than 1000 objects, even if max-keys is greater than 1000: + // https://cloud.google.com/storage/docs/xml-api/reference-headers#maxkeys + // + // Actual behavior: it returns a list longer than 1000 objects when + // max-keys is greater than 1000 + it.skip('should list at max 1000, ignoring MaxKeys', async () => { + const res = await gcpClient.send(new ListObjectsCommand({ + Bucket: bucketName, + MaxKeys: 1001, + })); + assert.strictEqual(res.Contents.length, listingHardLimit); + }); + }); + }); + }); +}); diff --git a/tests/functional/raw-node/test/GCP/bucket/get.js b/tests/functional/raw-node/test/GCP/bucket/get.js deleted file mode 100644 index c3491f593c..0000000000 --- a/tests/functional/raw-node/test/GCP/bucket/get.js +++ /dev/null @@ -1,183 +0,0 @@ -const assert = require('assert'); -const async = require('async'); -const arsenal = require('arsenal'); -const { - ListObjectsCommand, - HeadBucketCommand, - CreateBucketCommand, - DeleteBucketCommand, - PutObjectCommand, - DeleteObjectCommand, -} = require('@aws-sdk/client-s3'); -const { GCP } = arsenal.storage.data.external.GCP; -const { genUniqID, genBucketName, gcpRetry } = require('../../../utils/gcpUtils'); -const { getRealAwsConfig } = - require('../../../../aws-node-sdk/test/support/awsConfig'); -const { listingHardLimit } = require('../../../../../../constants'); - -const credentialOne = 'gcpbackend'; -const bucketName = genBucketName('get'); -const smallSize = 20; -const bigSize = listingHardLimit + 1; -const config = getRealAwsConfig(credentialOne); -const gcpClient = new GCP(config); - -function populateBucket(createdObjects, callback) { - process.stdout.write( - `Putting ${createdObjects.length} objects into bucket\n`); - async.mapLimit( - createdObjects, - 10, - async object => { - const command = new PutObjectCommand({ - Bucket: bucketName, - Key: object, - }); - await gcpClient.send(command); - }, - err => { - if (err) { - process.stdout - .write(`err putting objects ${err}\n`); - } - return callback(err); - } - ); -} - -function removeObjects(createdObjects, callback) { - process.stdout.write( - `Deleting ${createdObjects.length} objects from bucket\n`); - async.mapLimit( - createdObjects, - 10, - async object => { - const command = new DeleteObjectCommand({ - Bucket: bucketName, - Key: object, - }); - await gcpClient.send(command); - }, - err => { - if (err) { - process.stdout - .write(`err deleting objects ${err}\n`); - } - return callback(err); - } - ); -} - -describe('GCP: GET Bucket', function testSuite() { - this.timeout(180000); - - before(async () => { - await gcpRetry( - gcpClient, - new CreateBucketCommand({ Bucket: bucketName }), - ); - }); - - after(async () => { - await gcpRetry( - gcpClient, - new DeleteBucketCommand({ Bucket: bucketName }), - ); - }); - - describe('without existing bucket', () => { - it('should return 404 and NoSuchBucket', async () => { - const badBucketName = `nonexistingbucket-${genUniqID()}`; - try { - const command = new HeadBucketCommand({ - Bucket: badBucketName, - }); - await gcpClient.send(command); - assert.fail('Expected NoSuchBucket error, but got success'); - } catch (err) { - assert(err); - const statusCode = err.$metadata && err.$metadata.httpStatusCode; - assert.strictEqual(statusCode, 404); - const errorName = err.name === 'NotFound' ? 'NoSuchBucket' : err.name; - assert.strictEqual(errorName, 'NoSuchBucket'); - } - }); - - it('should return 200', async () => { - const command = new ListObjectsCommand({ - Bucket: bucketName, - }); - const res = await gcpClient.send(command); - assert.strictEqual(res.$metadata?.httpStatusCode, 200); - }); - }); - - describe('with existing bucket', () => { - describe('with less than listingHardLimit number of objects', () => { - const createdObjects = Array.from( - Array(smallSize).keys()).map(i => `someObject-${i}`); - - before(done => populateBucket(createdObjects, done)); - - after(done => removeObjects(createdObjects, done)); - - it(`should list all ${smallSize} created objects`, async () => { - const command = new ListObjectsCommand({ - Bucket: bucketName, - }); - const res = await gcpClient.send(command); - assert.strictEqual(res.Contents.length, smallSize); - }); - - describe('with MaxKeys at 10', () => { - it('should list MaxKeys number of objects', async () => { - const command = new ListObjectsCommand({ - Bucket: bucketName, - MaxKeys: 10, - }); - const res = await gcpClient.send(command); - assert.strictEqual(res.Contents.length, 10); - }); - }); - }); - - describe('with more than listingHardLimit number of objects', () => { - const createdObjects = Array.from( - Array(bigSize).keys()).map(i => `someObject-${i}`); - - before(done => populateBucket(createdObjects, done)); - - after(done => removeObjects(createdObjects, done)); - - it('should list at max 1000 of objects created', async () => { - const command = new ListObjectsCommand({ - Bucket: bucketName, - }); - const res = await gcpClient.send(command); - assert.strictEqual(res.Contents.length, listingHardLimit); - }); - - describe('with MaxKeys at 1001', () => { - // TODO: S3C-5445 - // Note: this test is testing GCP behaviour, not the Cloudserver one. - // It tests that GET https://.storage.googleapis.com/?max-keys=1001 - // returns only the first 1000 objects. - // - // Expected behavior: the GCP XML API should not return a list longer - // than 1000 objects, even if max-keys is greater than 1000: - // https://cloud.google.com/storage/docs/xml-api/reference-headers#maxkeys - // - // Actual behavior: it returns a list longer than 1000 objects when - // max-keys is greater than 1000 - it.skip('should list at max 1000, ignoring MaxKeys', async () => { - const command = new ListObjectsCommand({ - Bucket: bucketName, - MaxKeys: 1001, - }); - const res = await gcpClient.send(command); - assert.strictEqual(res.Contents.length, listingHardLimit); - }); - }); - }); - }); -}); diff --git a/tests/functional/raw-node/test/GCP/bucket/getVersioning.js b/tests/functional/raw-node/test/GCP/bucket/getVersioning.js deleted file mode 100644 index 2cbb852020..0000000000 --- a/tests/functional/raw-node/test/GCP/bucket/getVersioning.js +++ /dev/null @@ -1,64 +0,0 @@ -const assert = require('assert'); -const arsenal = require('arsenal'); -const { - PutBucketVersioningCommand, - GetBucketVersioningCommand, - CreateBucketCommand, - DeleteBucketCommand, -} = require('@aws-sdk/client-s3'); -const { GCP } = arsenal.storage.data.external.GCP; -const { genBucketName, gcpRetry } = require('../../../utils/gcpUtils'); -const { getRealAwsConfig } = - require('../../../../aws-node-sdk/test/support/awsConfig'); - -const credentialOne = 'gcpbackend'; -const verEnabledObj = 'Enabled'; -const verDisabledObj = 'Suspended'; - -describe('GCP: GET Bucket Versioning', () => { - const config = getRealAwsConfig(credentialOne); - const gcpClient = new GCP(config); - - beforeEach(async function beforeFn() { - this.currentTest.bucketName = genBucketName('getversioning'); - await gcpRetry( - gcpClient, - new CreateBucketCommand({ - Bucket: this.currentTest.bucketName, - }), - ); - }); - - afterEach(async function afterFn() { - await gcpRetry( - gcpClient, - new DeleteBucketCommand({ - Bucket: this.currentTest.bucketName, - }), - ); - }); - - it('should verify bucket versioning is enabled', async function testFn() { - await gcpClient.send(new PutBucketVersioningCommand({ - Bucket: this.test.bucketName, - VersioningConfiguration: { Status: 'Enabled' }, - })); - - const res = await gcpClient.send(new GetBucketVersioningCommand({ - Bucket: this.test.bucketName, - })); - assert.deepStrictEqual(res.Status, verEnabledObj); - }); - - it('should verify bucket versioning is disabled', async function testFn() { - await gcpClient.send(new PutBucketVersioningCommand({ - Bucket: this.test.bucketName, - VersioningConfiguration: { Status: 'Suspended' }, - })); - - const res = await gcpClient.send(new GetBucketVersioningCommand({ - Bucket: this.test.bucketName, - })); - assert.deepStrictEqual(res.Status, verDisabledObj); - }); -}); diff --git a/tests/functional/raw-node/test/GCP/bucket/head.js b/tests/functional/raw-node/test/GCP/bucket/head.js deleted file mode 100644 index 32b8809603..0000000000 --- a/tests/functional/raw-node/test/GCP/bucket/head.js +++ /dev/null @@ -1,75 +0,0 @@ -const assert = require('assert'); -const arsenal = require('arsenal'); -const { GCP } = arsenal.storage.data.external.GCP; -const { genBucketName, gcpRetry } = require('../../../utils/gcpUtils'); -const { getRealAwsConfig } = - require('../../../../aws-node-sdk/test/support/awsConfig'); -const { - CreateBucketCommand, - DeleteBucketCommand, -} = require('@aws-sdk/client-s3'); - -const credentialOne = 'gcpbackend'; - -describe('GCP: HEAD Bucket', () => { - const config = getRealAwsConfig(credentialOne); - const gcpClient = new GCP(config); - - describe('without existing bucket', () => { - beforeEach(async function beforeFn() { - this.currentTest.bucketName = genBucketName('head'); - }); - - it('should return 404', async function testFn() { - const badBucketName = this.test.bucketName; - try { - await new Promise((resolve, reject) => { - gcpClient.headBucket({ Bucket: badBucketName }, - (err, res) => (err ? reject(err) : resolve(res))); - }); - assert.fail('Expected 404 error, but got success'); - } catch (err) { - assert(err); - assert.strictEqual(err.$metadata?.httpStatusCode, 404); - } - }); - }); - - describe('with existing bucket', () => { - beforeEach(async function beforeFn() { - this.currentTest.bucketName = genBucketName('head'); - process.stdout - .write(`Creating test bucket ${this.currentTest.bucketName}\n`); - await gcpRetry( - gcpClient, - new CreateBucketCommand({ - Bucket: this.currentTest.bucketName, - }), - ); - }); - - afterEach(async function afterFn() { - await gcpRetry( - gcpClient, - new DeleteBucketCommand({ - Bucket: this.currentTest.bucketName, - }), - ); - }); - - it('should get bucket information', async function testFn() { - const res = await new Promise((resolve, reject) => { - gcpClient.headBucket({ - Bucket: this.test.bucketName, - }, (err, data) => (err ? reject(err) : resolve(data))); - }); - const { $metadata, ...data } = res; - assert.strictEqual($metadata?.httpStatusCode, 200); - // Ensure MetaVersionId is present and non-empty - assert.ok( - typeof data.MetaVersionId === 'string' - && data.MetaVersionId.length > 0 - ); - }); - }); -}); diff --git a/tests/functional/raw-node/test/GCP/bucket/putVersioning.js b/tests/functional/raw-node/test/GCP/bucket/putVersioning.js deleted file mode 100644 index d9d4955984..0000000000 --- a/tests/functional/raw-node/test/GCP/bucket/putVersioning.js +++ /dev/null @@ -1,64 +0,0 @@ -const assert = require('assert'); -const arsenal = require('arsenal'); -const { - PutBucketVersioningCommand, - GetBucketVersioningCommand, - CreateBucketCommand, - DeleteBucketCommand, -} = require('@aws-sdk/client-s3'); -const { GCP } = arsenal.storage.data.external.GCP; -const { genBucketName, gcpRetry } = require('../../../utils/gcpUtils'); -const { getRealAwsConfig } = - require('../../../../aws-node-sdk/test/support/awsConfig'); - -const credentialOne = 'gcpbackend'; -const verEnabledStatus = 'Enabled'; -const verDisabledStatus = 'Suspended'; -const bucketName = genBucketName('putversioning'); - -describe('GCP: PUT Bucket Versioning', () => { - const config = getRealAwsConfig(credentialOne); - const gcpClient = new GCP(config); - - before(async () => { - await gcpRetry( - gcpClient, - new CreateBucketCommand({ Bucket: bucketName }), - ); - }); - - after(async () => { - await gcpRetry( - gcpClient, - new DeleteBucketCommand({ Bucket: bucketName }), - ); - }); - - it('should enable bucket versioning', async () => { - await gcpClient.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: { - Status: 'Enabled', - }, - })); - - const res = await gcpClient.send(new GetBucketVersioningCommand({ - Bucket: bucketName, - })); - assert.strictEqual(res.Status, verEnabledStatus); - }); - - it('should disable bucket versioning', async () => { - await gcpClient.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: { - Status: 'Suspended', - }, - })); - - const res = await gcpClient.send(new GetBucketVersioningCommand({ - Bucket: bucketName, - })); - assert.strictEqual(res.Status, verDisabledStatus); - }); -}); diff --git a/tests/functional/raw-node/test/GCP/bucket/versioning.js b/tests/functional/raw-node/test/GCP/bucket/versioning.js new file mode 100644 index 0000000000..45163787a4 --- /dev/null +++ b/tests/functional/raw-node/test/GCP/bucket/versioning.js @@ -0,0 +1,51 @@ +const assert = require('assert'); +const arsenal = require('arsenal'); +const { + PutBucketVersioningCommand, + GetBucketVersioningCommand, + CreateBucketCommand, + DeleteBucketCommand, +} = require('@aws-sdk/client-s3'); +const { GCP } = arsenal.storage.data.external.GCP; +const { genBucketName, gcpRetry } = require('../../../utils/gcpUtils'); +const { getRealAwsConfig } = + require('../../../../aws-node-sdk/test/support/awsConfig'); + +const credentialOne = 'gcpbackend'; +const config = getRealAwsConfig(credentialOne); +const gcpClient = new GCP(config); +const bucketName = genBucketName('versioning'); + +describe('GCP: Bucket Versioning', function testSuite() { + this.timeout(120000); + + before(async () => { + await gcpRetry(gcpClient, new CreateBucketCommand({ Bucket: bucketName })); + }); + + after(async () => { + await gcpRetry(gcpClient, new DeleteBucketCommand({ Bucket: bucketName })); + }); + + it('should enable bucket versioning', async () => { + await gcpClient.send(new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: { Status: 'Enabled' }, + })); + + const res = await gcpClient.send( + new GetBucketVersioningCommand({ Bucket: bucketName })); + assert.strictEqual(res.Status, 'Enabled'); + }); + + it('should disable bucket versioning', async () => { + await gcpClient.send(new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: { Status: 'Suspended' }, + })); + + const res = await gcpClient.send( + new GetBucketVersioningCommand({ Bucket: bucketName })); + assert.strictEqual(res.Status, 'Suspended'); + }); +}); diff --git a/tests/functional/raw-node/test/GCP/object/copy.js b/tests/functional/raw-node/test/GCP/object/copy.js deleted file mode 100644 index 06eac31a43..0000000000 --- a/tests/functional/raw-node/test/GCP/object/copy.js +++ /dev/null @@ -1,157 +0,0 @@ -const assert = require('assert'); -const async = require('async'); -const arsenal = require('arsenal'); -const { GCP } = arsenal.storage.data.external.GCP; -const { genUniqID, genBucketName, gcpRetry } = require('../../../utils/gcpUtils'); -const { getRealAwsConfig } = - require('../../../../aws-node-sdk/test/support/awsConfig'); -const { - CreateBucketCommand, - DeleteBucketCommand, - PutObjectCommand, -} = require('@aws-sdk/client-s3'); - -const credentialOne = 'gcpbackend'; -const bucketName = genBucketName('copy'); - -describe('GCP: COPY Object', function testSuite() { - this.timeout(180000); - const config = getRealAwsConfig(credentialOne); - const gcpClient = new GCP(config); - - before(async () => { - await gcpRetry( - gcpClient, - new CreateBucketCommand({ Bucket: bucketName }), - ); - }); - - after(async () => { - await gcpRetry( - gcpClient, - new DeleteBucketCommand({ Bucket: bucketName }), - ); - }); - - describe('without existing object in bucket', () => { - it('should return 404 and \'NoSuchKey\'', done => { - const missingObject = `nonexistingkey-${genUniqID()}`; - const someKey = `somekey-${genUniqID()}`; - gcpClient.copyObject({ - Bucket: bucketName, - Key: someKey, - CopySource: `/${bucketName}/${missingObject}`, - }, err => { - assert(err); - assert.strictEqual(err.$metadata.httpStatusCode, 404); - assert.strictEqual(err.name, 'NoSuchKey'); - return done(); - }); - }); - }); - - describe('with existing object in bucket', () => { - beforeEach(async function beforeFn() { - this.currentTest.key = `somekey-${genUniqID()}`; - this.currentTest.copyKey = `copykey-${genUniqID()}`; - this.currentTest.initValue = `${genUniqID()}`; - const cmd = new PutObjectCommand({ - Bucket: bucketName, - Key: this.currentTest.copyKey, - Metadata: { - value: this.currentTest.initValue, - }, - }); - const res = await gcpClient.send(cmd); - this.currentTest.ETag = res.ETag; - }); - - afterEach(function afterFn(done) { - async.parallel([ - next => gcpClient.deleteObject({ - Bucket: bucketName, - Key: this.currentTest.key, - }, err => { - if (err) { - process.stdout.write(`err in deleting object ${err}\n`); - } - return next(err); - }), - next => gcpClient.deleteObject({ - Bucket: bucketName, - Key: this.currentTest.copyKey, - }, err => { - if (err) { - process.stdout - .write(`err in deleting copy object ${err}\n`); - } - return next(err); - }), - ], done); - }); - - it('should successfully copy with REPLACE directive', - function testFn(done) { - const newValue = `${genUniqID()}`; - async.waterfall([ - next => gcpClient.copyObject({ - Bucket: bucketName, - Key: this.test.key, - CopySource: `/${bucketName}/${this.test.copyKey}`, - MetadataDirective: 'REPLACE', - Metadata: { - value: newValue, - }, - }, err => { - assert.equal(err, null, - `Expected success, but got error ${err}`); - return next(); - }), - next => gcpClient.headObject({ - Bucket: bucketName, - Key: this.test.key, - }, (err, res) => { - if (err) { - process.stdout - .write(`err in retrieving object ${err}\n`); - return next(err); - } - assert.strictEqual(res.ETag, this.test.ETag); - assert.notStrictEqual(res.Metadata.value, - this.test.initValue); - return next(); - }), - ], done); - }); - - it('should successfully copy with COPY directive', - function testFn(done) { - async.waterfall([ - next => gcpClient.copyObject({ - Bucket: bucketName, - Key: this.test.key, - CopySource: `/${bucketName}/${this.test.copyKey}`, - MetadataDirective: 'COPY', - }, err => { - assert.equal(err, null, - `Expected success, but got error ${err}`); - return next(); - }), - next => gcpClient.headObject({ - Bucket: bucketName, - Key: this.test.key, - }, (err, res) => { - if (err) { - process.stdout - .write(`err in retrieving object ${err}\n`); - return next(err); - } - assert.strictEqual(res.ETag, this.test.ETag); - assert.strictEqual(res.Metadata.value, - this.test.initValue); - return next(); - }), - ], done); - }); - }); -}); diff --git a/tests/functional/raw-node/test/GCP/object/delete.js b/tests/functional/raw-node/test/GCP/object/delete.js deleted file mode 100644 index 06bba2d2d9..0000000000 --- a/tests/functional/raw-node/test/GCP/object/delete.js +++ /dev/null @@ -1,94 +0,0 @@ -const assert = require('assert'); -const async = require('async'); -const arsenal = require('arsenal'); -const { GCP } = arsenal.storage.data.external.GCP; -const { genUniqID, genBucketName, gcpRetry } = require('../../../utils/gcpUtils'); -const { getRealAwsConfig } = - require('../../../../aws-node-sdk/test/support/awsConfig'); -const { - CreateBucketCommand, - DeleteBucketCommand, - PutObjectCommand, - GetObjectCommand, -} = require('@aws-sdk/client-s3'); - -const credentialOne = 'gcpbackend'; -const bucketName = genBucketName('deleteobj'); -const objectKey = `somekey-${genUniqID()}`; -const badObjectKey = `nonexistingkey-${genUniqID()}`; - -describe('GCP: DELETE Object', function testSuite() { - this.timeout(30000); - const config = getRealAwsConfig(credentialOne); - const gcpClient = new GCP(config); - - before(async () => { - await gcpRetry( - gcpClient, - new CreateBucketCommand({ Bucket: bucketName }), - ); - }); - - after(async () => { - await gcpRetry( - gcpClient, - new DeleteBucketCommand({ Bucket: bucketName }), - ); - }); - - describe('with existing object in bucket', () => { - beforeEach(async () => { - const cmd = new PutObjectCommand({ - Bucket: bucketName, - Key: objectKey, - }); - await gcpClient.send(cmd); - }); - - it('should successfully delete object', done => { - async.waterfall([ - next => gcpClient.deleteObject({ - Bucket: bucketName, - Key: objectKey, - }, err => { - assert.equal(err, null, - `Expected success, got error ${err}`); - return next(); - }), - next => { - const cmd = new GetObjectCommand({ - Bucket: bucketName, - Key: objectKey, - }); - gcpClient.send(cmd) - .then(() => { - // Should not succeed - assert.fail('Expected NoSuchKey error'); - }) - .catch(err => { - assert(err); - assert.strictEqual( - err.$metadata && err.$metadata.httpStatusCode, - 404); - assert.strictEqual(err.name, 'NoSuchKey'); - return next(); - }); - }, - ], err => done(err)); - }); - }); - - describe('without existing object in bucket', () => { - it('should return 404 and NoSuchKey', done => { - gcpClient.deleteObject({ - Bucket: bucketName, - Key: badObjectKey, - }, err => { - assert(err); - assert.strictEqual(err.$metadata.httpStatusCode, 404); - assert.strictEqual(err.name, 'NoSuchKey'); - return done(); - }); - }); - }); -}); diff --git a/tests/functional/raw-node/test/GCP/object/deleteMpu.js b/tests/functional/raw-node/test/GCP/object/deleteMpu.js index 597fdd6228..e4accccd6c 100644 --- a/tests/functional/raw-node/test/GCP/object/deleteMpu.js +++ b/tests/functional/raw-node/test/GCP/object/deleteMpu.js @@ -36,7 +36,7 @@ function gcpMpuSetupWrapper(params, callback) { } describe('GCP: Abort MPU', function testSuite() { - this.timeout(30000); + this.timeout(120000); let config; let gcpClient; diff --git a/tests/functional/raw-node/test/GCP/object/deleteTagging.js b/tests/functional/raw-node/test/GCP/object/deleteTagging.js deleted file mode 100644 index 1647d8d6a6..0000000000 --- a/tests/functional/raw-node/test/GCP/object/deleteTagging.js +++ /dev/null @@ -1,142 +0,0 @@ -const assert = require('assert'); -const async = require('async'); -const arsenal = require('arsenal'); -const { GCP } = arsenal.storage.data.external.GCP; -const { genDelTagObj, genUniqID, genBucketName, gcpRetry } = - require('../../../utils/gcpUtils'); -const { getRealAwsConfig } = - require('../../../../aws-node-sdk/test/support/awsConfig'); -const { gcpTaggingPrefix } = require('../../../../../../constants'); -const { - CreateBucketCommand, - DeleteBucketCommand, - PutObjectCommand, -} = require('@aws-sdk/client-s3'); - -const credentialOne = 'gcpbackend'; -const bucketName = genBucketName('deletetagging'); -let config; -let gcpClient; - -function assertObjectMetaTag(params, callback) { - return gcpClient.headObject({ - Bucket: params.bucket, - Key: params.key, - VersionId: params.versionId, - }, (err, res) => { - if (err) { - process.stdout.write(`err in retrieving object ${err}`); - return callback(err); - } - const resMeta = Object.assign({}, res.Metadata || {}); - const tagRes = {}; - const metaRes = {}; - Object.keys(resMeta).forEach(key => { - if (key.startsWith(gcpTaggingPrefix)) { - tagRes[key] = resMeta[key]; - } else { - metaRes[key] = resMeta[key]; - } - }); - assert.deepStrictEqual(params.tag, tagRes); - assert.deepStrictEqual(params.meta, metaRes); - return callback(); - }); -} - -describe('GCP: DELETE Object Tagging', function testSuite() { - this.timeout(30000); - - before(async () => { - config = getRealAwsConfig(credentialOne); - gcpClient = new GCP(config); - await gcpRetry( - gcpClient, - new CreateBucketCommand({ Bucket: bucketName }), - ); - }); - - beforeEach(async function beforeFn() { - this.currentTest.key = `somekey-${genUniqID()}`; - this.currentTest.specialKey = `veryspecial-${genUniqID()}`; - const { expectedTagObj, expectedMetaObj } = - genDelTagObj(10, `x-goog-meta-${gcpTaggingPrefix}`); - - const expectedTagMeta = {}; - Object.keys(expectedTagObj).forEach(header => { - const key = header.replace('x-goog-meta-', ''); - expectedTagMeta[key] = expectedTagObj[header]; - }); - - const expectedMetaMeta = {}; - Object.keys(expectedMetaObj).forEach(header => { - const key = header.replace('x-goog-meta-', ''); - expectedMetaMeta[key] = expectedMetaObj[header]; - }); - - this.currentTest.expectedTagObj = expectedTagMeta; - this.currentTest.expectedMetaObj = expectedMetaMeta; - - const metadata = Object.assign( - {}, - expectedTagMeta, - expectedMetaMeta - ); - - const cmd = new PutObjectCommand({ - Bucket: bucketName, - Key: this.currentTest.key, - Metadata: metadata, - }); - - const res = await gcpClient.send(cmd); - this.currentTest.versionId = res.VersionId; - }); - - afterEach(function afterFn(done) { - gcpClient.deleteObject({ - Bucket: bucketName, - Key: this.currentTest.key, - }, err => { - if (err) { - process.stdout.write(`err in deleting object ${err}`); - } - return done(err); - }); - }); - - after(async () => { - await gcpRetry( - gcpClient, - new DeleteBucketCommand({ Bucket: bucketName }), - ); - }); - - it('should successfully delete object tags', function testFn(done) { - async.waterfall([ - next => assertObjectMetaTag({ - bucket: bucketName, - key: this.test.key, - versionId: this.test.versionId, - meta: this.test.expectedMetaObj, - tag: this.test.expectedTagObj, - }, next), - next => gcpClient.deleteObjectTagging({ - Bucket: bucketName, - Key: this.test.key, - VersionId: this.test.versionId, - }, err => { - assert.equal(err, null, - `Expected success, got error ${err}`); - return next(); - }), - next => assertObjectMetaTag({ - bucket: bucketName, - key: this.test.key, - versionId: this.test.versionId, - meta: this.test.expectedMetaObj, - tag: {}, - }, next), - ], done); - }); -}); diff --git a/tests/functional/raw-node/test/GCP/object/get.js b/tests/functional/raw-node/test/GCP/object/get.js deleted file mode 100644 index 841511e979..0000000000 --- a/tests/functional/raw-node/test/GCP/object/get.js +++ /dev/null @@ -1,80 +0,0 @@ -const assert = require('assert'); -const arsenal = require('arsenal'); -const { GCP } = arsenal.storage.data.external.GCP; -const { genUniqID, genBucketName, gcpRetry } = require('../../../utils/gcpUtils'); -const { getRealAwsConfig } = - require('../../../../aws-node-sdk/test/support/awsConfig'); -const { - CreateBucketCommand, - DeleteBucketCommand, - PutObjectCommand, - DeleteObjectCommand, - GetObjectCommand, -} = require('@aws-sdk/client-s3'); - -const credentialOne = 'gcpbackend'; -const bucketName = genBucketName('get'); - -describe('GCP: GET Object', function testSuite() { - this.timeout(30000); - const config = getRealAwsConfig(credentialOne); - const gcpClient = new GCP(config); - - before(async () => { - await gcpRetry( - gcpClient, - new CreateBucketCommand({ Bucket: bucketName }), - ); - }); - - after(async () => { - const cmd = new DeleteBucketCommand({ Bucket: bucketName }); - await gcpClient.send(cmd); - }); - - describe('with existing object in bucket', () => { - beforeEach(async function beforeFn() { - this.currentTest.key = `somekey-${genUniqID()}`; - const cmd = new PutObjectCommand({ - Bucket: bucketName, - Key: this.currentTest.key, - }); - const res = await gcpClient.send(cmd); - this.currentTest.uploadId = res.VersionId; - this.currentTest.ETag = res.ETag; - }); - - afterEach(async function afterFn() { - const cmd = new DeleteObjectCommand({ - Bucket: bucketName, - Key: this.currentTest.key, - }); - await gcpClient.send(cmd); - }); - - it('should successfully retrieve object', async function testFn() { - const cmd = new GetObjectCommand({ - Bucket: bucketName, - Key: this.test.key, - }); - const res = await gcpClient.send(cmd); - assert.strictEqual(res.ETag, this.test.ETag); - assert.strictEqual(res.VersionId, this.test.uploadId); - }); - }); - - describe('without existing object in bucket', () => { - it('should return 404 and NoSuchKey', done => { - const badObjectKey = `nonexistingkey-${genUniqID()}`; - gcpClient.getObject({ - Bucket: bucketName, - Key: badObjectKey, - }, err => { - assert(err); - assert.strictEqual(err.$metadata?.httpStatusCode, 404); - assert.strictEqual(err.name, 'NoSuchKey'); - return done(); - }); - }); - }); -}); diff --git a/tests/functional/raw-node/test/GCP/object/getTagging.js b/tests/functional/raw-node/test/GCP/object/getTagging.js deleted file mode 100644 index 96b640c656..0000000000 --- a/tests/functional/raw-node/test/GCP/object/getTagging.js +++ /dev/null @@ -1,103 +0,0 @@ -const assert = require('assert'); -const arsenal = require('arsenal'); -const { GCP } = arsenal.storage.data.external.GCP; -const { genGetTagObj, genUniqID, genBucketName, gcpRetry } = - require('../../../utils/gcpUtils'); -const { getRealAwsConfig } = - require('../../../../aws-node-sdk/test/support/awsConfig'); -const { gcpTaggingPrefix } = require('../../../../../../constants'); -const { - CreateBucketCommand, - DeleteBucketCommand, - PutObjectCommand, -} = require('@aws-sdk/client-s3'); - -const credentialOne = 'gcpbackend'; -const bucketName = genBucketName('gettagging'); -const tagSize = 10; - -describe('GCP: GET Object Tagging', () => { - let config; - let gcpClient; - let bucketCreated = false; - - before(async () => { - config = getRealAwsConfig(credentialOne); - gcpClient = new GCP(config); - await gcpRetry( - gcpClient, - new CreateBucketCommand({ Bucket: bucketName }), - ); - bucketCreated = true; - }); - - beforeEach(async function beforeFn() { - this.currentTest.key = `somekey-${genUniqID()}`; - this.currentTest.specialKey = `veryspecial-${genUniqID()}`; - const { expectedTagObj } = - genGetTagObj(tagSize, `x-goog-meta-${gcpTaggingPrefix}`); - this.currentTest.tagObj = expectedTagObj; - - const putCmd = new PutObjectCommand({ - Bucket: bucketName, - Key: this.currentTest.key, - }); - - const res = await gcpClient.send(putCmd); - this.currentTest.versionId = res.VersionId; - - await new Promise((resolve, reject) => { - gcpClient.putObjectTagging({ - Bucket: bucketName, - Key: this.currentTest.key, - VersionId: this.currentTest.versionId, - Tagging: { - TagSet: this.currentTest.tagObj, - }, - }, err => { - if (err) { - process.stdout - .write(`err in setting object tags ${err}`); - reject(err); - return; - } - resolve(); - }); - }); - }); - - afterEach(function afterFn(done) { - gcpClient.deleteObject({ - Bucket: bucketName, - Key: this.currentTest.key, - }, err => { - if (err) { - process.stdout.write(`err in deleting object ${err}`); - } - return done(err); - }); - }); - - after(async () => { - if (!bucketCreated) { - return; - } - await gcpRetry( - gcpClient, - new DeleteBucketCommand({ Bucket: bucketName }), - ); - }); - - it('should successfully get object tags', function testFn(done) { - gcpClient.getObjectTagging({ - Bucket: bucketName, - Key: this.test.key, - VersionId: this.test.versionId, - }, (err, res) => { - assert.equal(err, null, - `Expected success, got error ${err}`); - assert.deepStrictEqual(res.TagSet, this.test.tagObj); - return done(); - }); - }); -}); diff --git a/tests/functional/raw-node/test/GCP/object/head.js b/tests/functional/raw-node/test/GCP/object/head.js deleted file mode 100644 index a168994e91..0000000000 --- a/tests/functional/raw-node/test/GCP/object/head.js +++ /dev/null @@ -1,91 +0,0 @@ -const assert = require('assert'); -const arsenal = require('arsenal'); -const { GCP } = arsenal.storage.data.external.GCP; -const { genUniqID, genBucketName, gcpRetry } = require('../../../utils/gcpUtils'); -const { getRealAwsConfig } = - require('../../../../aws-node-sdk/test/support/awsConfig'); -const { - CreateBucketCommand, - DeleteBucketCommand, - PutObjectCommand, - HeadObjectCommand, -} = require('@aws-sdk/client-s3'); - -const credentialOne = 'gcpbackend'; -const bucketName = genBucketName('head'); - -describe('GCP: HEAD Object', function testSuite() { - this.timeout(30000); - const config = getRealAwsConfig(credentialOne); - const gcpClient = new GCP(config); - - before(async () => { - await gcpRetry( - gcpClient, - new CreateBucketCommand({ Bucket: bucketName }), - ); - }); - - after(async () => { - const cmd = new DeleteBucketCommand({ Bucket: bucketName }); - await gcpClient.send(cmd); - }); - - describe('with existing object in bucket', () => { - beforeEach(async function beforeFn() { - this.currentTest.key = `somekey-${genUniqID()}`; - const cmd = new PutObjectCommand({ - Bucket: bucketName, - Key: this.currentTest.key, - }); - const res = await gcpClient.send(cmd); - this.currentTest.uploadId = res.VersionId; - this.currentTest.ETag = res.ETag; - }); - - afterEach(async function afterFn() { - if (!this.currentTest.key) { - return; - } - await new Promise((resolve, reject) => { - gcpClient.deleteObject({ - Bucket: bucketName, - Key: this.currentTest.key, - }, err => { - if (err) { - process.stdout.write(`err in deleting object ${err}\n`); - reject(err); - return; - } - resolve(); - }); - }); - }); - - it('should successfully retrieve object', async function testFn() { - const cmd = new HeadObjectCommand({ - Bucket: bucketName, - Key: this.test.key, - }); - const res = await gcpClient.send(cmd); - assert.strictEqual(res.ETag, this.test.ETag); - assert.ok(res.$metadata && res.$metadata.httpStatusCode === 200); - }); - }); - - describe('without existing object in bucket', () => { - it('should return 404', async () => { - const badObjectkey = `nonexistingkey-${genUniqID()}`; - await new Promise(resolve => { - gcpClient.headObject({ - Bucket: bucketName, - Key: badObjectkey, - }, err => { - assert(err); - assert.strictEqual(err.$metadata.httpStatusCode, 404); - resolve(); - }); - }); - }); - }); -}); diff --git a/tests/functional/raw-node/test/GCP/object/object.js b/tests/functional/raw-node/test/GCP/object/object.js new file mode 100644 index 0000000000..5cc3e445b1 --- /dev/null +++ b/tests/functional/raw-node/test/GCP/object/object.js @@ -0,0 +1,355 @@ +const assert = require('assert'); +const async = require('async'); +const arsenal = require('arsenal'); +const { GCP } = arsenal.storage.data.external.GCP; +const { genUniqID, genBucketName, gcpRetry } = require('../../../utils/gcpUtils'); +const { getRealAwsConfig } = + require('../../../../aws-node-sdk/test/support/awsConfig'); +const { + CreateBucketCommand, + DeleteBucketCommand, + PutObjectCommand, + HeadObjectCommand, + DeleteObjectCommand, + GetObjectCommand, +} = require('@aws-sdk/client-s3'); + +const credentialOne = 'gcpbackend'; + +describe('GCP: Object', function testSuite() { + this.timeout(180000); + const config = getRealAwsConfig(credentialOne); + const gcpClient = new GCP(config); + const bucketName = genBucketName('object'); + + before(async () => { + await gcpRetry( + gcpClient, + new CreateBucketCommand({ Bucket: bucketName }), + ); + }); + + after(async () => { + await gcpRetry(gcpClient, new DeleteBucketCommand({ Bucket: bucketName })); + }); + + async function setupExistingObject(test) { + /* eslint-disable no-param-reassign */ + test.key = `somekey-${genUniqID()}`; + const res = await gcpClient.send(new PutObjectCommand({ + Bucket: bucketName, + Key: test.key, + })); + test.uploadId = res.VersionId; + test.ETag = res.ETag; + /* eslint-enable no-param-reassign */ + } + + async function cleanupObject(test) { + if (!test.key) { + return; + } + await gcpClient.send(new DeleteObjectCommand({ + Bucket: bucketName, + Key: test.key, + })); + } + + describe('HEAD Object', () => { + describe('with existing object in bucket', () => { + beforeEach(async function beforeFn() { + await setupExistingObject(this.currentTest); + }); + + afterEach(async function afterFn() { + await cleanupObject(this.currentTest); + }); + + it('should successfully retrieve object', async function testFn() { + const res = await gcpClient.send(new HeadObjectCommand({ + Bucket: bucketName, + Key: this.test.key, + })); + assert.strictEqual(res.ETag, this.test.ETag); + assert.ok(res.$metadata && res.$metadata.httpStatusCode === 200); + }); + }); + + describe('without existing object in bucket', () => { + it('should return 404', async () => { + const badObjectKey = `nonexistingkey-${genUniqID()}`; + await new Promise(resolve => { + gcpClient.headObject({ + Bucket: bucketName, + Key: badObjectKey, + }, err => { + assert(err); + assert.strictEqual(err.$metadata.httpStatusCode, 404); + resolve(); + }); + }); + }); + }); + }); + + describe('GET Object', () => { + describe('with existing object in bucket', () => { + beforeEach(async function beforeFn() { + await setupExistingObject(this.currentTest); + }); + + afterEach(async function afterFn() { + await cleanupObject(this.currentTest); + }); + + it('should successfully retrieve object', async function testFn() { + const res = await gcpClient.send(new GetObjectCommand({ + Bucket: bucketName, + Key: this.test.key, + })); + assert.strictEqual(res.ETag, this.test.ETag); + assert.strictEqual(res.VersionId, this.test.uploadId); + }); + }); + + describe('without existing object in bucket', () => { + it('should return 404 and NoSuchKey', done => { + const badObjectKey = `nonexistingkey-${genUniqID()}`; + gcpClient.getObject({ + Bucket: bucketName, + Key: badObjectKey, + }, err => { + assert(err); + assert.strictEqual(err.$metadata?.httpStatusCode, 404); + assert.strictEqual(err.name, 'NoSuchKey'); + return done(); + }); + }); + }); + }); + + describe('PUT Object', () => { + afterEach(async function afterFn() { + await cleanupObject(this.currentTest); + }); + + describe('with existing object in bucket', () => { + beforeEach(async function beforeFn() { + await setupExistingObject(this.currentTest); + }); + + it('should overwrite object', function testFn(done) { + gcpClient.putObject({ + Bucket: bucketName, + Key: this.test.key, + }, (err, res) => { + assert.notStrictEqual(res.VersionId, this.test.uploadId); + return done(); + }); + }); + }); + + describe('without existing object in bucket', () => { + it('should successfully put object', function testFn(done) { + this.test.key = `somekey-${genUniqID()}`; + gcpClient.putObject({ + Bucket: bucketName, + Key: this.test.key, + }, (err, putRes) => { + assert.equal(err, null, + `Expected success, got error ${err}`); + gcpClient.getObject({ + Bucket: bucketName, + Key: this.test.key, + }, (getErr, getRes) => { + assert.equal(getErr, null, + `Expected success, got error ${getErr}`); + assert.strictEqual(getRes.VersionId, putRes.VersionId); + return done(); + }); + }); + }); + }); + }); + + describe('DELETE Object', () => { + const objectKey = `somekey-${genUniqID()}`; + const badObjectKey = `nonexistingkey-${genUniqID()}`; + + describe('with existing object in bucket', () => { + beforeEach(async () => { + await gcpClient.send(new PutObjectCommand({ + Bucket: bucketName, + Key: objectKey, + })); + }); + + it('should successfully delete object', done => { + async.waterfall([ + next => gcpClient.deleteObject({ + Bucket: bucketName, + Key: objectKey, + }, err => { + assert.equal(err, null, + `Expected success, got error ${err}`); + return next(); + }), + next => { + gcpClient.send(new GetObjectCommand({ + Bucket: bucketName, + Key: objectKey, + })) + .then(() => { + assert.fail('Expected NoSuchKey error'); + }) + .catch(err => { + assert(err); + assert.strictEqual( + err.$metadata && err.$metadata.httpStatusCode, + 404); + assert.strictEqual(err.name, 'NoSuchKey'); + return next(); + }); + }, + ], err => done(err)); + }); + }); + + describe('without existing object in bucket', () => { + it('should return 404 and NoSuchKey', done => { + gcpClient.deleteObject({ + Bucket: bucketName, + Key: badObjectKey, + }, err => { + assert(err); + assert.strictEqual(err.$metadata.httpStatusCode, 404); + assert.strictEqual(err.name, 'NoSuchKey'); + return done(); + }); + }); + }); + }); + + describe('COPY Object', () => { + describe('without existing object in bucket', () => { + it('should return 404 and \'NoSuchKey\'', done => { + const missingObject = `nonexistingkey-${genUniqID()}`; + const someKey = `somekey-${genUniqID()}`; + gcpClient.copyObject({ + Bucket: bucketName, + Key: someKey, + CopySource: `/${bucketName}/${missingObject}`, + }, err => { + assert(err); + assert.strictEqual(err.$metadata.httpStatusCode, 404); + assert.strictEqual(err.name, 'NoSuchKey'); + return done(); + }); + }); + }); + + describe('with existing object in bucket', () => { + beforeEach(async function beforeFn() { + this.currentTest.key = `somekey-${genUniqID()}`; + this.currentTest.copyKey = `copykey-${genUniqID()}`; + this.currentTest.initValue = `${genUniqID()}`; + const res = await gcpClient.send(new PutObjectCommand({ + Bucket: bucketName, + Key: this.currentTest.copyKey, + Metadata: { + value: this.currentTest.initValue, + }, + })); + this.currentTest.ETag = res.ETag; + }); + + afterEach(function afterFn(done) { + async.parallel([ + next => gcpClient.deleteObject({ + Bucket: bucketName, + Key: this.currentTest.key, + }, err => { + if (err) { + process.stdout.write(`err in deleting object ${err}\n`); + } + return next(err); + }), + next => gcpClient.deleteObject({ + Bucket: bucketName, + Key: this.currentTest.copyKey, + }, err => { + if (err) { + process.stdout + .write(`err in deleting copy object ${err}\n`); + } + return next(err); + }), + ], done); + }); + + it('should successfully copy with REPLACE directive', + function testFn(done) { + const newValue = `${genUniqID()}`; + async.waterfall([ + next => gcpClient.copyObject({ + Bucket: bucketName, + Key: this.test.key, + CopySource: `/${bucketName}/${this.test.copyKey}`, + MetadataDirective: 'REPLACE', + Metadata: { + value: newValue, + }, + }, err => { + assert.equal(err, null, + `Expected success, but got error ${err}`); + return next(); + }), + next => gcpClient.headObject({ + Bucket: bucketName, + Key: this.test.key, + }, (err, res) => { + if (err) { + process.stdout + .write(`err in retrieving object ${err}\n`); + return next(err); + } + assert.strictEqual(res.ETag, this.test.ETag); + assert.notStrictEqual(res.Metadata.value, + this.test.initValue); + return next(); + }), + ], done); + }); + + it('should successfully copy with COPY directive', + function testFn(done) { + async.waterfall([ + next => gcpClient.copyObject({ + Bucket: bucketName, + Key: this.test.key, + CopySource: `/${bucketName}/${this.test.copyKey}`, + MetadataDirective: 'COPY', + }, err => { + assert.equal(err, null, + `Expected success, but got error ${err}`); + return next(); + }), + next => gcpClient.headObject({ + Bucket: bucketName, + Key: this.test.key, + }, (err, res) => { + if (err) { + process.stdout + .write(`err in retrieving object ${err}\n`); + return next(err); + } + assert.strictEqual(res.ETag, this.test.ETag); + assert.strictEqual(res.Metadata.value, + this.test.initValue); + return next(); + }), + ], done); + }); + }); + }); +}); diff --git a/tests/functional/raw-node/test/GCP/object/put.js b/tests/functional/raw-node/test/GCP/object/put.js deleted file mode 100644 index 6747401447..0000000000 --- a/tests/functional/raw-node/test/GCP/object/put.js +++ /dev/null @@ -1,92 +0,0 @@ -const assert = require('assert'); -const arsenal = require('arsenal'); -const { GCP } = arsenal.storage.data.external.GCP; -const { genUniqID, genBucketName, gcpRetry } = require('../../../utils/gcpUtils'); -const { getRealAwsConfig } = - require('../../../../aws-node-sdk/test/support/awsConfig'); -const { - CreateBucketCommand, - DeleteBucketCommand, - PutObjectCommand, -} = require('@aws-sdk/client-s3'); - -const credentialOne = 'gcpbackend'; -const bucketName = genBucketName('put'); - -describe('GCP: PUT Object', function testSuite() { - this.timeout(30000); - const config = getRealAwsConfig(credentialOne); - const gcpClient = new GCP(config); - - before(async () => { - await gcpRetry( - gcpClient, - new CreateBucketCommand({ Bucket: bucketName }), - ); - }); - - after(async () => { - const cmd = new DeleteBucketCommand({ Bucket: bucketName }); - await gcpClient.send(cmd); - }); - - afterEach(function afterFn(done) { - if (!this.currentTest.key) { - done(); - return; - } - gcpClient.deleteObject({ - Bucket: bucketName, - Key: this.currentTest.key, - }, err => { - if (err) { - process.stdout.write(`err in deleting object ${err}\n`); - } - return done(err); - }); - }); - - describe('with existing object in bucket', () => { - beforeEach(async function beforeFn() { - this.currentTest.key = `somekey-${genUniqID()}`; - const cmd = new PutObjectCommand({ - Bucket: bucketName, - Key: this.currentTest.key, - }); - const res = await gcpClient.send(cmd); - this.currentTest.uploadId = res.VersionId; - }); - - it('should overwrite object', function testFn(done) { - gcpClient.putObject({ - Bucket: bucketName, - Key: this.test.key, - }, (err, res) => { - assert.notStrictEqual(res.VersionId, this.test.uploadId); - return done(); - }); - }); - }); - - describe('without existing object in bucket', () => { - it('should successfully put object', function testFn(done) { - this.test.key = `somekey-${genUniqID()}`; - gcpClient.putObject({ - Bucket: bucketName, - Key: this.test.key, - }, (err, putRes) => { - assert.equal(err, null, - `Expected success, got error ${err}`); - gcpClient.getObject({ - Bucket: bucketName, - Key: this.test.key, - }, (getErr, getRes) => { - assert.equal(getErr, null, - `Expected success, got error ${getErr}`); - assert.strictEqual(getRes.VersionId, putRes.VersionId); - return done(); - }); - }); - }); - }); -}); diff --git a/tests/functional/raw-node/test/GCP/object/putTagging.js b/tests/functional/raw-node/test/GCP/object/putTagging.js deleted file mode 100644 index a5d1d68960..0000000000 --- a/tests/functional/raw-node/test/GCP/object/putTagging.js +++ /dev/null @@ -1,171 +0,0 @@ -const assert = require('assert'); -const async = require('async'); -const arsenal = require('arsenal'); -const { GCP } = arsenal.storage.data.external.GCP; -const { genPutTagObj, genUniqID, genBucketName, gcpRetry } = - require('../../../utils/gcpUtils'); -const { getRealAwsConfig } = - require('../../../../aws-node-sdk/test/support/awsConfig'); -const { gcpTaggingPrefix } = require('../../../../../../constants'); -const { - CreateBucketCommand, - DeleteBucketCommand, - PutObjectCommand, -} = require('@aws-sdk/client-s3'); - -const credentialOne = 'gcpbackend'; -const bucketName = genBucketName('puttagging'); - -describe('GCP: PUT Object Tagging', () => { - let config; - let gcpClient; - - before(async () => { - config = getRealAwsConfig(credentialOne); - gcpClient = new GCP(config); - await gcpRetry( - gcpClient, - new CreateBucketCommand({ Bucket: bucketName }), - ); - }); - - beforeEach(async function beforeFn() { - this.currentTest.key = `somekey-${genUniqID()}`; - this.currentTest.specialKey = `veryspecial-${genUniqID()}`; - const cmd = new PutObjectCommand({ - Bucket: bucketName, - Key: this.currentTest.key, - }); - const res = await gcpClient.send(cmd); - this.currentTest.versionId = res.VersionId; - }); - - afterEach(function afterFn(done) { - gcpClient.deleteObject({ - Bucket: bucketName, - Key: this.currentTest.key, - }, err => { - if (err) { - process.stdout.write(`err in deleting object ${err}`); - } - return done(err); - }); - }); - - after(async () => { - await gcpRetry( - gcpClient, - new DeleteBucketCommand({ Bucket: bucketName }), - ); - }); - - it('should successfully put object tags', function testFn(done) { - async.waterfall([ - next => gcpClient.putObjectTagging({ - Bucket: bucketName, - Key: this.test.key, - VersionId: this.test.versionId, - Tagging: { - TagSet: [ - { - Key: this.test.specialKey, - Value: this.test.specialKey, - }, - ], - }, - }, err => { - assert.equal(err, null, - `Expected success, got error ${err}`); - return next(); - }), - next => gcpClient.headObject({ - Bucket: bucketName, - Key: this.test.key, - VersionId: this.test.versionId, - }, (err, res) => { - if (err) { - process.stdout.write(`err in retrieving object ${err}`); - return next(err); - } - const metaKey = `${gcpTaggingPrefix}${this.test.specialKey}`; - const toCompare = res.Metadata[metaKey]; - assert.strictEqual(toCompare, this.test.specialKey); - return next(); - }), - ], done); - }); - - describe('when tagging parameter is incorrect', () => { - it('should return 400 and BadRequest if more than ' + - '10 tags are given', function testFun(done) { - return gcpClient.putObjectTagging({ - Bucket: bucketName, - Key: this.test.key, - VersionId: this.test.versionId, - Tagging: { - TagSet: genPutTagObj(11), - }, - }, err => { - assert(err); - assert.strictEqual(err.code, 400); - assert.strictEqual(err.message, 'BadRequest'); - return done(); - }); - }); - - it('should return 400 and InvalidTag if given duplicate keys', - function testFn(done) { - return gcpClient.putObjectTagging({ - Bucket: bucketName, - Key: this.test.key, - VersionId: this.test.versionId, - Tagging: { - TagSet: genPutTagObj(10, true), - }, - }, err => { - assert(err); - assert.strictEqual(err.code, 400); - assert.strictEqual(err.message, 'InvalidTag'); - return done(); - }); - }); - - it('should return 400 and InvalidTag if given invalid key', - function testFn(done) { - return gcpClient.putObjectTagging({ - Bucket: bucketName, - Key: this.test.key, - VersionId: this.test.versionId, - Tagging: { - TagSet: [ - { Key: Buffer.alloc(129, 'a'), Value: 'bad tag' }, - ], - }, - }, err => { - assert(err); - assert.strictEqual(err.code, 400); - assert.strictEqual(err.message, 'InvalidTag'); - return done(); - }); - }); - - it('should return 400 and InvalidTag if given invalid value', - function testFn(done) { - return gcpClient.putObjectTagging({ - Bucket: bucketName, - Key: this.test.key, - VersionId: this.test.versionId, - Tagging: { - TagSet: [ - { Key: 'badtag', Value: Buffer.alloc(257, 'a') }, - ], - }, - }, err => { - assert(err); - assert.strictEqual(err.code, 400); - assert.strictEqual(err.message, 'InvalidTag'); - return done(); - }); - }); - }); -}); diff --git a/tests/functional/raw-node/test/GCP/object/tagging.js b/tests/functional/raw-node/test/GCP/object/tagging.js new file mode 100644 index 0000000000..50f0d4c9e7 --- /dev/null +++ b/tests/functional/raw-node/test/GCP/object/tagging.js @@ -0,0 +1,306 @@ +const assert = require('assert'); +const async = require('async'); +const arsenal = require('arsenal'); +const { GCP } = arsenal.storage.data.external.GCP; +const { genPutTagObj, genGetTagObj, genDelTagObj, genUniqID, genBucketName, gcpRetry } = + require('../../../utils/gcpUtils'); +const { getRealAwsConfig } = + require('../../../../aws-node-sdk/test/support/awsConfig'); +const { gcpTaggingPrefix } = require('../../../../../../constants'); +const { + CreateBucketCommand, + DeleteBucketCommand, + PutObjectCommand, +} = require('@aws-sdk/client-s3'); + +const credentialOne = 'gcpbackend'; + +describe('GCP: Object Tagging', function testSuite() { + this.timeout(120000); + const config = getRealAwsConfig(credentialOne); + const gcpClient = new GCP(config); + const bucketName = genBucketName('tagging'); + + before(async () => { + await gcpRetry( + gcpClient, + new CreateBucketCommand({ Bucket: bucketName }), + ); + }); + + after(async () => { + await gcpRetry( + gcpClient, + new DeleteBucketCommand({ Bucket: bucketName }), + ); + }); + + afterEach(function afterFn(done) { + gcpClient.deleteObject({ + Bucket: bucketName, + Key: this.currentTest.key, + }, err => { + if (err) { + process.stdout.write(`err in deleting object ${err}`); + } + return done(err); + }); + }); + + describe('PUT Object Tagging', () => { + beforeEach(async function beforeFn() { + this.currentTest.key = `somekey-${genUniqID()}`; + this.currentTest.specialKey = `veryspecial-${genUniqID()}`; + const res = await gcpClient.send(new PutObjectCommand({ + Bucket: bucketName, + Key: this.currentTest.key, + })); + this.currentTest.versionId = res.VersionId; + }); + + it('should successfully put object tags', function testFn(done) { + async.waterfall([ + next => gcpClient.putObjectTagging({ + Bucket: bucketName, + Key: this.test.key, + VersionId: this.test.versionId, + Tagging: { + TagSet: [ + { + Key: this.test.specialKey, + Value: this.test.specialKey, + }, + ], + }, + }, err => { + assert.equal(err, null, + `Expected success, got error ${err}`); + return next(); + }), + next => gcpClient.headObject({ + Bucket: bucketName, + Key: this.test.key, + VersionId: this.test.versionId, + }, (err, res) => { + if (err) { + process.stdout.write(`err in retrieving object ${err}`); + return next(err); + } + const metaKey = `${gcpTaggingPrefix}${this.test.specialKey}`; + const toCompare = res.Metadata[metaKey]; + assert.strictEqual(toCompare, this.test.specialKey); + return next(); + }), + ], done); + }); + + describe('when tagging parameter is incorrect', () => { + it('should return 400 and BadRequest if more than ' + + '10 tags are given', function testFun(done) { + return gcpClient.putObjectTagging({ + Bucket: bucketName, + Key: this.test.key, + VersionId: this.test.versionId, + Tagging: { + TagSet: genPutTagObj(11), + }, + }, err => { + assert(err); + assert.strictEqual(err.code, 400); + assert.strictEqual(err.message, 'BadRequest'); + return done(); + }); + }); + + it('should return 400 and InvalidTag if given duplicate keys', + function testFn(done) { + return gcpClient.putObjectTagging({ + Bucket: bucketName, + Key: this.test.key, + VersionId: this.test.versionId, + Tagging: { + TagSet: genPutTagObj(10, true), + }, + }, err => { + assert(err); + assert.strictEqual(err.code, 400); + assert.strictEqual(err.message, 'InvalidTag'); + return done(); + }); + }); + + it('should return 400 and InvalidTag if given invalid key', + function testFn(done) { + return gcpClient.putObjectTagging({ + Bucket: bucketName, + Key: this.test.key, + VersionId: this.test.versionId, + Tagging: { + TagSet: [ + { Key: Buffer.alloc(129, 'a'), Value: 'bad tag' }, + ], + }, + }, err => { + assert(err); + assert.strictEqual(err.code, 400); + assert.strictEqual(err.message, 'InvalidTag'); + return done(); + }); + }); + + it('should return 400 and InvalidTag if given invalid value', + function testFn(done) { + return gcpClient.putObjectTagging({ + Bucket: bucketName, + Key: this.test.key, + VersionId: this.test.versionId, + Tagging: { + TagSet: [ + { Key: 'badtag', Value: Buffer.alloc(257, 'a') }, + ], + }, + }, err => { + assert(err); + assert.strictEqual(err.code, 400); + assert.strictEqual(err.message, 'InvalidTag'); + return done(); + }); + }); + }); + }); + + describe('GET Object Tagging', () => { + const tagSize = 10; + + beforeEach(async function beforeFn() { + this.currentTest.key = `somekey-${genUniqID()}`; + this.currentTest.specialKey = `veryspecial-${genUniqID()}`; + const { expectedTagObj } = + genGetTagObj(tagSize, `x-goog-meta-${gcpTaggingPrefix}`); + this.currentTest.tagObj = expectedTagObj; + + const res = await gcpClient.send(new PutObjectCommand({ + Bucket: bucketName, + Key: this.currentTest.key, + })); + this.currentTest.versionId = res.VersionId; + + await new Promise((resolve, reject) => { + gcpClient.putObjectTagging({ + Bucket: bucketName, + Key: this.currentTest.key, + VersionId: this.currentTest.versionId, + Tagging: { + TagSet: this.currentTest.tagObj, + }, + }, err => { + if (err) { + process.stdout + .write(`err in setting object tags ${err}`); + reject(err); + return; + } + resolve(); + }); + }); + }); + + it('should successfully get object tags', function testFn(done) { + gcpClient.getObjectTagging({ + Bucket: bucketName, + Key: this.test.key, + VersionId: this.test.versionId, + }, (err, res) => { + assert.equal(err, null, + `Expected success, got error ${err}`); + assert.deepStrictEqual(res.TagSet, this.test.tagObj); + return done(); + }); + }); + }); + + describe('DELETE Object Tagging', () => { + function assertObjectMetaTag(params, callback) { + return gcpClient.headObject({ + Bucket: params.bucket, + Key: params.key, + VersionId: params.versionId, + }, (err, res) => { + if (err) { + process.stdout.write(`err in retrieving object ${err}`); + return callback(err); + } + const resMeta = Object.assign({}, res.Metadata || {}); + const tagRes = {}; + const metaRes = {}; + Object.keys(resMeta).forEach(key => { + if (key.startsWith(gcpTaggingPrefix)) { + tagRes[key] = resMeta[key]; + } else { + metaRes[key] = resMeta[key]; + } + }); + assert.deepStrictEqual(params.tag, tagRes); + assert.deepStrictEqual(params.meta, metaRes); + return callback(); + }); + } + + beforeEach(async function beforeFn() { + this.currentTest.key = `somekey-${genUniqID()}`; + this.currentTest.specialKey = `veryspecial-${genUniqID()}`; + const { expectedTagObj, expectedMetaObj } = + genDelTagObj(10, `x-goog-meta-${gcpTaggingPrefix}`); + + const expectedTagMeta = {}; + Object.keys(expectedTagObj).forEach(header => { + const key = header.replace('x-goog-meta-', ''); + expectedTagMeta[key] = expectedTagObj[header]; + }); + + const expectedMetaMeta = {}; + Object.keys(expectedMetaObj).forEach(header => { + const key = header.replace('x-goog-meta-', ''); + expectedMetaMeta[key] = expectedMetaObj[header]; + }); + + this.currentTest.expectedTagObj = expectedTagMeta; + this.currentTest.expectedMetaObj = expectedMetaMeta; + + const res = await gcpClient.send(new PutObjectCommand({ + Bucket: bucketName, + Key: this.currentTest.key, + Metadata: Object.assign({}, expectedTagMeta, expectedMetaMeta), + })); + this.currentTest.versionId = res.VersionId; + }); + + it('should successfully delete object tags', function testFn(done) { + async.waterfall([ + next => assertObjectMetaTag({ + bucket: bucketName, + key: this.test.key, + versionId: this.test.versionId, + meta: this.test.expectedMetaObj, + tag: this.test.expectedTagObj, + }, next), + next => gcpClient.deleteObjectTagging({ + Bucket: bucketName, + Key: this.test.key, + VersionId: this.test.versionId, + }, err => { + assert.equal(err, null, + `Expected success, got error ${err}`); + return next(); + }), + next => assertObjectMetaTag({ + bucket: bucketName, + key: this.test.key, + versionId: this.test.versionId, + meta: this.test.expectedMetaObj, + tag: {}, + }, next), + ], done); + }); + }); +}); diff --git a/tests/functional/raw-node/utils/gcpUtils.js b/tests/functional/raw-node/utils/gcpUtils.js index c0a5104fb9..773fd0ec83 100644 --- a/tests/functional/raw-node/utils/gcpUtils.js +++ b/tests/functional/raw-node/utils/gcpUtils.js @@ -16,7 +16,8 @@ const defaultShouldRetry = err => async function gcpRetryCall(callFn, retryOptions) { const { - maxAttempts = 3, + // 6 attempts with exponential backoff gives up to ~63s of total wait. + maxAttempts = 6, shouldRetry = defaultShouldRetry, getDelayMs = attempt => Math.pow(2, attempt) * 1000, } = retryOptions || {};