diff --git a/lib/api/apiUtils/object/createAndStoreObject.js b/lib/api/apiUtils/object/createAndStoreObject.js index 542b9f5296..f306863a8e 100644 --- a/lib/api/apiUtils/object/createAndStoreObject.js +++ b/lib/api/apiUtils/object/createAndStoreObject.js @@ -14,11 +14,49 @@ const { config } = require('../../../Config'); const validateWebsiteHeader = require('./websiteServing') .validateWebsiteHeader; const applyZenkoUserMD = require('./applyZenkoUserMD'); +const { + algorithms, + getChecksumDataFromHeaders, + arsenalErrorFromChecksumError, +} = require('../integrity/validateChecksums'); const { externalBackends, versioningNotImplBackends } = constants; const externalVersioningErrorMessage = 'We do not currently support putting ' + -'a versioned object to a location-constraint of type Azure or GCP.'; + 'a versioned object to a location-constraint of type Azure or GCP.'; + +/** + * Validate and compute the checksum for a zero-size object body. + * Parses the checksum headers, validates the client-supplied digest against + * the empty-body hash, sets metadataStoreParams.checksum on success, and + * calls back with an error on mismatch or invalid headers. + * + * @param {object} headers - request headers + * @param {object} metadataStoreParams - metadata params (checksum field set in-place) + * @param {function} callback - (err) callback + * @return {undefined} + */ +function zeroSizeBodyChecksumCheck(headers, metadataStoreParams, callback) { + const checksumData = getChecksumDataFromHeaders(headers); + if (checksumData.error) { + return callback(arsenalErrorFromChecksumError(checksumData)); + } + // For trailer format with zero decoded bytes, the trailer in the body is + // never read (stream bypassed), so expected is always undefined here. + // We still compute and store the empty-body hash for the announced algorithm. + const { algorithm, expected } = checksumData; + return Promise.resolve(algorithms[algorithm].digest(Buffer.alloc(0))) + .then(value => { + if (expected !== undefined && expected !== value) { + return callback(errors.BadDigest.customizeDescription( + `The ${algorithm.toUpperCase()} you specified did not match the calculated checksum.` + )); + } + // eslint-disable-next-line no-param-reassign + metadataStoreParams.checksum = { algorithm, value, type: 'FULL_OBJECT' }; + return callback(null); + }, err => callback(err)); +} function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle, metadataStoreParams, dataToDelete, log, requestMethod, callback) { @@ -217,7 +255,13 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo, if (size === 0) { if (!dontSkipBackend[locationType]) { metadataStoreParams.contentMD5 = constants.emptyFileMd5; - return next(null, null, null); + // Delete markers are zero-byte versioned tombstones with + // no body, ETag, or checksum — skip checksum handling. + if (isDeleteMarker) { + return next(null, null, null, null); + } + return zeroSizeBodyChecksumCheck(request.headers, metadataStoreParams, + err => next(err, null, null, null)); } // Handle mdOnlyHeader as a metadata only operation. If @@ -243,14 +287,14 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo, dataStoreVersionId: versionId, dataStoreMD5: _md5, }; - return next(null, dataGetInfo, _md5); + return next(null, dataGetInfo, _md5, null); } } return dataStore(objectKeyContext, cipherBundle, request, size, streamingV4Params, backendInfo, log, next); }, - function processDataResult(dataGetInfo, calculatedHash, next) { + function processDataResult(dataGetInfo, calculatedHash, checksum, next) { if (dataGetInfo === null || dataGetInfo === undefined) { return next(null, null); } @@ -264,7 +308,8 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo, : `1:${calculatedHash}`; const dataGetInfoArr = [{ key, size, start: 0, dataStoreName, dataStoreType, dataStoreETag: prefixedDataStoreETag, - dataStoreVersionId }]; + dataStoreVersionId + }]; if (cipherBundle) { dataGetInfoArr[0].cryptoScheme = cipherBundle.cryptoScheme; dataGetInfoArr[0].cipheredDataKey = @@ -275,6 +320,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo, dataGetInfoArr[0].size = mdOnlySize; } metadataStoreParams.contentMD5 = calculatedHash; + metadataStoreParams.checksum = checksum; return next(null, dataGetInfoArr); }, function getVersioningInfo(infoArr, next) { diff --git a/lib/api/apiUtils/object/storeObject.js b/lib/api/apiUtils/object/storeObject.js index 68a5312c4f..f4a90e8066 100644 --- a/lib/api/apiUtils/object/storeObject.js +++ b/lib/api/apiUtils/object/storeObject.js @@ -12,12 +12,13 @@ const { arsenalErrorFromChecksumError } = require('../../apiUtils/integrity/vali * @param {object} dataRetrievalInfo - object containing the keys of stored data * @param {number} dataRetrievalInfo.key - key of the stored data * @param {string} dataRetrievalInfo.dataStoreName - the implName of the data + * @param {object} checksumStream - checksum transform stream with digest/algoName properties * @param {object} log - request logger instance * @param {function} cb - callback to send error or move to next task * @return {function} - calls callback with arguments: * error, dataRetrievalInfo, and completedHash (if any) */ -function checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo, log, cb) { +function checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo, checksumStream, log, cb) { const contentMD5 = stream.contentMD5; const completedHash = hashedStream.completedHash; if (contentMD5 && completedHash && contentMD5 !== completedHash) { @@ -37,7 +38,10 @@ function checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo, log, cb) { return cb(errors.BadDigest); }); } - return cb(null, dataRetrievalInfo, completedHash); + const checksum = checksumStream.digest + ? { algorithm: checksumStream.algoName, value: checksumStream.digest, type: 'FULL_OBJECT' } + : null; + return cb(null, dataRetrievalInfo, completedHash, checksum); } /** @@ -107,7 +111,8 @@ function dataStore(objectContext, cipherBundle, stream, size, return cbOnce(arsenalErrorFromChecksumError(checksumErr)); }); } - return checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo, log, cbOnce); + return checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo, + checksumedStream.stream, log, cbOnce); }; // ChecksumTransform._flush computes the digest asynchronously for diff --git a/lib/api/objectPut.js b/lib/api/objectPut.js index 8b92e0e9a6..3b1641ca88 100644 --- a/lib/api/objectPut.js +++ b/lib/api/objectPut.js @@ -225,6 +225,10 @@ function objectPut(authInfo, request, streamingV4Params, log, callback) { if (storingResult) { // ETag's hex should always be enclosed in quotes responseHeaders.ETag = `"${storingResult.contentMD5}"`; + if (storingResult.checksum) { + const { checksumAlgorithm, checksumValue } = storingResult.checksum; + responseHeaders[`x-amz-checksum-${checksumAlgorithm}`] = checksumValue; + } } const vcfg = bucket.getVersioningConfiguration(); const isVersionedObj = vcfg && vcfg.Status === 'Enabled'; diff --git a/lib/routes/routeBackbeat.js b/lib/routes/routeBackbeat.js index ba834e5e3e..718d19d9eb 100644 --- a/lib/routes/routeBackbeat.js +++ b/lib/routes/routeBackbeat.js @@ -447,7 +447,14 @@ function putData(request, response, bucketInfo, objMd, log, callback) { } return dataStore( context, cipherBundle, request, payloadLen, {}, - backendInfo, log, (err, retrievalInfo, md5) => { + backendInfo, log, + // The callback's 4th arg (checksum) is intentionally ignored: any + // x-amz-checksum-* header sent by Backbeat is already validated + // inside dataStore by ChecksumTransform. The computed value is not + // stored here because this is a data-only write — metadata is + // written separately by Backbeat, which should propagate the source + // object's checksum. + (err, retrievalInfo, md5) => { if (err) { log.error('error putting data', { error: err, @@ -853,6 +860,12 @@ function putObject(request, response, log, callback) { const payloadLen = parseInt(request.headers['content-length'], 10); const backendInfo = new BackendInfo(config, storageLocation); return dataStore(context, CIPHER, request, payloadLen, {}, backendInfo, log, + // The callback's 4th arg (checksum) is intentionally ignored: any + // x-amz-checksum-* header sent by Backbeat is already validated inside + // dataStore by ChecksumTransform. The computed value is not stored here + // because this is a data-only write to an external backend — metadata + // is managed separately by Backbeat, which should propagate the source + // object's checksum. (err, retrievalInfo, md5) => { if (err) { log.error('error putting data', { diff --git a/lib/services.js b/lib/services.js index f2980f1fc5..966b8d4665 100644 --- a/lib/services.js +++ b/lib/services.js @@ -6,6 +6,7 @@ const { errors, s3middleware } = require('arsenal'); const ObjectMD = require('arsenal').models.ObjectMD; const BucketInfo = require('arsenal').models.BucketInfo; const ObjectMDArchive = require('arsenal').models.ObjectMDArchive; +const ObjectMDChecksum = require('arsenal').models.ObjectMDChecksum; const { versioning } = require('arsenal'); const acl = require('./metadata/acl'); const constants = require('../constants'); @@ -102,7 +103,7 @@ const services = { * @return {function} executes callback with err or ETag as arguments */ metadataStoreObject(bucketName, dataGetInfo, cipherBundle, params, cb) { - const { objectKey, authInfo, size, contentMD5, metaHeaders, + const { objectKey, authInfo, size, contentMD5, checksum, metaHeaders, contentType, cacheControl, contentDisposition, contentEncoding, expires, multipart, headers, overrideMetadata, log, lastModifiedDate, versioning, versionId, uploadId, @@ -138,6 +139,9 @@ const services = { // CreationTime needs to be carried over so that it remains static .setCreationTime(creationTime) .setOriginOp(originOp); + if (checksum) { + md.setChecksum(new ObjectMDChecksum(checksum.algorithm, checksum.value, checksum.type)); + } // Sending in last modified date in object put copy since need // to return the exact date in the response if (lastModifiedDate) { @@ -329,6 +333,7 @@ const services = { tags: md.getTags(), contentMD5, versionId, + checksum: md.getChecksum(), }); }); }, diff --git a/package.json b/package.json index eaee9d3d60..c8786e5d68 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "@azure/storage-blob": "^12.28.0", "@hapi/joi": "^17.1.1", "@smithy/node-http-handler": "^3.0.0", - "arsenal": "git+https://github.com/scality/Arsenal#8.3.6", + "arsenal": "git+https://github.com/scality/Arsenal#improvement/ARSN-557-add-checksum-to-object-metadata", "async": "2.6.4", "bucketclient": "scality/bucketclient#8.2.7", "bufferutil": "^4.0.8", diff --git a/tests/functional/aws-node-sdk/test/object/mpuVersion.js b/tests/functional/aws-node-sdk/test/object/mpuVersion.js index 98c3155e95..0cdeee830f 100644 --- a/tests/functional/aws-node-sdk/test/object/mpuVersion.js +++ b/tests/functional/aws-node-sdk/test/object/mpuVersion.js @@ -141,6 +141,14 @@ function checkObjMdAndUpdate(objMDBefore, objMDAfter, props) { // eslint-disable-next-line no-param-reassign delete objMDBefore['content-type']; } + if (objMDBefore.checksum && !objMDAfter.checksum) { + // The initial PutObject stores a checksum, but the MPU restore path does not + // (CompleteMultipartUpload checksum storage is not yet implemented). + // Once it is, the restored object should carry a checksum and this workaround + // should be removed. + // eslint-disable-next-line no-param-reassign + delete objMDBefore.checksum; + } } function clearUploadIdAndRestoreStatusFromVersions(versions) { diff --git a/tests/functional/aws-node-sdk/test/object/putVersion.js b/tests/functional/aws-node-sdk/test/object/putVersion.js index efa68708be..cbea941a8c 100644 --- a/tests/functional/aws-node-sdk/test/object/putVersion.js +++ b/tests/functional/aws-node-sdk/test/object/putVersion.js @@ -258,7 +258,7 @@ describe('PUT object with x-scal-s3-version-id header', () => { assert.deepStrictEqual(versionsAfter, versionsBefore); checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', - 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'originOp']); + 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'originOp', 'checksum']); assert.deepStrictEqual(objMDAfter, objMDBefore); return done(); }); @@ -309,7 +309,7 @@ describe('PUT object with x-scal-s3-version-id header', () => { assert.deepStrictEqual(versionsAfter, versionsBefore); checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp', - 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName']); + 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'checksum']); assert.deepStrictEqual(objMDAfter, objMDBefore); return done(); }); @@ -360,7 +360,7 @@ describe('PUT object with x-scal-s3-version-id header', () => { assert.deepStrictEqual(versionsAfter, versionsBefore); checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp', - 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName']); + 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'checksum']); assert.deepStrictEqual(objMDAfter, objMDBefore); return done(); }); @@ -408,7 +408,7 @@ describe('PUT object with x-scal-s3-version-id header', () => { assert.deepStrictEqual(versionsAfter, versionsBefore); checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp', - 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName']); + 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'checksum']); assert.deepStrictEqual(objMDAfter, objMDBefore); return done(); }); @@ -460,7 +460,7 @@ describe('PUT object with x-scal-s3-version-id header', () => { assert.deepStrictEqual(versionsAfter, versionsBefore); checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp', - 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName']); + 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'checksum']); assert.deepStrictEqual(objMDAfter, objMDBefore); return done(); }); @@ -515,7 +515,7 @@ describe('PUT object with x-scal-s3-version-id header', () => { assert.deepStrictEqual(versionsAfter, versionsBefore); checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp', - 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName']); + 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'checksum']); assert.deepStrictEqual(objMDAfter, objMDBefore); return done(); }); @@ -568,7 +568,7 @@ describe('PUT object with x-scal-s3-version-id header', () => { assert.deepStrictEqual(versionsAfter, versionsBefore); checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp', - 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName']); + 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'checksum']); assert.deepStrictEqual(objMDAfter, objMDBefore); return done(); }); @@ -620,7 +620,7 @@ describe('PUT object with x-scal-s3-version-id header', () => { assert.deepStrictEqual(versionsAfter, versionsBefore); checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp', - 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName']); + 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'checksum']); assert.deepStrictEqual(objMDAfter, objMDBefore); return done(); }); @@ -679,7 +679,7 @@ describe('PUT object with x-scal-s3-version-id header', () => { assert.deepStrictEqual(versionsAfter, versionsBefore); checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp', - 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName']); + 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'checksum']); assert.deepStrictEqual(objMDAfter, objMDBefore); return done(); }); @@ -726,7 +726,7 @@ describe('PUT object with x-scal-s3-version-id header', () => { assert.deepStrictEqual(versionsAfter, versionsBefore); checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp', - 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName']); + 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'checksum']); assert.deepStrictEqual(objMDAfter, objMDBefore); return done(); }); diff --git a/tests/functional/raw-node/test/checksumPutObjectUploadPart.js b/tests/functional/raw-node/test/checksumPutObjectUploadPart.js index 132d8b1a60..0a4fc09ce3 100644 --- a/tests/functional/raw-node/test/checksumPutObjectUploadPart.js +++ b/tests/functional/raw-node/test/checksumPutObjectUploadPart.js @@ -1,6 +1,7 @@ const assert = require('assert'); const crypto = require('crypto'); const async = require('async'); +const { algorithms } = require('../../../../lib/api/apiUtils/integrity/validateChecksums'); const { makeS3Request } = require('../utils/makeRequest'); const HttpRequestAuthV4 = require('../utils/HttpRequestAuthV4'); @@ -38,7 +39,11 @@ function doPutRequest(url, headers, body, callback) { res => { let data = ''; res.on('data', chunk => { data += chunk; }); - res.on('end', () => callback(null, { statusCode: res.statusCode, body: data })); + res.on('end', () => callback(null, { + statusCode: res.statusCode, + body: data, + headers: res.headers, + })); } ); req.on('error', callback); @@ -141,16 +146,38 @@ const msgMalformedTrailer = 'The request contained trailing data that was not we const msgSdkMissingTrailer = 'x-amz-sdk-checksum-algorithm specified, but no corresponding' + ' x-amz-checksum-* or x-amz-trailer headers were found.'; +// Module-level variables for computed crc64nvme checksums (filled in before hook) +let crc64nvmeOfTestContent2; +let crc64nvmeOfTrailerContent; + // Create the 24 common protocol-scenario tests for a given URL factory. // urlFn() is called lazily at test runtime so that uploadId is available. -function makeScenarioTests(urlFn) { +// checkChecksumResponse: if true, assert x-amz-checksum-* response headers on 200 OK tests. +function makeScenarioTests(urlFn, checkChecksumResponse = false) { + before(async () => { + if (!crc64nvmeOfTestContent2) { + crc64nvmeOfTestContent2 = await algorithms.crc64nvme.digest(testContent2); + } + if (!crc64nvmeOfTrailerContent) { + crc64nvmeOfTrailerContent = await algorithms.crc64nvme.digest(trailerContent); + } + }); + itSkipIfAWS( 'testS3PutNoChecksum: signed sha256 in x-amz-content-sha256, no x-amz-checksum header -> 200 OK', done => { doPutRequest(urlFn(), { 'x-amz-content-sha256': testContent2Sha256Hex, 'content-length': testContent2.length, - }, testContent2, (err, res) => assertStatus(200)(err, res, done)); + }, testContent2, (err, res) => { + assertStatus(200)(err, res, () => { + if (checkChecksumResponse) { + assert.strictEqual(res.headers['x-amz-checksum-crc64nvme'], crc64nvmeOfTestContent2, + `expected x-amz-checksum-crc64nvme: ${crc64nvmeOfTestContent2}`); + } + done(); + }); + }); }); itSkipIfAWS( @@ -161,7 +188,15 @@ function makeScenarioTests(urlFn) { 'x-amz-sdk-checksum-algorithm': 'SHA256', 'x-amz-checksum-sha256': testContent2Sha256B64, 'content-length': testContent2.length, - }, testContent2, (err, res) => assertStatus(200)(err, res, done)); + }, testContent2, (err, res) => { + assertStatus(200)(err, res, () => { + if (checkChecksumResponse) { + assert.strictEqual(res.headers['x-amz-checksum-sha256'], testContent2Sha256B64, + `expected x-amz-checksum-sha256: ${testContent2Sha256B64}`); + } + done(); + }); + }); }); itSkipIfAWS( @@ -184,7 +219,15 @@ function makeScenarioTests(urlFn) { 'x-amz-sdk-checksum-algorithm': 'SHA256', 'x-amz-checksum-sha256': testContent2Sha256B64, 'content-length': testContent2.length, - }, testContent2, (err, res) => assertStatus(200)(err, res, done)); + }, testContent2, (err, res) => { + assertStatus(200)(err, res, () => { + if (checkChecksumResponse) { + assert.strictEqual(res.headers['x-amz-checksum-sha256'], testContent2Sha256B64, + `expected x-amz-checksum-sha256: ${testContent2Sha256B64}`); + } + done(); + }); + }); }); itSkipIfAWS( @@ -208,7 +251,15 @@ function makeScenarioTests(urlFn) { 'x-amz-trailer': 'x-amz-checksum-sha256', 'x-amz-decoded-content-length': trailerContent.length, 'content-length': Buffer.byteLength(body), - }, body, (err, res) => assertStatus(200)(err, res, done)); + }, body, (err, res) => { + assertStatus(200)(err, res, () => { + if (checkChecksumResponse) { + assert.strictEqual(res.headers['x-amz-checksum-sha256'], trailerContentSha256, + `expected x-amz-checksum-sha256: ${trailerContentSha256}`); + } + done(); + }); + }); }); itSkipIfAWS( @@ -295,7 +346,15 @@ function makeScenarioTests(urlFn) { 'x-amz-trailer': 'x-amz-checksum-sha256', 'x-amz-decoded-content-length': trailerContent.length, 'content-length': Buffer.byteLength(body), - }, body, (err, res) => assertStatus(200)(err, res, done)); + }, body, (err, res) => { + assertStatus(200)(err, res, () => { + if (checkChecksumResponse) { + assert.strictEqual(res.headers['x-amz-checksum-sha256'], trailerContentSha256, + `expected x-amz-checksum-sha256: ${trailerContentSha256}`); + } + done(); + }); + }); }); itSkipIfAWS( @@ -308,7 +367,15 @@ function makeScenarioTests(urlFn) { 'x-amz-sdk-checksum-algorithm': 'SHA256', 'x-amz-decoded-content-length': trailerContent.length, 'content-length': Buffer.byteLength(body), - }, body, (err, res) => assertStatus(200)(err, res, done)); + }, body, (err, res) => { + assertStatus(200)(err, res, () => { + if (checkChecksumResponse) { + assert.strictEqual(res.headers['x-amz-checksum-sha256'], trailerContentSha256, + `expected x-amz-checksum-sha256: ${trailerContentSha256}`); + } + done(); + }); + }); }); itSkipIfAWS( @@ -403,7 +470,15 @@ function makeScenarioTests(urlFn) { // no x-amz-trailer 'x-amz-decoded-content-length': trailerContent.length, 'content-length': Buffer.byteLength(body), - }, body, (err, res) => assertStatus(200)(err, res, done)); + }, body, (err, res) => { + assertStatus(200)(err, res, () => { + if (checkChecksumResponse) { + assert.strictEqual(res.headers['x-amz-checksum-crc64nvme'], crc64nvmeOfTrailerContent, + `expected x-amz-checksum-crc64nvme: ${crc64nvmeOfTrailerContent}`); + } + done(); + }); + }); }); itSkipIfAWS( @@ -417,7 +492,15 @@ function makeScenarioTests(urlFn) { // no x-amz-trailer 'x-amz-decoded-content-length': trailerContent.length, 'content-length': Buffer.byteLength(body), - }, body, (err, res) => assertStatus(200)(err, res, done)); + }, body, (err, res) => { + assertStatus(200)(err, res, () => { + if (checkChecksumResponse) { + assert.strictEqual(res.headers['x-amz-checksum-crc64nvme'], crc64nvmeOfTrailerContent, + `expected x-amz-checksum-crc64nvme: ${crc64nvmeOfTrailerContent}`); + } + done(); + }); + }); }); itSkipIfAWS( @@ -431,7 +514,15 @@ function makeScenarioTests(urlFn) { 'content-md5': trailerContentMd5B64, 'x-amz-decoded-content-length': trailerContent.length, 'content-length': Buffer.byteLength(body), - }, body, (err, res) => assertStatus(200)(err, res, done)); + }, body, (err, res) => { + assertStatus(200)(err, res, () => { + if (checkChecksumResponse) { + assert.strictEqual(res.headers['x-amz-checksum-sha256'], trailerContentSha256, + `expected x-amz-checksum-sha256: ${trailerContentSha256}`); + } + done(); + }); + }); }); itSkipIfAWS( @@ -445,7 +536,15 @@ function makeScenarioTests(urlFn) { 'x-amz-trailer': 'x-amz-checksum-sha256', 'x-amz-decoded-content-length': trailerContent.length, 'content-length': Buffer.byteLength(body), - }, body, (err, res) => assertStatus(200)(err, res, done)); + }, body, (err, res) => { + assertStatus(200)(err, res, () => { + if (checkChecksumResponse) { + assert.strictEqual(res.headers['x-amz-checksum-sha256'], trailerContentSha256, + `expected x-amz-checksum-sha256: ${trailerContentSha256}`); + } + done(); + }); + }); }); } @@ -557,7 +656,7 @@ describe('PutObject: trailer and checksum protocol scenarios', () => { }); }); - makeScenarioTests(() => `http://localhost:8000/${bucket}/${objectKey}`); + makeScenarioTests(() => `http://localhost:8000/${bucket}/${objectKey}`, true); }); @@ -608,3 +707,197 @@ describe('UploadPart: trailer and checksum protocol scenarios', () => { () => `http://localhost:8000/${bucket}/${objectKey}?partNumber=1&uploadId=${uploadId2}` ); }); + +describe('PutObject: checksum response header per algorithm', () => { + const url = `http://localhost:8000/${bucket}/${objectKey}`; + const body = testContent2; + const sha256Hex = testContent2Sha256Hex; + + let expectedCrc64nvme; + + before(async () => { + await new Promise((resolve, reject) => + makeS3Request({ method: 'PUT', authCredentials, bucket }, + err => (err ? reject(err) : resolve()))); + expectedCrc64nvme = await algorithms.crc64nvme.digest(body); + }); + + after(done => { + makeS3Request({ method: 'DELETE', authCredentials, bucket, objectKey }, () => { + makeS3Request({ method: 'DELETE', authCredentials, bucket }, err => { + assert.ifError(err); + done(); + }); + }); + }); + + const checksumAlgos = [ + { name: 'crc32', computeExpected: () => algorithms.crc32.digest(body) }, + { name: 'crc32c', computeExpected: () => algorithms.crc32c.digest(body) }, + { name: 'crc64nvme', computeExpected: () => expectedCrc64nvme }, + { name: 'sha1', computeExpected: () => algorithms.sha1.digest(body) }, + { name: 'sha256', computeExpected: () => algorithms.sha256.digest(body) }, + ]; + + for (const algo of checksumAlgos) { + itSkipIfAWS( + `returns x-amz-checksum-${algo.name} response header with correct value`, + done => { + const expectedValue = algo.computeExpected(); + const headerName = `x-amz-checksum-${algo.name}`; + doPutRequest(url, { + 'x-amz-content-sha256': sha256Hex, + [headerName]: expectedValue, + 'content-length': body.length, + }, body, (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 200, + `expected 200, got ${res.statusCode}: ${res.body}`); + assert.strictEqual(res.headers[headerName], expectedValue, + `expected ${headerName}: ${expectedValue}`); + done(); + }); + } + ); + } + + itSkipIfAWS( + 'returns x-amz-checksum-crc64nvme response header when no checksum header is sent', + done => { + doPutRequest(url, { + 'x-amz-content-sha256': sha256Hex, + 'content-length': body.length, + }, body, (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 200, + `expected 200, got ${res.statusCode}: ${res.body}`); + assert.strictEqual(res.headers['x-amz-checksum-crc64nvme'], expectedCrc64nvme, + `expected x-amz-checksum-crc64nvme: ${expectedCrc64nvme}`); + done(); + }); + } + ); +}); + +describe('PutObject: zero-byte object checksum handling', () => { + const zeroBucket = 'checksumzerobytebucket'; + const zeroUrl = `http://localhost:8000/${zeroBucket}/${objectKey}`; + const emptyBody = Buffer.alloc(0); + + // Known empty-body checksums (well-known constants except crc64nvme which is async) + // crc32("") = crc32c("") = 0x00000000 => AAAAAA== + // sha1("") = 2jmj7l5rSw0yVb/vlWAYkK/YBwk= + // sha256("") = 47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU= + const expectedEmptyChecksums = { + crc32: 'AAAAAA==', + crc32c: 'AAAAAA==', + sha1: '2jmj7l5rSw0yVb/vlWAYkK/YBwk=', + sha256: '47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=', + crc64nvme: null, // filled in before hook (async) + }; + + // Wrong digests for empty body: valid format but not the correct empty-body hash. + // crc32/crc32c of "" = AAAAAA==, so we use a different value for those. + const wrongEmptyDigests = { + crc32: 'EyV5Tg==', + crc32c: 'EyV5Tg==', + crc64nvme: 'skQv82y5rgE=', + sha1: 'AAAAAAAAAAAAAAAAAAAAAAAAAAA=', + sha256: 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=', + }; + + before(async () => { + await new Promise((resolve, reject) => + makeS3Request({ method: 'PUT', authCredentials, bucket: zeroBucket }, + err => (err ? reject(err) : resolve()))); + expectedEmptyChecksums.crc64nvme = await algorithms.crc64nvme.digest(emptyBody); + }); + + after(done => { + makeS3Request({ method: 'DELETE', authCredentials, bucket: zeroBucket, objectKey }, () => { + makeS3Request({ method: 'DELETE', authCredentials, bucket: zeroBucket }, err => { + assert.ifError(err); + done(); + }); + }); + }); + + itSkipIfAWS( + 'no checksum header: returns 200 with x-amz-checksum-crc64nvme of empty body', + done => { + doPutRequest(zeroUrl, { + 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', + 'content-length': 0, + }, emptyBody, (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 200, + `expected 200, got ${res.statusCode}: ${res.body}`); + assert.strictEqual(res.headers['x-amz-checksum-crc64nvme'], + expectedEmptyChecksums.crc64nvme, + `expected x-amz-checksum-crc64nvme: ${expectedEmptyChecksums.crc64nvme}`); + done(); + }); + }); + + for (const algoName of ['crc32', 'crc32c', 'crc64nvme', 'sha1', 'sha256']) { + itSkipIfAWS( + `correct empty-body ${algoName} checksum: returns 200 with echoed response header`, + done => { + const expected = expectedEmptyChecksums[algoName]; + const headerName = `x-amz-checksum-${algoName}`; + doPutRequest(zeroUrl, { + 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', + [headerName]: expected, + 'content-length': 0, + }, emptyBody, (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 200, + `expected 200, got ${res.statusCode}: ${res.body}`); + assert.strictEqual(res.headers[headerName], expected, + `expected ${headerName}: ${expected}`); + done(); + }); + }); + + itSkipIfAWS( + `wrong empty-body ${algoName} checksum: returns 400 BadDigest`, + done => { + const wrong = wrongEmptyDigests[algoName]; + const headerName = `x-amz-checksum-${algoName}`; + doPutRequest(zeroUrl, { + 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', + [headerName]: wrong, + 'content-length': 0, + }, emptyBody, (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 400, + `expected 400, got ${res.statusCode}: ${res.body}`); + assert(res.body.includes('BadDigest'), + `expected BadDigest in: ${res.body}`); + done(); + }); + }); + } + + itSkipIfAWS( + 'TRAILER with zero decoded content length: returns 200 with checksum for the trailer algorithm', + done => { + // x-amz-decoded-content-length: 0 → parsedContentLength === 0, hits the zero-byte path. + // The trailer body is never consumed; server computes and stores the empty-body hash itself. + const emptySha256 = expectedEmptyChecksums.sha256; + const trailerBody = `0\r\nx-amz-checksum-sha256:${emptySha256}\n\r\n\r\n\r\n`; + doPutRequest(zeroUrl, { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha256', + 'x-amz-decoded-content-length': 0, + 'content-length': Buffer.byteLength(trailerBody), + }, trailerBody, (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 200, + `expected 200, got ${res.statusCode}: ${res.body}`); + assert.strictEqual(res.headers['x-amz-checksum-sha256'], emptySha256, + `expected x-amz-checksum-sha256: ${emptySha256}`); + done(); + }); + }); +}); diff --git a/tests/functional/raw-node/test/routes/routeMetadata.js b/tests/functional/raw-node/test/routes/routeMetadata.js index 81b24a2bfa..84f698a7cc 100644 --- a/tests/functional/raw-node/test/routes/routeMetadata.js +++ b/tests/functional/raw-node/test/routes/routeMetadata.js @@ -1,7 +1,9 @@ const assert = require('assert'); +const crypto = require('crypto'); const http = require('http'); -const { CreateBucketCommand, - PutObjectCommand, +const { CreateBucketCommand, + PutObjectCommand, + DeleteObjectCommand, DeleteBucketCommand } = require('@aws-sdk/client-s3'); const { makeRequest } = require('../../utils/makeRequest'); @@ -144,3 +146,78 @@ describe('metadata routes with metadata', () => { }); }); }); + +describe('checksum stored in object metadata after PutObject', () => { + const bucketUtil = new BucketUtility('default', { signatureVersion: 'v4' }); + const s3 = bucketUtil.s3; + + const bucket = 'bucket-checksum-test'; + const objectBody = 'hello checksum'; + const sha256Key = 'object-with-sha256-checksum'; + const defaultKey = 'object-with-default-checksum'; + + // CRC32 of 'hello checksum' in base64 — the AWS SDK v3 injects x-amz-checksum-crc32 by default + const expectedCrc32 = 'EyV5Tg=='; + + before(async function () { + if (!process.env.S3_END_TO_END) { + this.skip(); + } + await s3.send(new CreateBucketCommand({ Bucket: bucket })); + + const sha256Value = crypto.createHash('sha256').update(objectBody).digest('base64'); + await s3.send(new PutObjectCommand({ + Bucket: bucket, + Key: sha256Key, + Body: objectBody, + ChecksumSHA256: sha256Value, + })); + await s3.send(new PutObjectCommand({ + Bucket: bucket, + Key: defaultKey, + Body: objectBody, + })); + }); + + after(async () => { + if (!process.env.S3_END_TO_END) { + return; + } + await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: sha256Key })); + await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: defaultKey })); + await s3.send(new DeleteBucketCommand({ Bucket: bucket })); + }); + + it('stores sha256 checksum in metadata when x-amz-checksum-sha256 is provided', done => { + const expectedValue = crypto.createHash('sha256').update(objectBody).digest('base64'); + makeMetadataRequest({ + method: 'GET', + authCredentials: metadataAuthCredentials, + path: `/_/metadata/default/bucket/${bucket}/${sha256Key}`, + }, (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 200); + const md = JSON.parse(res.body); + assert.strictEqual(md.checksum.checksumAlgorithm, 'sha256'); + assert.strictEqual(md.checksum.checksumValue, expectedValue); + assert.strictEqual(md.checksum.checksumType, 'FULL_OBJECT'); + return done(); + }); + }); + + it('stores crc32 checksum in metadata when no explicit checksum header is provided (AWS SDK default)', done => { + makeMetadataRequest({ + method: 'GET', + authCredentials: metadataAuthCredentials, + path: `/_/metadata/default/bucket/${bucket}/${defaultKey}`, + }, (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 200); + const md = JSON.parse(res.body); + assert.strictEqual(md.checksum.checksumAlgorithm, 'crc32'); + assert.strictEqual(md.checksum.checksumValue, expectedCrc32); + assert.strictEqual(md.checksum.checksumType, 'FULL_OBJECT'); + return done(); + }); + }); +}); diff --git a/tests/multipleBackend/routes/routeBackbeat.js b/tests/multipleBackend/routes/routeBackbeat.js index d8a43b7b75..2cec218254 100644 --- a/tests/multipleBackend/routes/routeBackbeat.js +++ b/tests/multipleBackend/routes/routeBackbeat.js @@ -3642,4 +3642,107 @@ describe('backbeat routes', () => { ], done); }); }); + + describe('checksum validation', () => { + const testDataSha256B64 = crypto.createHash('sha256') + .update(testData, 'utf-8').digest('base64'); + // A valid-length but wrong sha256 digest (44 base64 chars). + const wrongSha256B64 = 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA='; + + describe('putData', () => { + it('should return 400 BadDigest when x-amz-checksum-sha256 does not match body', done => { + makeBackbeatRequest({ + method: 'PUT', + resourceType: 'data', + bucket: TEST_BUCKET, + objectKey: TEST_KEY, + headers: { + 'x-scal-canonical-id': testMd['owner-id'], + 'content-md5': testDataMd5, + 'content-length': testData.length, + 'x-amz-checksum-sha256': wrongSha256B64, + }, + requestBody: testData, + authCredentials: backbeatAuthCredentials, + }, err => { + assert(err, 'expected an error response'); + assert.strictEqual(err.statusCode, 400); + assert.strictEqual(err.code, 'BadDigest'); + done(); + }); + }); + + it('should return 200 when x-amz-checksum-sha256 matches body', done => { + makeBackbeatRequest({ + method: 'PUT', + resourceType: 'data', + bucket: TEST_BUCKET, + objectKey: TEST_KEY, + headers: { + 'x-scal-canonical-id': testMd['owner-id'], + 'content-md5': testDataMd5, + 'content-length': testData.length, + 'x-amz-checksum-sha256': testDataSha256B64, + }, + requestBody: testData, + authCredentials: backbeatAuthCredentials, + }, (err, data) => { + assert.ifError(err); + assert.strictEqual(data.statusCode, 200); + done(); + }); + }); + }); + + describe('putObject (multiplebackenddata)', () => { + itIfLocationAws('should return 400 BadDigest when x-amz-checksum-sha256 does not match body', done => { + makeBackbeatRequest({ + method: 'PUT', + resourceType: 'multiplebackenddata', + bucket: TEST_BUCKET, + objectKey: TEST_KEY, + queryObj: { operation: 'putobject' }, + headers: { + 'x-scal-canonical-id': testMd['owner-id'], + 'x-scal-storage-type': 'aws_s3', + 'x-scal-storage-class': awsLocation, + 'content-md5': testDataMd5, + 'content-length': testData.length, + 'x-amz-checksum-sha256': wrongSha256B64, + }, + requestBody: testData, + authCredentials: backbeatAuthCredentials, + }, err => { + assert(err, 'expected an error response'); + assert.strictEqual(err.statusCode, 400); + assert.strictEqual(err.code, 'BadDigest'); + done(); + }); + }); + + itIfLocationAws('should return 200 when x-amz-checksum-sha256 matches body', done => { + makeBackbeatRequest({ + method: 'PUT', + resourceType: 'multiplebackenddata', + bucket: TEST_BUCKET, + objectKey: TEST_KEY, + queryObj: { operation: 'putobject' }, + headers: { + 'x-scal-canonical-id': testMd['owner-id'], + 'x-scal-storage-type': 'aws_s3', + 'x-scal-storage-class': awsLocation, + 'content-md5': testDataMd5, + 'content-length': testData.length, + 'x-amz-checksum-sha256': testDataSha256B64, + }, + requestBody: testData, + authCredentials: backbeatAuthCredentials, + }, (err, data) => { + assert.ifError(err); + assert.strictEqual(data.statusCode, 200); + done(); + }); + }); + }); + }); }); diff --git a/tests/unit/api/objectPut.js b/tests/unit/api/objectPut.js index 3f4c458d60..4b3e3ec8bc 100644 --- a/tests/unit/api/objectPut.js +++ b/tests/unit/api/objectPut.js @@ -1,5 +1,6 @@ const assert = require('assert'); const async = require('async'); +const crypto = require('crypto'); const moment = require('moment'); const { s3middleware, storage, versioning } = require('arsenal'); const sinon = require('sinon'); @@ -865,6 +866,54 @@ describe('objectPut API', () => { }); }); }); + + it('should store sha256 checksum in metadata when x-amz-checksum-sha256 header is provided', done => { + const sha256Value = crypto.createHash('sha256').update(postBody).digest('base64'); + const request = new DummyRequest({ + bucketName, + namespace, + objectKey: objectName, + headers: { + host: `${bucketName}.s3.amazonaws.com`, + 'x-amz-checksum-sha256': sha256Value, + }, + url: '/', + }, postBody); + + bucketPut(authInfo, testPutBucketRequest, log, err => { + assert.ifError(err); + objectPut(authInfo, request, undefined, log, (err, resHeaders) => { + assert.ifError(err); + assert.strictEqual(resHeaders['x-amz-checksum-sha256'], sha256Value); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { + assert.ifError(err); + assert(md.checksum, 'checksum should be set in metadata'); + assert.strictEqual(md.checksum.checksumAlgorithm, 'sha256'); + assert.strictEqual(md.checksum.checksumValue, sha256Value); + assert.strictEqual(md.checksum.checksumType, 'FULL_OBJECT'); + done(); + }); + }); + }); + }); + + it('should store crc64nvme checksum in metadata when no checksum header is provided', done => { + bucketPut(authInfo, testPutBucketRequest, log, err => { + assert.ifError(err); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.ifError(err); + assert(resHeaders['x-amz-checksum-crc64nvme'], 'crc64nvme response header should be set'); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { + assert.ifError(err); + assert(md.checksum, 'checksum should be set in metadata'); + assert.strictEqual(md.checksum.checksumAlgorithm, 'crc64nvme'); + assert.strictEqual(md.checksum.checksumValue, resHeaders['x-amz-checksum-crc64nvme']); + assert.strictEqual(md.checksum.checksumType, 'FULL_OBJECT'); + done(); + }); + }); + }); + }); }); describe('objectPut API with versioning', () => { diff --git a/yarn.lock b/yarn.lock index 82bf9887f0..85e2947183 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5766,9 +5766,9 @@ arraybuffer.prototype.slice@^1.0.4: optionalDependencies: ioctl "^2.0.2" -"arsenal@git+https://github.com/scality/Arsenal#8.3.6": +"arsenal@git+https://github.com/scality/Arsenal#improvement/ARSN-557-add-checksum-to-object-metadata": version "8.3.6" - resolved "git+https://github.com/scality/Arsenal#21f6a54ac70f2f6c94b592760780b3ca3bab7a95" + resolved "git+https://github.com/scality/Arsenal#af4442f234f0c460ea7607f8f7f9eaac22fe8fa4" dependencies: "@aws-sdk/client-kms" "^3.975.0" "@aws-sdk/client-s3" "^3.975.0"