From 4838bb07ed880910c64a335859d6d6d2eb1e8f2c Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Thu, 12 Mar 2026 18:09:27 +0100 Subject: [PATCH 1/8] CLDSRV-863: bump arsenal to 8.3.6 --- package.json | 2 +- yarn.lock | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 18dbec49d8..eaee9d3d60 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "@azure/storage-blob": "^12.28.0", "@hapi/joi": "^17.1.1", "@smithy/node-http-handler": "^3.0.0", - "arsenal": "git+https://github.com/scality/Arsenal#8.3.5", + "arsenal": "git+https://github.com/scality/Arsenal#8.3.6", "async": "2.6.4", "bucketclient": "scality/bucketclient#8.2.7", "bufferutil": "^4.0.8", diff --git a/yarn.lock b/yarn.lock index f8ee0a1ff1..82bf9887f0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5766,9 +5766,9 @@ arraybuffer.prototype.slice@^1.0.4: optionalDependencies: ioctl "^2.0.2" -"arsenal@git+https://github.com/scality/Arsenal#8.3.5": - version "8.3.5" - resolved "git+https://github.com/scality/Arsenal#720a2a8e30998a8b08839bcbb73f059968e303b2" +"arsenal@git+https://github.com/scality/Arsenal#8.3.6": + version "8.3.6" + resolved "git+https://github.com/scality/Arsenal#21f6a54ac70f2f6c94b592760780b3ca3bab7a95" dependencies: "@aws-sdk/client-kms" "^3.975.0" "@aws-sdk/client-s3" "^3.975.0" From 464f12d2dc3d8a65495246b6de6bc6b3a33f34a2 Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Mon, 16 Mar 2026 15:57:42 +0100 Subject: [PATCH 2/8] CLDSRV-863: handle NoSuchKey error in isCompleteMPUInProgress --- lib/services.js | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/services.js b/lib/services.js index cbf8740170..f2980f1fc5 100644 --- a/lib/services.js +++ b/lib/services.js @@ -682,6 +682,16 @@ const services = { return metadata.getObjectMD(mpuBucketName, mpuOverviewKey, {}, log, (err, res) => { if (err) { + if (err.is && err.is.NoSuchKey) { + // The overview key no longer exists, meaning completeMultipartUpload + // already ran to completion and cleaned up the MPU bucket. + // This is a race condition: objectPutPart checked for old + // part locations after completeMultipartUpload deleted the overview. + // Returning true (complete in progress) prevents objectPutPart + // from deleting part data that may have already been committed + // as the final object. + return cb(null, true); + } log.error('error getting the overview object from mpu bucket', { error: err, method: 'services.isCompleteMPUInProgress', @@ -737,7 +747,7 @@ const services = { metadata.getObjectMD(mpuBucket.getName(), mpuOverviewKey, {}, log, (err, storedMetadata) => { if (err) { - if (err.is.NoSuchKey) { + if (err.is && err.is.NoSuchKey) { return cb(errors.NoSuchUpload); } log.error('error from metadata', { error: err }); From 09435e2e42c4eaa63f25f107cec5d9bbcc2ca17c Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Mon, 16 Mar 2026 15:59:11 +0100 Subject: [PATCH 3/8] CLDSRV-863: emit trailer in TrailingChecksumTransform --- .../streamingV4/trailingChecksumTransform.js | 76 +++++- .../raw-node/test/trailingChecksums.js | 3 +- tests/unit/auth/TrailingChecksumTransform.js | 232 +++++++++++++++++- 3 files changed, 293 insertions(+), 18 deletions(-) diff --git a/lib/auth/streamingV4/trailingChecksumTransform.js b/lib/auth/streamingV4/trailingChecksumTransform.js index 48870c80e7..687e2befac 100644 --- a/lib/auth/streamingV4/trailingChecksumTransform.js +++ b/lib/auth/streamingV4/trailingChecksumTransform.js @@ -3,10 +3,11 @@ const { errors } = require('arsenal'); const { maximumAllowedPartSize } = require('../../../constants'); /** - * This class is designed to handle the chunks sent in a streaming - * unsigned playload trailer request. In this iteration, we are not checking - * the checksums, but we are removing them from the stream. - * S3C-9732 will deal with checksum verification. + * This class handles the chunked-upload body format used by + * STREAMING-UNSIGNED-PAYLOAD-TRAILER requests. It strips the chunk-size + * headers and trailing checksum trailer from the stream, forwarding only + * the raw object data. The trailer name and value are emitted via a + * 'trailer' event so that ChecksumTransform can validate the checksum. */ class TrailingChecksumTransform extends Transform { /** @@ -20,6 +21,10 @@ class TrailingChecksumTransform extends Transform { this.bytesToDiscard = 0; // when trailing \r\n are present, we discard them but they can be in different chunks this.bytesToRead = 0; // when a chunk is advertised, the size is put here and we forward all bytes this.streamClosed = false; + this.readingTrailer = false; + this.trailerBuffer = Buffer.alloc(0); + this.trailerName = null; + this.trailerValue = null; } /** @@ -30,9 +35,18 @@ class TrailingChecksumTransform extends Transform { * @return {function} executes callback with err if applicable */ _flush(callback) { - if (!this.streamClosed) { + if (!this.streamClosed && this.readingTrailer && this.trailerBuffer.length === 0) { + // Nothing came after "0\r\n", don't fail. + // If the x-amz-trailer header was present then the trailer is required and ChecksumTransform will fail. + return callback(); + } else if (!this.streamClosed && this.readingTrailer && this.trailerBuffer.length !== 0) { + this.log.error('stream ended without trailer "\r\n"'); + return callback(errors.IncompleteBody.customizeDescription( + 'The request body terminated unexpectedly')); + } else if (!this.streamClosed && !this.readingTrailer) { this.log.error('stream ended without closing chunked encoding'); - return callback(errors.InvalidArgument); + return callback(errors.IncompleteBody.customizeDescription( + 'The request body terminated unexpectedly')); } return callback(); } @@ -66,6 +80,50 @@ class TrailingChecksumTransform extends Transform { continue; } + // after the 0-size chunk, read the trailer line (e.g. "x-amz-checksum-crc32:YABb/g==") + if (this.readingTrailer) { + const combined = Buffer.concat([this.trailerBuffer, chunk]); + const lineBreakIndex = combined.indexOf('\r\n'); + if (lineBreakIndex === -1) { + if (combined.byteLength > 1024) { + this.log.error('trailer line too long'); + return callback(errors.MalformedTrailerError); + } + // The trailer is not complete yet, continue. + this.trailerBuffer = combined; + return callback(); + } + this.trailerBuffer = Buffer.alloc(0); + const fullTrailer = combined.subarray(0, lineBreakIndex); + if (fullTrailer.length === 0) { + // The trailer is empty, stop reading. + this.readingTrailer = false; + this.streamClosed = true; + return callback(); + } + let trailerLine = fullTrailer.toString(); + // Some clients terminate the trailer with \n\r\n instead of + // just \r\n, producing a trailing \n in the parsed line. + if (trailerLine.endsWith('\n')) { + trailerLine = trailerLine.slice(0, -1); + } + const colonIndex = trailerLine.indexOf(':'); + if (colonIndex > 0) { + this.trailerName = trailerLine.slice(0, colonIndex).trim(); + this.trailerValue = trailerLine.slice(colonIndex + 1).trim(); + this.emit('trailer', this.trailerName, this.trailerValue); + } else { + this.log.error('incomplete trailer missing ":"', { trailerLine }); + return callback(errors.IncompleteBody.customizeDescription( + 'The request body terminated unexpectedly')); + } + this.readingTrailer = false; + this.streamClosed = true; + // The trailer \r\n is the last bytes of the stream per the AWS + // chunked upload format, so any remaining bytes are discarded. + return callback(); + } + // we are now looking for the chunk size field // no need to look further than 10 bytes since the field cannot be bigger: the max // chunk size is 5GB (see constants.maximumAllowedPartSize) @@ -100,9 +158,9 @@ class TrailingChecksumTransform extends Transform { } this.chunkSizeBuffer = Buffer.alloc(0); if (dataSize === 0) { - // TODO: check if the checksum is correct (S3C-9732) - // last chunk, no more data to read, the stream is closed - this.streamClosed = true; + // last chunk, no more data to read; enter trailer-reading mode + // bytesToDiscard = 2 below will consume the \r\n after "0" + this.readingTrailer = true; } if (dataSize > maximumAllowedPartSize) { this.log.error('chunk size too big', { dataSize }); diff --git a/tests/functional/raw-node/test/trailingChecksums.js b/tests/functional/raw-node/test/trailingChecksums.js index bad429c3c8..925d959a7f 100644 --- a/tests/functional/raw-node/test/trailingChecksums.js +++ b/tests/functional/raw-node/test/trailingChecksums.js @@ -6,10 +6,9 @@ const HttpRequestAuthV4 = require('../utils/HttpRequestAuthV4'); const bucket = 'testunsupportedchecksumsbucket'; const objectKey = 'key'; const objData = Buffer.alloc(1024, 'a'); -// note this is not the correct checksum in objDataWithTrailingChecksum const objDataWithTrailingChecksum = '10\r\n0123456789abcdef\r\n' + '10\r\n0123456789abcdef\r\n' + - '0\r\nx-amz-checksum-crc64nvme:YeIDuLa7tU0=\r\n'; + '0\r\nx-amz-checksum-crc64nvme:skQv82y5rgE=\r\n'; const objDataWithoutTrailingChecksum = '0123456789abcdef0123456789abcdef'; const config = require('../../config.json'); diff --git a/tests/unit/auth/TrailingChecksumTransform.js b/tests/unit/auth/TrailingChecksumTransform.js index dd88b3719b..cbb39a0309 100644 --- a/tests/unit/auth/TrailingChecksumTransform.js +++ b/tests/unit/auth/TrailingChecksumTransform.js @@ -4,9 +4,39 @@ const async = require('async'); const { Readable } = require('stream'); const TrailingChecksumTransform = require('../../../lib/auth/streamingV4/trailingChecksumTransform'); -const { stripTrailingChecksumStream } = require('../../../lib/api/apiUtils/object/prepareStream'); const { DummyRequestLogger } = require('../helpers'); +// Helper: pipe input chunks through TrailingChecksumTransform, collect output and trailer events +function runTransform(inputChunks) { + const stream = new TrailingChecksumTransform(new DummyRequestLogger()); + return new Promise((resolve, reject) => { + const output = []; + const trailerEvents = []; + stream.on('data', chunk => output.push(Buffer.from(chunk))); + stream.on('trailer', (name, value) => trailerEvents.push({ name, value })); + stream.on('finish', () => resolve({ data: Buffer.concat(output), trailers: trailerEvents, stream })); + stream.on('error', reject); + for (const chunk of inputChunks) { + stream.write(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + stream.end(); + }); +} + +// Helper: expect a stream error from the given input chunks +function expectError(inputChunks) { + const stream = new TrailingChecksumTransform(new DummyRequestLogger()); + return new Promise((resolve, reject) => { + stream.on('error', resolve); + stream.on('finish', () => reject(new Error('expected error but stream finished cleanly'))); + for (const chunk of inputChunks) { + stream.write(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + stream.end(); + stream.resume(); + }); +} + const log = new DummyRequestLogger(); // note this is not the correct checksum in objDataWithTrailingChecksum @@ -174,30 +204,33 @@ describe('TrailingChecksumTransform class', () => { it('should propagate _flush error via errCb when stream closes without chunked encoding', done => { const incompleteData = '10\r\n01234\r6789abcd\r\n\r\n'; const source = new ChunkedReader([Buffer.from(incompleteData)]); - source.headers = { 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER' }; - const stream = stripTrailingChecksumStream(source, log, err => { - assert.deepStrictEqual(err, errors.InvalidArgument); + const stream = new TrailingChecksumTransform(log); + stream.on('error', err => { + assert.deepStrictEqual(err, errors.IncompleteBody); done(); }); + source.pipe(stream); stream.resume(); }); it('should propagate _transform error via errCb for invalid chunk size', done => { const badData = '500000000000\r\n'; const source = new ChunkedReader([Buffer.from(badData)]); - source.headers = { 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER' }; - const stream = stripTrailingChecksumStream(source, log, err => { + const stream = new TrailingChecksumTransform(log); + stream.on('error', err => { assert.deepStrictEqual(err, errors.InvalidArgument); done(); }); + source.pipe(stream); stream.resume(); }); - it('should return early if supplied with an out of specification chunk size', done => { + it('should return early if supplied with an out-of-specification chunk size', done => { const trailingChecksumTransform = new TrailingChecksumTransform(log); const chunks = [ Buffer.from('500000'), Buffer.from('000000\r\n'), + Buffer.alloc(1000000), Buffer.alloc(1000000), Buffer.alloc(1000000), @@ -227,3 +260,188 @@ describe('TrailingChecksumTransform class', () => { chunkedReader.pipe(trailingChecksumTransform); }); }); + +describe('TrailingChecksumTransform trailer parsing and emitting', () => { + describe('happy path', () => { + it('single chunk with data and trailer: forwards data, emits trailer name and value', async () => { + const input = '5\r\nhello\r\n0\r\nx-amz-checksum-crc32:AAAAAA==\r\n'; + const { data, trailers } = await runTransform([input]); + assert.strictEqual(data.toString(), 'hello'); + assert.strictEqual(trailers.length, 1); + assert.strictEqual(trailers[0].name, 'x-amz-checksum-crc32'); + assert.strictEqual(trailers[0].value, 'AAAAAA=='); + }); + + it('multiple data chunks followed by trailer: forwards all data, emits trailer once', async () => { + const input = '5\r\nhello\r\n5\r\nworld\r\n0\r\nx-amz-checksum-sha256:AAAAAA==\r\n'; + const { data, trailers } = await runTransform([input]); + assert.strictEqual(data.toString(), 'helloworld'); + assert.strictEqual(trailers.length, 1); + assert.strictEqual(trailers[0].name, 'x-amz-checksum-sha256'); + }); + + it('data chunk containing \\r\\n in payload is forwarded correctly', async () => { + // 7 bytes: h e l \r \n l o + const input = '7\r\nhel\r\nlo\r\n0\r\nx-amz-checksum-crc32:AAAAAA==\r\n'; + const { data } = await runTransform([input]); + assert.strictEqual(data.toString(), 'hel\r\nlo'); + }); + + it('trailer with whitespace around name and value: name and value are trimmed', async () => { + const input = '0\r\n x-amz-checksum-crc32 : AAAAAA== \r\n'; + const { trailers } = await runTransform([input]); + assert.strictEqual(trailers.length, 1); + assert.strictEqual(trailers[0].name, 'x-amz-checksum-crc32'); + assert.strictEqual(trailers[0].value, 'AAAAAA=='); + }); + + it('trailer terminated with \\n\\r\\n: trailing \\n stripped from parsed line', async () => { + const input = '0\r\nx-amz-checksum-crc32:AAAAAA==\n\r\n'; + const { trailers } = await runTransform([input]); + assert.strictEqual(trailers.length, 1); + assert.strictEqual(trailers[0].name, 'x-amz-checksum-crc32'); + assert.strictEqual(trailers[0].value, 'AAAAAA=='); + }); + + it('trailer value containing a colon: only first colon used as separator', async () => { + const input = '0\r\nx-amz-checksum-crc32:AA:BB==\r\n'; + const { trailers } = await runTransform([input]); + assert.strictEqual(trailers.length, 1); + assert.strictEqual(trailers[0].name, 'x-amz-checksum-crc32'); + assert.strictEqual(trailers[0].value, 'AA:BB=='); + }); + + it('bytes after trailer \\r\\n are silently discarded', async () => { + const input = '5\r\nhello\r\n0\r\nx-amz-checksum-crc32:AAAAAA==\r\nextra bytes ignored'; + const { data, trailers } = await runTransform([input]); + assert.strictEqual(data.toString(), 'hello'); + assert.strictEqual(trailers.length, 1); + }); + }); + + describe('chunk boundary edge cases', () => { + it('chunk size field split across two input chunks: parsed correctly', async () => { + // size 'a' (hex) = 10 bytes; split the size field across two chunks + const c1 = 'a'; + const c2 = '\r\nAAAAAAAAAA\r\n0\r\nx-amz-checksum-crc32:AAAAAA==\r\n'; + const { data, trailers } = await runTransform([c1, c2]); + assert.strictEqual(data.toString(), 'AAAAAAAAAA'); + assert.strictEqual(trailers.length, 1); + assert.strictEqual(trailers[0].name, 'x-amz-checksum-crc32'); + }); + + it('data bytes split across two input chunks: all forwarded', async () => { + // 5 bytes 'hello', split after 'hel' + const c1 = '5\r\nhel'; + const c2 = 'lo\r\n0\r\nx-amz-checksum-crc32:AAAAAA==\r\n'; + const { data, trailers } = await runTransform([c1, c2]); + assert.strictEqual(data.toString(), 'hello'); + assert.strictEqual(trailers.length, 1); + }); + + it('\\r\\n delimiter after chunk size split across two input chunks: parsed correctly', async () => { + // '5\r' in chunk1, '\nhello\r\n0\r\n...' in chunk2 + const c1 = '5\r'; + const c2 = '\nhello\r\n0\r\nx-amz-checksum-crc32:AAAAAA==\r\n'; + const { data, trailers } = await runTransform([c1, c2]); + assert.strictEqual(data.toString(), 'hello'); + assert.strictEqual(trailers.length, 1); + }); + + it('trailer line split across two input chunks: emits trailer correctly', async () => { + const c1 = '5\r\nhello\r\n0\r\nx-amz-checksum-'; + const c2 = 'crc32:AAAAAA==\r\n'; + const { data, trailers } = await runTransform([c1, c2]); + assert.strictEqual(data.toString(), 'hello'); + assert.strictEqual(trailers.length, 1); + assert.strictEqual(trailers[0].name, 'x-amz-checksum-crc32'); + assert.strictEqual(trailers[0].value, 'AAAAAA=='); + }); + + it('trailer \\r\\n split across two input chunks: emits trailer correctly', async () => { + const c1 = '5\r\nhello\r\n0\r\nx-amz-checksum-crc32:AAAAAA==\r'; + const c2 = '\n'; + const { data, trailers } = await runTransform([c1, c2]); + assert.strictEqual(data.toString(), 'hello'); + assert.strictEqual(trailers.length, 1); + assert.strictEqual(trailers[0].name, 'x-amz-checksum-crc32'); + assert.strictEqual(trailers[0].value, 'AAAAAA=='); + }); + }); + + describe('zero-size terminator and trailer', () => { + it('empty trailer line (0\\r\\n\\r\\n): no trailer event emitted, stream closes cleanly', async () => { + const input = '5\r\nhello\r\n0\r\n\r\n'; + const { data, trailers } = await runTransform([input]); + assert.strictEqual(data.toString(), 'hello'); + assert.strictEqual(trailers.length, 0); + }); + + it('zero data chunks (only terminator + trailer): no data forwarded, trailer emitted', async () => { + const input = '0\r\nx-amz-checksum-crc32:AAAAAA==\r\n'; + const { data, trailers } = await runTransform([input]); + assert.strictEqual(data.length, 0); + assert.strictEqual(trailers.length, 1); + assert.strictEqual(trailers[0].name, 'x-amz-checksum-crc32'); + assert.strictEqual(trailers[0].value, 'AAAAAA=='); + }); + }); + + describe('_flush error cases', () => { + it('stream ends mid-data (no zero-chunk): IncompleteBody error', async () => { + // 5 bytes declared but stream ends after only 3 + const err = await expectError(['5\r\nhel']); + assert.deepStrictEqual(err, errors.IncompleteBody); + }); + + it('stream ends after zero-chunk with partial trailer content: IncompleteBody error', async () => { + // zero-chunk received, trailer starts but no \r\n terminator + const err = await expectError(['0\r\nx-amz-checksum-crc32:AAAAAA==']); + assert.deepStrictEqual(err, errors.IncompleteBody); + }); + + it('stream ends after zero-chunk with no trailer content: no error', async () => { + // only '0\r\n' — readingTrailer=true, trailerBuffer empty → no error + const { data, trailers } = await runTransform(['0\r\n']); + assert.strictEqual(data.length, 0); + assert.strictEqual(trailers.length, 0); + }); + }); + + describe('_transform error cases', () => { + it('chunk size field larger than 10 bytes: InvalidArgument error', async () => { + // 11 hex digits — exceeds the 10-byte field size limit + const err = await expectError(['12345678901\r\n']); + assert.deepStrictEqual(err, errors.InvalidArgument); + }); + + it('chunk size is not valid hex: InvalidArgument error', async () => { + // 2 chars, short enough to pass size check, but not valid hex + const err = await expectError(['zz\r\n']); + assert.deepStrictEqual(err, errors.InvalidArgument); + }); + + it('chunk size exceeds maximumAllowedPartSize: EntityTooLarge error', async () => { + // 0x200000000 = 8589934592 > maximumAllowedPartSize (5GB = 0x140000000) + const err = await expectError(['200000000\r\n']); + assert.deepStrictEqual(err, errors.EntityTooLarge); + }); + + it('trailer line longer than 1024 bytes: MalformedTrailerError', async () => { + // send zero-chunk then a trailer line > 1024 bytes with no \r\n + const longTrailer = 'x'.repeat(1025); + const err = await expectError([`0\r\n${longTrailer}`]); + assert.deepStrictEqual(err, errors.MalformedTrailerError); + }); + + it('trailer line missing colon: IncompleteBody error', async () => { + const err = await expectError(['0\r\nnocolon\r\n']); + assert.deepStrictEqual(err, errors.IncompleteBody); + }); + + it('trailer line with colon at position 0 (empty name): IncompleteBody error', async () => { + const err = await expectError(['0\r\n:value\r\n']); + assert.deepStrictEqual(err, errors.IncompleteBody); + }); + }); +}); From 7c7367dbac5f7ae381a17c311586ee758be9658a Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Mon, 16 Mar 2026 16:00:30 +0100 Subject: [PATCH 4/8] CLDSRV-863: handle trailer headers and add new trailer errors --- .../apiUtils/integrity/validateChecksums.js | 155 +++++++++- .../apiUtils/integrity/validateChecksums.js | 281 +++++++++++++++++- 2 files changed, 423 insertions(+), 13 deletions(-) diff --git a/lib/api/apiUtils/integrity/validateChecksums.js b/lib/api/apiUtils/integrity/validateChecksums.js index 9643a71258..e440389ab2 100644 --- a/lib/api/apiUtils/integrity/validateChecksums.js +++ b/lib/api/apiUtils/integrity/validateChecksums.js @@ -37,6 +37,12 @@ const ChecksumError = Object.freeze({ MultipleChecksumTypes: 'MultipleChecksumTypes', MissingCorresponding: 'MissingCorresponding', MalformedChecksum: 'MalformedChecksum', + TrailerAlgoMismatch: 'TrailerAlgoMismatch', + TrailerChecksumMalformed: 'TrailerChecksumMalformed', + TrailerMissing: 'TrailerMissing', + TrailerUnexpected: 'TrailerUnexpected', + TrailerAndChecksum: 'TrailerAndChecksum', + TrailerNotSupported: 'TrailerNotSupported', }); const base64Regex = /^[A-Za-z0-9+/]*={0,2}$/; @@ -56,35 +62,51 @@ const algorithms = Object.freeze({ const result = await crc.digest(); return Buffer.from(result).toString('base64'); }, + digestFromHash: async hash => { + const result = await hash.digest(); + return Buffer.from(result).toString('base64'); + }, isValidDigest: expected => typeof expected === 'string' && expected.length === 12 && base64Regex.test(expected), + createHash: () => new CrtCrc64Nvme() }, crc32: { digest: data => { const input = Buffer.isBuffer(data) ? data : Buffer.from(data); return uint32ToBase64(new Crc32().update(input).digest() >>> 0); // >>> 0 coerce number to uint32 }, + digestFromHash: hash => { + const result = hash.digest(); + return uint32ToBase64(result >>> 0); + }, isValidDigest: expected => typeof expected === 'string' && expected.length === 8 && base64Regex.test(expected), + createHash: () => new Crc32() }, crc32c: { digest: data => { const input = Buffer.isBuffer(data) ? data : Buffer.from(data); return uint32ToBase64(new Crc32c().update(input).digest() >>> 0); // >>> 0 coerce number to uint32 }, + digestFromHash: hash => uint32ToBase64(hash.digest() >>> 0), isValidDigest: expected => typeof expected === 'string' && expected.length === 8 && base64Regex.test(expected), + createHash: () => new Crc32c() }, sha1: { digest: data => { const input = Buffer.isBuffer(data) ? data : Buffer.from(data); return crypto.createHash('sha1').update(input).digest('base64'); }, + digestFromHash: hash => hash.digest('base64'), isValidDigest: expected => typeof expected === 'string' && expected.length === 28 && base64Regex.test(expected), + createHash: () => crypto.createHash('sha1') }, sha256: { digest: data => { const input = Buffer.isBuffer(data) ? data : Buffer.from(data); return crypto.createHash('sha256').update(input).digest('base64'); }, + digestFromHash: hash => hash.digest('base64'), isValidDigest: expected => typeof expected === 'string' && expected.length === 44 && base64Regex.test(expected), + createHash: () => crypto.createHash('sha256') } }); @@ -132,7 +154,7 @@ async function validateXAmzChecksums(headers, body) { return { error: ChecksumError.AlgoNotSupportedSDK, details: { algorithm: sdkAlgo } }; } - // If AWS there is a mismatch, AWS returns the same error as if the algo was invalid. + // If there is a mismatch, AWS returns the same error as if the algo was invalid. if (sdkLowerAlgo !== algo) { return { error: ChecksumError.AlgoNotSupportedSDK, details: { algorithm: sdkAlgo } }; } @@ -141,6 +163,94 @@ async function validateXAmzChecksums(headers, body) { return null; } +function getChecksumDataFromHeaders(headers) { + const checkSdk = algo => { + if (!('x-amz-sdk-checksum-algorithm' in headers)) { + return null; + } + + const sdkAlgo = headers['x-amz-sdk-checksum-algorithm']; + if (typeof sdkAlgo !== 'string') { + return { error: ChecksumError.AlgoNotSupportedSDK, details: { algorithm: sdkAlgo } }; + } + + const sdkLowerAlgo = sdkAlgo.toLowerCase(); + if (!(sdkLowerAlgo in algorithms)) { + return { error: ChecksumError.AlgoNotSupportedSDK, details: { algorithm: sdkAlgo } }; + } + + // If there is a mismatch, AWS returns the same error as if the algo was invalid. + if (sdkLowerAlgo !== algo) { + return { error: ChecksumError.AlgoNotSupportedSDK, details: { algorithm: sdkAlgo } }; + } + + return null; + }; + + const checksumHeaders = Object.keys(headers).filter(header => header.startsWith('x-amz-checksum-')); + const xAmzChecksumCnt = checksumHeaders.length; + if (xAmzChecksumCnt > 1) { + return { error: ChecksumError.MultipleChecksumTypes, details: { algorithms: checksumHeaders } }; + } + + if (xAmzChecksumCnt === 0 && !('x-amz-trailer' in headers) && 'x-amz-sdk-checksum-algorithm' in headers) { + return { + error: ChecksumError.MissingCorresponding, + details: { expected: headers['x-amz-sdk-checksum-algorithm'] } + }; + } + + if ('x-amz-trailer' in headers) { + if (xAmzChecksumCnt !== 0) { + return { + error: ChecksumError.TrailerAndChecksum, + details: { trailer: headers['x-amz-trailer'], checksum: checksumHeaders }, + }; + } + + const trailer = headers['x-amz-trailer']; + if (!trailer.startsWith('x-amz-checksum-')) { + return { error: ChecksumError.TrailerNotSupported, details: { value: trailer } }; + } + + const trailerAlgo = trailer.slice('x-amz-checksum-'.length); + if (!(trailerAlgo in algorithms)) { + return { error: ChecksumError.TrailerNotSupported, details: { value: trailer } }; + } + + const err = checkSdk(trailerAlgo); + if (err) { + return err; + } + + return { algorithm: trailerAlgo, isTrailer: true, expected: undefined }; + } + + if (xAmzChecksumCnt === 0) { + // There was no x-amz-checksum- or x-amz-trailer return crc64nvme. + // The calculated crc64nvme will be stored in the object metadata. + return { algorithm: 'crc64nvme', isTrailer: false, expected: undefined }; + } + + // No x-amz-sdk-checksum-algorithm we expect one x-amz-checksum-[crc64nvme, crc32, crc32C, sha1, sha256]. + const algo = checksumHeaders[0].slice('x-amz-checksum-'.length); + if (!(algo in algorithms)) { + return { error: ChecksumError.AlgoNotSupported, details: { algorithm: algo } }; + } + + const expected = headers[`x-amz-checksum-${algo}`]; + if (!algorithms[algo].isValidDigest(expected)) { + return { error: ChecksumError.MalformedChecksum, details: { algorithm: algo, expected } }; + } + + const err = checkSdk(algo); + if (err) { + return err; + } + + return { algorithm: algo, isTrailer: false, expected }; +} + /** * validateChecksumsNoChunking - Validate the checksums of a request. * @param {object} headers - http headers @@ -183,16 +293,7 @@ async function validateChecksumsNoChunking(headers, body) { return err; } -async function defaultValidationFunc(request, body, log) { - const err = await validateChecksumsNoChunking(request.headers, body); - if (!err) { - return null; - } - - if (err.error !== ChecksumError.MissingChecksum) { - log.debug('failed checksum validation', { method: request.apiMethod }, err); - } - +function arsenalErrorFromChecksumError(err) { switch (err.error) { case ChecksumError.MissingChecksum: return null; @@ -225,11 +326,40 @@ async function defaultValidationFunc(request, body, log) { ); case ChecksumError.MD5Invalid: return ArsenalErrors.InvalidDigest; + case ChecksumError.TrailerAlgoMismatch: + return ArsenalErrors.MalformedTrailerError; + case ChecksumError.TrailerMissing: + return ArsenalErrors.MalformedTrailerError; + case ChecksumError.TrailerUnexpected: + return ArsenalErrors.MalformedTrailerError; + case ChecksumError.TrailerChecksumMalformed: + return ArsenalErrors.InvalidRequest.customizeDescription( + `Value for x-amz-checksum-${err.details.algorithm} trailing header is invalid.` + ); + case ChecksumError.TrailerAndChecksum: + return ArsenalErrors.InvalidRequest.customizeDescription('Expecting a single x-amz-checksum- header'); + case ChecksumError.TrailerNotSupported: + return ArsenalErrors.InvalidRequest.customizeDescription( + 'The value specified in the x-amz-trailer header is not supported' + ); default: return ArsenalErrors.BadDigest; } } +async function defaultValidationFunc(request, body, log) { + const err = await validateChecksumsNoChunking(request.headers, body); + if (!err) { + return null; + } + + if (err.error !== ChecksumError.MissingChecksum) { + log.debug('failed checksum validation', { method: request.apiMethod }, err); + } + + return arsenalErrorFromChecksumError(err); +} + /** * validateMethodChecksumsNoChunking - Validate the checksums of a request. * @param {object} request - http request @@ -253,5 +383,8 @@ module.exports = { ChecksumError, validateChecksumsNoChunking, validateMethodChecksumNoChunking, + getChecksumDataFromHeaders, + arsenalErrorFromChecksumError, + algorithms, checksumedMethods, }; diff --git a/tests/unit/api/apiUtils/integrity/validateChecksums.js b/tests/unit/api/apiUtils/integrity/validateChecksums.js index a234dcfd60..925d79a713 100644 --- a/tests/unit/api/apiUtils/integrity/validateChecksums.js +++ b/tests/unit/api/apiUtils/integrity/validateChecksums.js @@ -2,8 +2,14 @@ const assert = require('assert'); const crypto = require('crypto'); const sinon = require('sinon'); -const { validateChecksumsNoChunking, ChecksumError, validateMethodChecksumNoChunking, checksumedMethods } = - require('../../../../../lib/api/apiUtils/integrity/validateChecksums'); +const { + validateChecksumsNoChunking, + ChecksumError, + validateMethodChecksumNoChunking, + checksumedMethods, + getChecksumDataFromHeaders, + arsenalErrorFromChecksumError, +} = require('../../../../../lib/api/apiUtils/integrity/validateChecksums'); const { errors: ArsenalErrors } = require('arsenal'); const { config } = require('../../../../../lib/Config'); @@ -453,3 +459,274 @@ describe('validateMethodChecksumNoChunking', () => { }); }); }); + +describe('getChecksumDataFromHeaders', () => { + // Valid-format digests (correct length and base64, content not verified by getChecksumDataFromHeaders) + const validDigests = { + crc32: 'AAAAAA==', // 8 chars + crc32c: 'AAAAAA==', // 8 chars + sha1: 'AAAAAAAAAAAAAAAAAAAAAAAAAAA=', // 28 chars + sha256: 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=', // 44 chars + crc64nvme: 'AAAAAAAAAAA=', // 12 chars + }; + + it('no headers: returns crc64nvme with isTrailer=false and expected=undefined', () => { + const result = getChecksumDataFromHeaders({}); + assert.deepStrictEqual(result, { algorithm: 'crc64nvme', isTrailer: false, expected: undefined }); + }); + + it('no checksum headers, no trailer, no sdk algo: returns crc64nvme default', () => { + const result = getChecksumDataFromHeaders({ 'content-type': 'application/octet-stream' }); + assert.deepStrictEqual(result, { algorithm: 'crc64nvme', isTrailer: false, expected: undefined }); + }); + + for (const [algo, digest] of Object.entries(validDigests)) { + it(`x-amz-checksum-${algo} with valid digest: returns algorithm, isTrailer=false, expected`, () => { + const result = getChecksumDataFromHeaders({ [`x-amz-checksum-${algo}`]: digest }); + assert.deepStrictEqual(result, { algorithm: algo, isTrailer: false, expected: digest }); + }); + } + + it('x-amz-checksum-unknown-algo: returns AlgoNotSupported error', () => { + const result = getChecksumDataFromHeaders({ 'x-amz-checksum-md4': 'AAAAAA==' }); + assert.strictEqual(result.error, ChecksumError.AlgoNotSupported); + assert.strictEqual(result.details.algorithm, 'md4'); + }); + + it('x-amz-checksum-crc32 with malformed digest (wrong length): returns MalformedChecksum error', () => { + const result = getChecksumDataFromHeaders({ 'x-amz-checksum-crc32': 'AAAAA==' }); // 7 chars, crc32 needs 8 + assert.strictEqual(result.error, ChecksumError.MalformedChecksum); + assert.strictEqual(result.details.algorithm, 'crc32'); + }); + + it('x-amz-checksum-crc32 with malformed digest (invalid base64): returns MalformedChecksum error', () => { + const result = getChecksumDataFromHeaders({ 'x-amz-checksum-crc32': '!!!!!!!!' }); + assert.strictEqual(result.error, ChecksumError.MalformedChecksum); + assert.strictEqual(result.details.algorithm, 'crc32'); + }); + + it('two x-amz-checksum- headers: returns MultipleChecksumTypes error', () => { + const result = getChecksumDataFromHeaders({ + 'x-amz-checksum-crc32': validDigests.crc32, + 'x-amz-checksum-sha256': validDigests.sha256, + }); + assert.strictEqual(result.error, ChecksumError.MultipleChecksumTypes); + }); + + it('x-amz-sdk-checksum-algorithm with no x-amz-checksum- and no x-amz-trailer: returns MissingCorresponding', + () => { + const result = getChecksumDataFromHeaders({ 'x-amz-sdk-checksum-algorithm': 'crc32' }); + assert.strictEqual(result.error, ChecksumError.MissingCorresponding); + assert.strictEqual(result.details.expected, 'crc32'); + }); + + it('x-amz-checksum-crc32 with matching x-amz-sdk-checksum-algorithm CRC32: returns success', () => { + const result = getChecksumDataFromHeaders({ + 'x-amz-checksum-crc32': validDigests.crc32, + 'x-amz-sdk-checksum-algorithm': 'crc32', + }); + assert.deepStrictEqual(result, { algorithm: 'crc32', isTrailer: false, expected: validDigests.crc32 }); + }); + + it('x-amz-checksum-crc32 with mismatched x-amz-sdk-checksum-algorithm SHA256: returns AlgoNotSupportedSDK', () => { + const result = getChecksumDataFromHeaders({ + 'x-amz-checksum-crc32': validDigests.crc32, + 'x-amz-sdk-checksum-algorithm': 'sha256', + }); + assert.strictEqual(result.error, ChecksumError.AlgoNotSupportedSDK); + assert.strictEqual(result.details.algorithm, 'sha256'); + }); + + it('x-amz-checksum-crc32 with non-string x-amz-sdk-checksum-algorithm: returns AlgoNotSupportedSDK', () => { + const result = getChecksumDataFromHeaders({ + 'x-amz-checksum-crc32': validDigests.crc32, + 'x-amz-sdk-checksum-algorithm': 1234, + }); + assert.strictEqual(result.error, ChecksumError.AlgoNotSupportedSDK); + assert.strictEqual(result.details.algorithm, 1234); + }); + + it('x-amz-checksum-crc32 with unknown x-amz-sdk-checksum-algorithm: returns AlgoNotSupportedSDK', () => { + const result = getChecksumDataFromHeaders({ + 'x-amz-checksum-crc32': validDigests.crc32, + 'x-amz-sdk-checksum-algorithm': 'md4', + }); + assert.strictEqual(result.error, ChecksumError.AlgoNotSupportedSDK); + assert.strictEqual(result.details.algorithm, 'md4'); + }); + + it('x-amz-trailer: x-amz-checksum-crc32: returns isTrailer=true', () => { + const result = getChecksumDataFromHeaders({ 'x-amz-trailer': 'x-amz-checksum-crc32' }); + assert.deepStrictEqual(result, { algorithm: 'crc32', isTrailer: true, expected: undefined }); + }); + + it('x-amz-trailer: x-amz-checksum-crc64nvme: returns isTrailer=true', () => { + const result = getChecksumDataFromHeaders({ 'x-amz-trailer': 'x-amz-checksum-crc64nvme' }); + assert.deepStrictEqual(result, { algorithm: 'crc64nvme', isTrailer: true, expected: undefined }); + }); + + it('x-amz-trailer with unsupported value (not x-amz-checksum- prefix): returns TrailerNotSupported', () => { + const result = getChecksumDataFromHeaders({ 'x-amz-trailer': 'x-custom-header' }); + assert.strictEqual(result.error, ChecksumError.TrailerNotSupported); + assert.strictEqual(result.details.value, 'x-custom-header'); + }); + + it('x-amz-trailer: x-amz-checksum-unknown-algo: returns TrailerNotSupported', () => { + const result = getChecksumDataFromHeaders({ 'x-amz-trailer': 'x-amz-checksum-md4' }); + assert.strictEqual(result.error, ChecksumError.TrailerNotSupported); + assert.strictEqual(result.details.value, 'x-amz-checksum-md4'); + }); + + it('x-amz-trailer with also an x-amz-checksum- header: returns TrailerAndChecksum', () => { + const result = getChecksumDataFromHeaders({ + 'x-amz-trailer': 'x-amz-checksum-crc32', + 'x-amz-checksum-crc32': validDigests.crc32, + }); + assert.strictEqual(result.error, ChecksumError.TrailerAndChecksum); + }); + + it('x-amz-trailer with matching x-amz-sdk-checksum-algorithm: returns success', () => { + const result = getChecksumDataFromHeaders({ + 'x-amz-trailer': 'x-amz-checksum-crc32', + 'x-amz-sdk-checksum-algorithm': 'crc32', + }); + assert.deepStrictEqual(result, { algorithm: 'crc32', isTrailer: true, expected: undefined }); + }); + + it('x-amz-trailer with mismatched x-amz-sdk-checksum-algorithm: returns AlgoNotSupportedSDK', () => { + const result = getChecksumDataFromHeaders({ + 'x-amz-trailer': 'x-amz-checksum-crc32', + 'x-amz-sdk-checksum-algorithm': 'sha256', + }); + assert.strictEqual(result.error, ChecksumError.AlgoNotSupportedSDK); + assert.strictEqual(result.details.algorithm, 'sha256'); + }); +}); + +describe('arsenalErrorFromChecksumError', () => { + it('MissingChecksum: returns null', () => { + const result = arsenalErrorFromChecksumError({ error: ChecksumError.MissingChecksum, details: null }); + assert.strictEqual(result, null); + }); + + it('XAmzMismatch (crc32): returns BadDigest mentioning CRC32', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.XAmzMismatch, + details: { algorithm: 'crc32', calculated: 'a', expected: 'b' }, + }); + assert(result.is.BadDigest); + assert.strictEqual(result.description, 'The CRC32 you specified did not match the calculated checksum.'); + }); + + it('XAmzMismatch (sha256): returns BadDigest mentioning SHA256', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.XAmzMismatch, + details: { algorithm: 'sha256', calculated: 'a', expected: 'b' }, + }); + assert(result.is.BadDigest); + assert.strictEqual(result.description, 'The SHA256 you specified did not match the calculated checksum.'); + }); + + it('AlgoNotSupported: returns InvalidRequest', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.AlgoNotSupported, + details: { algorithm: 'md4' }, + }); + assert(result.is.InvalidRequest); + }); + + it('AlgoNotSupportedSDK: returns InvalidRequest', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.AlgoNotSupportedSDK, + details: { algorithm: 'md4' }, + }); + assert(result.is.InvalidRequest); + }); + + it('MissingCorresponding: returns InvalidRequest', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.MissingCorresponding, + details: { expected: 'crc32' }, + }); + assert(result.is.InvalidRequest); + }); + + it('MultipleChecksumTypes: returns InvalidRequest', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.MultipleChecksumTypes, + details: { algorithms: ['x-amz-checksum-crc32', 'x-amz-checksum-sha256'] }, + }); + assert(result.is.InvalidRequest); + }); + + it('MalformedChecksum (crc32): returns InvalidRequest mentioning crc32', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.MalformedChecksum, + details: { algorithm: 'crc32', expected: 'bad' }, + }); + assert(result.is.InvalidRequest); + assert.strictEqual(result.description, 'Value for x-amz-checksum-crc32 header is invalid.'); + }); + + it('MD5Invalid: returns InvalidDigest', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.MD5Invalid, + details: { expected: 'bad' }, + }); + assert.deepStrictEqual(result, ArsenalErrors.InvalidDigest); + }); + + it('TrailerAlgoMismatch: returns MalformedTrailerError', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.TrailerAlgoMismatch, + details: { algorithm: 'crc32' }, + }); + assert.deepStrictEqual(result, ArsenalErrors.MalformedTrailerError); + }); + + it('TrailerMissing: returns MalformedTrailerError', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.TrailerMissing, + details: { expectedTrailer: 'x-amz-checksum-crc32' }, + }); + assert.deepStrictEqual(result, ArsenalErrors.MalformedTrailerError); + }); + + it('TrailerUnexpected: returns MalformedTrailerError', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.TrailerUnexpected, + details: { name: 'x-amz-checksum-crc32', val: 'AAAAAA==' }, + }); + assert.deepStrictEqual(result, ArsenalErrors.MalformedTrailerError); + }); + + it('TrailerChecksumMalformed: returns InvalidRequest mentioning the algorithm', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.TrailerChecksumMalformed, + details: { algorithm: 'sha256', expected: 'bad' }, + }); + assert(result.is.InvalidRequest); + assert.strictEqual(result.description, 'Value for x-amz-checksum-sha256 trailing header is invalid.'); + }); + + it('TrailerAndChecksum: returns InvalidRequest', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.TrailerAndChecksum, + details: { trailer: 'x-amz-checksum-crc32', checksum: ['x-amz-checksum-crc32'] }, + }); + assert(result.is.InvalidRequest); + }); + + it('TrailerNotSupported: returns InvalidRequest', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.TrailerNotSupported, + details: { value: 'x-custom-header' }, + }); + assert(result.is.InvalidRequest); + }); + + it('unknown error type (default): returns BadDigest', () => { + const result = arsenalErrorFromChecksumError({ error: 'SomeUnknownError', details: null }); + assert.deepStrictEqual(result, ArsenalErrors.BadDigest); + }); +}); From 588f37e424166dd43dbf5cab9734be4966417d11 Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Mon, 16 Mar 2026 16:00:58 +0100 Subject: [PATCH 5/8] CLDSRV-863: add ChecksumTransform to calculate and verify stream checksums --- lib/auth/streamingV4/ChecksumTransform.js | 92 +++++++++++ tests/unit/auth/ChecksumTransform.js | 182 ++++++++++++++++++++++ 2 files changed, 274 insertions(+) create mode 100644 lib/auth/streamingV4/ChecksumTransform.js create mode 100644 tests/unit/auth/ChecksumTransform.js diff --git a/lib/auth/streamingV4/ChecksumTransform.js b/lib/auth/streamingV4/ChecksumTransform.js new file mode 100644 index 0000000000..38ad6f1d8b --- /dev/null +++ b/lib/auth/streamingV4/ChecksumTransform.js @@ -0,0 +1,92 @@ +const { errors } = require('arsenal'); +const { algorithms, ChecksumError } = require('../../api/apiUtils/integrity/validateChecksums'); +const { Transform } = require('stream'); + +class ChecksumTransform extends Transform { + constructor(algoName, expectedDigest, isTrailer, log) { + super({}); + this.log = log; + this.algoName = algoName; + this.algo = algorithms[algoName]; + this.hash = this.algo.createHash(); + this.digest = undefined; + this.expectedDigest = expectedDigest; + this.isTrailer = isTrailer; + this.trailerChecksumName = undefined; + this.trailerChecksumValue = undefined; + } + + setExpectedChecksum(name, value) { + this.trailerChecksumName = name; + this.trailerChecksumValue = value; + } + + validateChecksum() { + if (this.isTrailer) { + // x-amz-trailer in headers but no trailer in body. + if (this.trailerChecksumValue === undefined) { + return { + error: ChecksumError.TrailerMissing, + details: { expectedTrailer: `x-amz-checksum-${this.algoName}` }, + }; + } + + if (this.trailerChecksumName !== `x-amz-checksum-${this.algoName}`) { + return { error: ChecksumError.TrailerAlgoMismatch, details: { algorithm: this.algoName } }; + } + + const expected = this.trailerChecksumValue; + if (!this.algo.isValidDigest(expected)) { + return { + error: ChecksumError.TrailerChecksumMalformed, + details: { algorithm: this.algoName, expected }, + }; + } + + if (this.digest !== this.trailerChecksumValue) { + return { + error: ChecksumError.XAmzMismatch, + details: { algorithm: this.algoName, calculated: this.digest, expected }, + }; + } + + return null; + } + + if (this.trailerChecksumValue) { + // Trailer found in the body but no x-amz-trailer in the headers. + return { + error: ChecksumError.TrailerUnexpected, + details: { name: this.trailerChecksumName, val: this.trailerChecksumValue }, + }; + } + + if (this.expectedDigest) { + if (this.digest !== this.expectedDigest) { + return { + error: ChecksumError.XAmzMismatch, + details: { algorithm: this.algoName, calculated: this.digest, expected: this.expectedDigest }, + }; + } + } + + return null; + } + + _flush(callback) { + Promise.resolve(this.algo.digestFromHash(this.hash)) + .then(digest => { this.digest = digest; }) + .then(() => callback(), err => { + this.log.error('failed to compute checksum digest', { error: err, algorithm: this.algoName }); + callback(errors.InternalError); + }); + } + + _transform(chunk, encoding, callback) { + const input = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + this.hash.update(input, encoding); + callback(null, input); + } +} + +module.exports = ChecksumTransform; diff --git a/tests/unit/auth/ChecksumTransform.js b/tests/unit/auth/ChecksumTransform.js new file mode 100644 index 0000000000..732d7f8cd4 --- /dev/null +++ b/tests/unit/auth/ChecksumTransform.js @@ -0,0 +1,182 @@ +const assert = require('assert'); +const { errors } = require('arsenal'); + +const { algorithms, ChecksumError } = require('../../../lib/api/apiUtils/integrity/validateChecksums'); +const ChecksumTransform = require('../../../lib/auth/streamingV4/ChecksumTransform'); +const { DummyRequestLogger } = require('../helpers'); + +const log = new DummyRequestLogger(); +const testData = Buffer.from('hello world'); +const algos = ['crc32', 'crc32c', 'sha1', 'sha256', 'crc64nvme']; + +// Helper: pipe chunks into a ChecksumTransform, collect output, resolve on finish +function runTransform(stream, chunks) { + return new Promise((resolve, reject) => { + const output = []; + stream.on('data', chunk => output.push(chunk)); + stream.on('end', () => resolve(Buffer.concat(output))); + stream.on('error', reject); + for (const chunk of chunks) { + stream.write(chunk); + } + stream.end(); + }); +} + +// Helper: pipe data through and wait for finish without collecting output +function drainTransform(stream, chunks) { + return new Promise((resolve, reject) => { + stream.resume(); + stream.on('finish', resolve); + stream.on('error', reject); + for (const chunk of chunks) { + stream.write(chunk); + } + stream.end(); + }); +} + +describe('ChecksumTransform basic behaviour', () => { + let expectedDigests; + + before(async () => { + expectedDigests = {}; + for (const algo of algos) { + expectedDigests[algo] = await Promise.resolve(algorithms[algo].digest(testData)); + } + }); + + for (const algo of algos) { + it(`[${algo}] passes data through unchanged`, async () => { + const stream = new ChecksumTransform(algo, undefined, false, log); + const output = await runTransform(stream, [testData]); + assert.deepStrictEqual(output, testData); + }); + + it(`[${algo}] computes digest correctly after stream ends`, async () => { + const stream = new ChecksumTransform(algo, undefined, false, log); + await drainTransform(stream, [testData]); + assert.strictEqual(stream.digest, expectedDigests[algo]); + }); + + it(`[${algo}] handles multi-chunk input: digest matches single-chunk equivalent`, async () => { + const half = Math.floor(testData.length / 2); + const stream = new ChecksumTransform(algo, undefined, false, log); + await drainTransform(stream, [testData.subarray(0, half), testData.subarray(half)]); + assert.strictEqual(stream.digest, expectedDigests[algo]); + }); + + it(`[${algo}] handles Buffer and string chunks equally`, async () => { + const streamBuf = new ChecksumTransform(algo, undefined, false, log); + const streamStr = new ChecksumTransform(algo, undefined, false, log); + await drainTransform(streamBuf, [testData]); + await drainTransform(streamStr, [testData.toString()]); + assert.strictEqual(streamBuf.digest, streamStr.digest); + }); + } + + it('emits error via stream error event if digestFromHash fails', done => { + const stream = new ChecksumTransform('crc32', undefined, false, log); + // Replace digestFromHash to return a rejected Promise + stream.algo = Object.assign({}, stream.algo, { + digestFromHash: () => Promise.reject(new Error('simulated digest failure')), + }); + stream.on('error', err => { + assert.deepStrictEqual(err, errors.InternalError); + done(); + }); + stream.write(testData); + stream.end(); + stream.resume(); + }); +}); + +describe('ChecksumTransform validateChecksum — non-trailer mode (isTrailer=false)', () => { + let crc32Digest; + + before(async () => { + crc32Digest = await Promise.resolve(algorithms.crc32.digest(testData)); + }); + + it('returns null when no expectedDigest and no trailer received', async () => { + const stream = new ChecksumTransform('crc32', undefined, false, log); + await drainTransform(stream, [testData]); + assert.strictEqual(stream.validateChecksum(), null); + }); + + it('returns null when expectedDigest matches computed digest', async () => { + const stream = new ChecksumTransform('crc32', crc32Digest, false, log); + await drainTransform(stream, [testData]); + assert.strictEqual(stream.validateChecksum(), null); + }); + + it('returns XAmzMismatch when expectedDigest does not match computed digest', async () => { + const stream = new ChecksumTransform('crc32', 'AAAAAA==', false, log); + await drainTransform(stream, [testData]); + const result = stream.validateChecksum(); + assert.strictEqual(result.error, ChecksumError.XAmzMismatch); + assert.strictEqual(result.details.algorithm, 'crc32'); + assert.strictEqual(result.details.calculated, crc32Digest); + assert.strictEqual(result.details.expected, 'AAAAAA=='); + }); + + it('returns TrailerUnexpected when setExpectedChecksum was called but isTrailer=false', async () => { + const stream = new ChecksumTransform('crc32', undefined, false, log); + stream.setExpectedChecksum('x-amz-checksum-crc32', crc32Digest); + await drainTransform(stream, [testData]); + const result = stream.validateChecksum(); + assert.strictEqual(result.error, ChecksumError.TrailerUnexpected); + }); +}); + +describe('ChecksumTransform validateChecksum — trailer mode (isTrailer=true)', () => { + let crc32Digest; + + before(async () => { + crc32Digest = await Promise.resolve(algorithms.crc32.digest(testData)); + }); + + it('returns TrailerMissing when setExpectedChecksum was never called', async () => { + const stream = new ChecksumTransform('crc32', undefined, true, log); + await drainTransform(stream, [testData]); + const result = stream.validateChecksum(); + assert.strictEqual(result.error, ChecksumError.TrailerMissing); + assert.strictEqual(result.details.expectedTrailer, 'x-amz-checksum-crc32'); + }); + + it('returns TrailerAlgoMismatch when trailer name does not match algo', async () => { + const stream = new ChecksumTransform('crc32', undefined, true, log); + stream.setExpectedChecksum('x-amz-checksum-sha256', 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA='); + await drainTransform(stream, [testData]); + const result = stream.validateChecksum(); + assert.strictEqual(result.error, ChecksumError.TrailerAlgoMismatch); + assert.strictEqual(result.details.algorithm, 'crc32'); + }); + + it('returns TrailerChecksumMalformed when trailer value is not a valid digest for the algo', async () => { + const stream = new ChecksumTransform('crc32', undefined, true, log); + stream.setExpectedChecksum('x-amz-checksum-crc32', 'not-valid!'); + await drainTransform(stream, [testData]); + const result = stream.validateChecksum(); + assert.strictEqual(result.error, ChecksumError.TrailerChecksumMalformed); + assert.strictEqual(result.details.algorithm, 'crc32'); + }); + + it('returns XAmzMismatch when trailer value is valid but does not match computed digest', async () => { + const stream = new ChecksumTransform('crc32', undefined, true, log); + stream.setExpectedChecksum('x-amz-checksum-crc32', 'AAAAAA=='); + await drainTransform(stream, [testData]); + const result = stream.validateChecksum(); + assert.strictEqual(result.error, ChecksumError.XAmzMismatch); + assert.strictEqual(result.details.algorithm, 'crc32'); + assert.strictEqual(result.details.calculated, crc32Digest); + assert.strictEqual(result.details.expected, 'AAAAAA=='); + }); + + it('returns null when trailer name and value match computed digest', async () => { + const stream = new ChecksumTransform('crc32', undefined, true, log); + stream.setExpectedChecksum('x-amz-checksum-crc32', crc32Digest); + await drainTransform(stream, [testData]); + assert.strictEqual(stream.validateChecksum(), null); + }); +}); From 176ee41f7be34c8f7fbf2ffa21d5adfbe689a8b4 Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Mon, 16 Mar 2026 16:01:49 +0100 Subject: [PATCH 6/8] CLDSRV-863: pipe requests to ChecksumTransform and validate checksum after upload --- lib/api/apiUtils/object/prepareStream.js | 144 ++++++--- lib/api/apiUtils/object/storeObject.js | 71 +++- .../unit/api/apiUtils/object/prepareStream.js | 200 ++++++++++++ tests/unit/api/apiUtils/object/storeObject.js | 303 ++++++++++++++++++ 4 files changed, 661 insertions(+), 57 deletions(-) create mode 100644 tests/unit/api/apiUtils/object/prepareStream.js create mode 100644 tests/unit/api/apiUtils/object/storeObject.js diff --git a/lib/api/apiUtils/object/prepareStream.js b/lib/api/apiUtils/object/prepareStream.js index be3f6c69ca..043a296049 100644 --- a/lib/api/apiUtils/object/prepareStream.js +++ b/lib/api/apiUtils/object/prepareStream.js @@ -1,50 +1,114 @@ const V4Transform = require('../../../auth/streamingV4/V4Transform'); const TrailingChecksumTransform = require('../../../auth/streamingV4/trailingChecksumTransform'); +const ChecksumTransform = require('../../../auth/streamingV4/ChecksumTransform'); +const { + getChecksumDataFromHeaders, + arsenalErrorFromChecksumError, +} = require('../../apiUtils/integrity/validateChecksums'); +const { errors, jsutil } = require('arsenal'); +const { unsupportedSignatureChecksums } = require('../../../../constants'); /** - * Prepares the stream if the chunks are sent in a v4 Auth request - * @param {object} stream - stream containing the data - * @param {object | null } streamingV4Params - if v4 auth, object containing - * accessKey, signatureFromRequest, region, scopeDate, timestamp, and - * credentialScope (to be used for streaming v4 auth if applicable) - * @param {RequestLogger} log - the current request logger - * @param {function} errCb - callback called if an error occurs - * @return {object|null} - V4Transform object if v4 Auth request, or - * the original stream, or null if the request has no V4 params but - * the type of request requires them + * Prepares the request stream for data storage by wrapping it in the + * appropriate transform pipeline based on the x-amz-content-sha256 header. + * Always returns a ChecksumTransform as the final stream. + * If no checksum was sent by the client a CRC64NVME ChecksumTransform is returned. + * + * @param {object} request - incoming HTTP request with headers and body stream + * @param {object|null} streamingV4Params - v4 streaming auth params (accessKey, + * signatureFromRequest, region, scopeDate, timestamp, credentialScope), or + * null/undefined for non-v4 requests + * @param {RequestLogger} log - request logger + * @param {function} errCb - error callback invoked if a stream error occurs + * @return {{ error: Arsenal.Error|null, stream: ChecksumTransform|null }} + * error is set and stream is null if the request headers are invalid; + * otherwise error is null and stream is the ChecksumTransform to read from */ -function prepareStream(stream, streamingV4Params, log, errCb) { - if (stream.headers['x-amz-content-sha256'] === - 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD') { - if (typeof streamingV4Params !== 'object') { - // this might happen if the user provided a valid V2 - // Authentication header, while the chunked upload method - // requires V4: in such case we don't get any V4 params - // and we should return an error to the client. - return null; - } - const v4Transform = new V4Transform(streamingV4Params, log, errCb); - stream.pipe(v4Transform); - v4Transform.headers = stream.headers; - return v4Transform; - } - return stream; -} +function prepareStream(request, streamingV4Params, log, errCb) { + const xAmzContentSHA256 = request.headers['x-amz-content-sha256']; -function stripTrailingChecksumStream(stream, log, errCb) { - // don't do anything if we are not in the correct integrity check mode - if (stream.headers['x-amz-content-sha256'] !== 'STREAMING-UNSIGNED-PAYLOAD-TRAILER') { - return stream; + const checksumAlgo = getChecksumDataFromHeaders(request.headers); + if (checksumAlgo.error) { + log.debug('prepareStream invalid checksum headers', checksumAlgo); + return { error: arsenalErrorFromChecksumError(checksumAlgo), stream: null }; } - const trailingChecksumTransform = new TrailingChecksumTransform(log); - trailingChecksumTransform.on('error', errCb); - stream.pipe(trailingChecksumTransform); - trailingChecksumTransform.headers = stream.headers; - return trailingChecksumTransform; + let stream = request; + switch (xAmzContentSHA256) { + case 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD': { + if (streamingV4Params === null || typeof streamingV4Params !== 'object') { + // this might happen if the user provided a valid V2 + // Authentication header, while the chunked upload method + // requires V4: in such case we don't get any V4 params + // and we should return an error to the client. + log.error('missing v4 streaming params for chunked upload', { + method: 'prepareStream', + streamingV4ParamsType: typeof streamingV4Params, + streamingV4Params, + }); + return { error: errors.InvalidArgument, stream: null }; + } + // Use a once-guard so that if both V4Transform and ChecksumTransform + // independently error, errCb is only called once. + const onStreamError = jsutil.once(errCb); + const v4Transform = new V4Transform(streamingV4Params, log, onStreamError); + request.pipe(v4Transform); + v4Transform.headers = request.headers; + stream = v4Transform; + + const checksumedStream = new ChecksumTransform( + checksumAlgo.algorithm, + checksumAlgo.expected, + checksumAlgo.isTrailer, + log, + ); + checksumedStream.on('error', onStreamError); + stream.pipe(checksumedStream); + return { error: null, stream: checksumedStream }; + } + case 'STREAMING-UNSIGNED-PAYLOAD-TRAILER': { + // Use a once-guard so that auto-destroying both piped streams + // when one errors does not result in errCb being called twice. + const onStreamError = jsutil.once(errCb); + const trailingChecksumTransform = new TrailingChecksumTransform(log); + trailingChecksumTransform.on('error', onStreamError); + request.pipe(trailingChecksumTransform); + trailingChecksumTransform.headers = request.headers; + stream = trailingChecksumTransform; + + const checksumedStream = new ChecksumTransform( + checksumAlgo.algorithm, + checksumAlgo.expected, + checksumAlgo.isTrailer, + log, + ); + checksumedStream.on('error', onStreamError); + trailingChecksumTransform.on('trailer', (name, value) => { + checksumedStream.setExpectedChecksum(name, value); + }); + stream.pipe(checksumedStream); + return { error: null, stream: checksumedStream }; + } + case 'UNSIGNED-PAYLOAD': // Fallthrough + default: { + if (unsupportedSignatureChecksums.has(xAmzContentSHA256)) { + return { + error: errors.BadRequest.customizeDescription(`${xAmzContentSHA256} is not supported`), + stream: null, + }; + } + + const checksumedStream = new ChecksumTransform( + checksumAlgo.algorithm, + checksumAlgo.expected, + checksumAlgo.isTrailer, + log, + ); + checksumedStream.on('error', errCb); + stream.pipe(checksumedStream); + return { error: null, stream: checksumedStream }; + } + } } -module.exports = { - prepareStream, - stripTrailingChecksumStream, -}; +module.exports = { prepareStream }; diff --git a/lib/api/apiUtils/object/storeObject.js b/lib/api/apiUtils/object/storeObject.js index 8beea03ecb..68a5312c4f 100644 --- a/lib/api/apiUtils/object/storeObject.js +++ b/lib/api/apiUtils/object/storeObject.js @@ -1,7 +1,8 @@ const { errors, jsutil } = require('arsenal'); const { data } = require('../../../data/wrapper'); -const { prepareStream, stripTrailingChecksumStream } = require('./prepareStream'); +const { prepareStream } = require('./prepareStream'); +const { arsenalErrorFromChecksumError } = require('../../apiUtils/integrity/validateChecksums'); /** * Check that `hashedStream.completedHash` matches header `stream.contentMD5` @@ -58,31 +59,67 @@ function checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo, log, cb) { function dataStore(objectContext, cipherBundle, stream, size, streamingV4Params, backendInfo, log, cb) { const cbOnce = jsutil.once(cb); - const dataStreamTmp = prepareStream(stream, streamingV4Params, log, cbOnce); - if (!dataStreamTmp) { - return process.nextTick(() => cb(errors.InvalidArgument)); + + // errCb is delegated through a mutable reference so it can be upgraded to + // include batchDelete once data.put has actually stored data. + let onStreamError = cbOnce; + const errCb = err => onStreamError(err); + + const checksumedStream = prepareStream(stream, streamingV4Params, log, errCb); + if (checksumedStream.error) { + log.debug('dataStore failed to prepare stream', checksumedStream); + return process.nextTick(() => cbOnce(checksumedStream.error)); } - const dataStream = stripTrailingChecksumStream(dataStreamTmp, log, cbOnce); return data.put( - cipherBundle, dataStream, size, objectContext, backendInfo, log, + cipherBundle, checksumedStream.stream, size, objectContext, backendInfo, log, (err, dataRetrievalInfo, hashedStream) => { if (err) { - log.error('error in datastore', { - error: err, - }); + log.error('error in datastore', { error: err }); return cbOnce(err); } if (!dataRetrievalInfo) { - log.fatal('data put returned neither an error nor a key', { - method: 'storeObject::dataStore', - }); + log.fatal('data put returned neither an error nor a key', { method: 'storeObject::dataStore' }); return cbOnce(errors.InternalError); } - log.trace('dataStore: backend stored key', { - dataRetrievalInfo, - }); - return checkHashMatchMD5(stream, hashedStream, - dataRetrievalInfo, log, cbOnce); + log.trace('dataStore: backend stored key', { dataRetrievalInfo }); + + // Data is now stored. Upgrade the error handler so that any stream + // error from this point on cleans up the stored data first. + onStreamError = streamErr => { + log.error('checksum stream error after data.put', { error: streamErr }); + return data.batchDelete([dataRetrievalInfo], null, null, log, deleteErr => { + if (deleteErr) { + log.error('dataStore failed to delete old data', { error: deleteErr }); + } + cbOnce(streamErr); + }); + }; + + const doValidate = () => { + const checksumErr = checksumedStream.stream.validateChecksum(); + if (checksumErr) { + log.debug('failed checksum validation stream', checksumErr); + return data.batchDelete([dataRetrievalInfo], null, null, log, deleteErr => { + if (deleteErr) { + // Failure of batch delete is only logged. + log.error('dataStore failed to delete old data', { error: deleteErr }); + } + return cbOnce(arsenalErrorFromChecksumError(checksumErr)); + }); + } + return checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo, log, cbOnce); + }; + + // ChecksumTransform._flush computes the digest asynchronously for + // some algorithms (e.g. crc64nvme). writableFinished is true once + // _flush has called its callback, guaranteeing this.digest is set. + // Stream piping ordering means this is virtually always true here, + // but we wait for 'finish' explicitly to be safe. + if (checksumedStream.stream.writableFinished) { + return doValidate(); + } + checksumedStream.stream.once('finish', doValidate); + return undefined; }); } diff --git a/tests/unit/api/apiUtils/object/prepareStream.js b/tests/unit/api/apiUtils/object/prepareStream.js new file mode 100644 index 0000000000..3ae5768f02 --- /dev/null +++ b/tests/unit/api/apiUtils/object/prepareStream.js @@ -0,0 +1,200 @@ +const assert = require('assert'); +const { errors } = require('arsenal'); + +const { prepareStream } = require('../../../../../lib/api/apiUtils/object/prepareStream'); +const ChecksumTransform = require('../../../../../lib/auth/streamingV4/ChecksumTransform'); +const { DummyRequestLogger } = require('../../../helpers'); +const DummyRequest = require('../../../DummyRequest'); + +const log = new DummyRequestLogger(); + +function makeRequest(headers, body) { + return new DummyRequest({ headers }, body != null ? Buffer.from(body) : undefined); +} + +const mockV4Params = { + accessKey: 'AKIAIOSFODNN7EXAMPLE', + signatureFromRequest: 'abc123', + region: 'us-east-1', + scopeDate: '20210101', + timestamp: '20210101T000000Z', + credentialScope: '20210101/us-east-1/s3/aws4_request', +}; + +describe('prepareStream return value shape', () => { + it('returns { error: null, stream: ChecksumTransform } for UNSIGNED-PAYLOAD', () => { + const request = makeRequest({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + const result = prepareStream(request, null, log, () => {}); + assert.strictEqual(result.error, null); + assert(result.stream instanceof ChecksumTransform); + }); + + it('returns { error: InvalidRequest, stream: null } on invalid checksum headers', () => { + const request = makeRequest({ + 'x-amz-checksum-crc32': 'AAAAAA==', + 'x-amz-checksum-sha256': 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=', + }); + const result = prepareStream(request, null, log, () => {}); + assert(result.error.is.InvalidRequest); + assert.strictEqual(result.stream, null); + }); + + it('returns { error: BadRequest, stream: null } for unsupported x-amz-content-sha256', () => { + const request = makeRequest({ + 'x-amz-content-sha256': 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER', + }); + const result = prepareStream(request, null, log, () => {}); + assert(result.error.is.BadRequest); + assert.strictEqual(result.stream, null); + }); +}); + +describe('prepareStream STREAMING-AWS4-HMAC-SHA256-PAYLOAD', () => { + it('with valid streamingV4Params: returns ChecksumTransform as final stream', () => { + const request = makeRequest({ 'x-amz-content-sha256': 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD' }); + const result = prepareStream(request, mockV4Params, log, () => {}); + assert.strictEqual(result.error, null); + assert(result.stream instanceof ChecksumTransform); + }); + + it('with valid streamingV4Params: returned stream uses crc64nvme by default', () => { + const request = makeRequest({ 'x-amz-content-sha256': 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD' }); + const result = prepareStream(request, mockV4Params, log, () => {}); + assert.strictEqual(result.stream.algoName, 'crc64nvme'); + }); + + it('with x-amz-checksum-crc32c header: ChecksumTransform uses crc32c algorithm', () => { + const request = makeRequest({ + 'x-amz-content-sha256': 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD', + 'x-amz-checksum-crc32c': 'AAAAAA==', + }); + const result = prepareStream(request, mockV4Params, log, () => {}); + assert.strictEqual(result.error, null); + assert.strictEqual(result.stream.algoName, 'crc32c'); + assert.strictEqual(result.stream.expectedDigest, 'AAAAAA=='); + }); + + it('with null streamingV4Params: returns InvalidArgument error', () => { + const request = makeRequest({ 'x-amz-content-sha256': 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD' }); + const result = prepareStream(request, null, log, () => {}); + assert.deepStrictEqual(result.error, errors.InvalidArgument); + assert.strictEqual(result.stream, null); + }); + + it('with non-object streamingV4Params (string): returns InvalidArgument error', () => { + const request = makeRequest({ 'x-amz-content-sha256': 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD' }); + const result = prepareStream(request, 'not-an-object', log, () => {}); + assert.deepStrictEqual(result.error, errors.InvalidArgument); + assert.strictEqual(result.stream, null); + }); +}); + +describe('prepareStream STREAMING-UNSIGNED-PAYLOAD-TRAILER', () => { + it('returns ChecksumTransform with isTrailer=true', () => { + const request = makeRequest({ + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-crc32', + }); + const result = prepareStream(request, null, log, () => {}); + assert.strictEqual(result.error, null); + assert(result.stream instanceof ChecksumTransform); + assert.strictEqual(result.stream.isTrailer, true); + assert.strictEqual(result.stream.algoName, 'crc32'); + }); + + it('trailer event from TrailingChecksumTransform calls setExpectedChecksum on ChecksumTransform', done => { + const body = '0\r\nx-amz-checksum-crc32:AAAAAA==\r\n'; + const request = makeRequest({ + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-crc32', + }, body); + const result = prepareStream(request, null, log, done); + result.stream.resume(); + result.stream.on('finish', () => { + assert.strictEqual(result.stream.trailerChecksumName, 'x-amz-checksum-crc32'); + assert.strictEqual(result.stream.trailerChecksumValue, 'AAAAAA=='); + done(); + }); + result.stream.on('error', done); + }); + + it('errCb is called when TrailingChecksumTransform emits an error', done => { + // malformed chunked data triggers an error in TrailingChecksumTransform + const request = makeRequest({ + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-crc32', + }, 'zz\r\n'); // invalid hex chunk size + prepareStream(request, null, log, err => { + assert(err.is.InvalidArgument); + done(); + }); + }); + + it('errCb is called when ChecksumTransform emits an error', done => { + const request = makeRequest({ + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-crc32', + }); + const result = prepareStream(request, null, log, err => { + assert.deepStrictEqual(err, errors.InternalError); + done(); + }); + result.stream.emit('error', errors.InternalError); + }); +}); + +describe('prepareStream UNSIGNED-PAYLOAD', () => { + it('returns ChecksumTransform as final stream', () => { + const request = makeRequest({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + const result = prepareStream(request, null, log, () => {}); + assert.strictEqual(result.error, null); + assert(result.stream instanceof ChecksumTransform); + }); + + it('ChecksumTransform receives the algorithm and expected digest from headers', () => { + const request = makeRequest({ + 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', + 'x-amz-checksum-crc32': 'AAAAAA==', + }); + const result = prepareStream(request, null, log, () => {}); + assert.strictEqual(result.stream.algoName, 'crc32'); + assert.strictEqual(result.stream.expectedDigest, 'AAAAAA=='); + }); + + it('errCb is called when ChecksumTransform emits an error', done => { + const request = makeRequest({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + const result = prepareStream(request, null, log, err => { + assert.deepStrictEqual(err, errors.InternalError); + done(); + }); + result.stream.emit('error', errors.InternalError); + }); +}); + +describe('prepareStream default (no x-amz-content-sha256)', () => { + it('no x-amz-content-sha256 header: returns ChecksumTransform with crc64nvme algorithm', () => { + const request = makeRequest({}); + const result = prepareStream(request, null, log, () => {}); + assert.strictEqual(result.error, null); + assert(result.stream instanceof ChecksumTransform); + assert.strictEqual(result.stream.algoName, 'crc64nvme'); + }); + + it('unsupported x-amz-content-sha256 (STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER): returns BadRequest', () => { + const request = makeRequest({ + 'x-amz-content-sha256': 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER', + }); + const result = prepareStream(request, null, log, () => {}); + assert(result.error.is.BadRequest); + assert.strictEqual(result.stream, null); + }); + + it('errCb is called when ChecksumTransform emits an error', done => { + const request = makeRequest({}); + const result = prepareStream(request, null, log, err => { + assert.deepStrictEqual(err, errors.InternalError); + done(); + }); + result.stream.emit('error', errors.InternalError); + }); +}); diff --git a/tests/unit/api/apiUtils/object/storeObject.js b/tests/unit/api/apiUtils/object/storeObject.js new file mode 100644 index 0000000000..31fd8c6847 --- /dev/null +++ b/tests/unit/api/apiUtils/object/storeObject.js @@ -0,0 +1,303 @@ +const assert = require('assert'); +const sinon = require('sinon'); +const { errors } = require('arsenal'); + +const { dataStore } = require('../../../../../lib/api/apiUtils/object/storeObject'); +const dataWrapper = require('../../../../../lib/data/wrapper'); +const { DummyRequestLogger } = require('../../../helpers'); +const DummyRequest = require('../../../DummyRequest'); + +const log = new DummyRequestLogger(); + +const fakeDataRetrievalInfo = { key: 'test-key', dataStoreName: 'mem' }; + +function makeStream(headers = {}, body = '') { + return new DummyRequest({ headers }, body ? Buffer.from(body) : undefined); +} + +describe('dataStore', () => { + let putStub, batchDeleteStub; + + beforeEach(() => { + putStub = sinon.stub(dataWrapper.data, 'put'); + batchDeleteStub = sinon.stub(dataWrapper.data, 'batchDelete'); + }); + + afterEach(() => { + sinon.restore(); + }); + + // Stub data.put to drain the readable side and succeed once the stream finishes. + function putSucceeds(completedHash = null) { + putStub.callsFake((cipher, stream, size, ctx, backend, log2, cb) => { + stream.resume(); + stream.once('finish', () => cb(null, fakeDataRetrievalInfo, { completedHash })); + }); + } + + // Stub data.batchDelete to succeed. + function batchDeleteSucceeds() { + batchDeleteStub.callsFake((keys, a, b, log2, cb) => cb(null)); + } + + describe('normal behaviour', () => { + it('calls data.put with the stream returned by prepareStream', done => { + putSucceeds(); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + dataStore({}, null, request, 0, null, {}, log, err => { + assert.strictEqual(err, null); + assert(putStub.calledOnce); + done(); + }); + }); + + it('calls cb with (null, dataRetrievalInfo, completedHash) on success', done => { + putSucceeds('abc123'); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + dataStore({}, null, request, 0, null, {}, log, (err, dataInfo, completedHash) => { + assert.strictEqual(err, null); + assert.strictEqual(dataInfo, fakeDataRetrievalInfo); + assert.strictEqual(completedHash, 'abc123'); + done(); + }); + }); + + it('calls cb with the error from data.put when data.put fails', done => { + putStub.callsFake((cipher, stream, size, ctx, backend, log2, cb) => { + stream.resume(); + cb(errors.InternalError); + }); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + dataStore({}, null, request, 0, null, {}, log, err => { + assert.deepStrictEqual(err, errors.InternalError); + done(); + }); + }); + + it('does not delete stored data when data.put fails', done => { + putStub.callsFake((cipher, stream, size, ctx, backend, log2, cb) => { + stream.resume(); + cb(errors.InternalError); + }); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + dataStore({}, null, request, 0, null, {}, log, () => { + assert(batchDeleteStub.notCalled); + done(); + }); + }); + + it('calls cb with InternalError when data.put returns neither error nor dataRetrievalInfo', done => { + putStub.callsFake((cipher, stream, size, ctx, backend, log2, cb) => { + stream.resume(); + cb(null, null, null); + }); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + dataStore({}, null, request, 0, null, {}, log, err => { + assert.deepStrictEqual(err, errors.InternalError); + done(); + }); + }); + + it('calls cb with BadDigest and deletes stored data when content-md5 does not match', done => { + batchDeleteSucceeds(); + putSucceeds('correct-md5'); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + request.contentMD5 = 'wrong-md5'; + dataStore({}, null, request, 0, null, {}, log, err => { + assert.deepStrictEqual(err, errors.BadDigest); + assert(batchDeleteStub.calledOnce); + done(); + }); + }); + + it('calls cb with (null, ...) and does not delete stored data when content-md5 matches', done => { + putSucceeds('abc123'); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + request.contentMD5 = 'abc123'; + dataStore({}, null, request, 0, null, {}, log, err => { + assert.strictEqual(err, null); + assert(batchDeleteStub.notCalled); + done(); + }); + }); + + it('cb is called exactly once on success', done => { + putSucceeds(); + let cbCount = 0; + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + dataStore({}, null, request, 0, null, {}, log, () => { + cbCount++; + setImmediate(() => { + assert.strictEqual(cbCount, 1); + done(); + }); + }); + }); + }); + + describe('checksum behaviour', () => { + it('calls cb with error from prepareStream when stream headers are invalid', done => { + const request = makeStream({ + 'x-amz-checksum-crc32': 'AAAAAA==', + 'x-amz-checksum-sha256': 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=', + }); + dataStore({}, null, request, 0, null, {}, log, err => { + assert(err.is.InvalidRequest); + done(); + }); + }); + + it('does not call data.put when prepareStream returns an error', done => { + const request = makeStream({ + 'x-amz-checksum-crc32': 'AAAAAA==', + 'x-amz-checksum-sha256': 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=', + }); + dataStore({}, null, request, 0, null, {}, log, () => { + assert(putStub.notCalled); + done(); + }); + }); + + it('calls cb with BadDigest and deletes stored data when checksum validation fails', done => { + batchDeleteSucceeds(); + putSucceeds(); + // CRC32 of 'hello world' is not 0x00000000 (AAAAAA==) + const request = makeStream({ + 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', + 'x-amz-checksum-crc32': 'AAAAAA==', + }, 'hello world'); + dataStore({}, null, request, 0, null, {}, log, err => { + assert(err.is.BadDigest); + assert(batchDeleteStub.calledOnce); + done(); + }); + }); + + it('does not delete stored data when checksum validation passes', done => { + putSucceeds(); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + dataStore({}, null, request, 0, null, {}, log, err => { + assert.strictEqual(err, null); + assert(batchDeleteStub.notCalled); + done(); + }); + }); + + it('when checksumedStream is not yet writableFinished after data.put, waits for finish before validating', + done => { + let capturedStream; + putStub.callsFake((cipher, stream, size, ctx, backend, log2, cb) => { + capturedStream = stream; + stream.resume(); + // Call cb synchronously — _flush uses Promise.resolve().then() so + // writableFinished is false here, exercising the finish-wait path. + cb(null, fakeDataRetrievalInfo, { completedHash: null }); + }); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + dataStore({}, null, request, 0, null, {}, log, err => { + assert.strictEqual(err, null); + assert(capturedStream.writableFinished); + done(); + }); + }); + + it('when checksumedStream emits error after data.put, deletes stored data and calls cb with the error', + done => { + batchDeleteSucceeds(); + let capturedStream; + putStub.callsFake((cipher, stream, size, ctx, backend, log2, cb) => { + capturedStream = stream; + // Do not resume — keeps writableFinished false, so onError listener is registered. + cb(null, fakeDataRetrievalInfo, { completedHash: null }); + }); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + dataStore({}, null, request, 0, null, {}, log, err => { + assert.deepStrictEqual(err, errors.InternalError); + assert(batchDeleteStub.calledOnce); + done(); + }); + // process.nextTick fires before Promise microtasks, so the error arrives + // before _flush resolves, ensuring onError fires rather than onFinish. + process.nextTick(() => capturedStream.emit('error', errors.InternalError)); + }); + + it('cb is called exactly once when finish fires (no double callback)', done => { + let cbCount = 0; + putStub.callsFake((cipher, stream, size, ctx, backend, log2, cb) => { + stream.resume(); + // Synchronous cb → writableFinished is false → finish-wait path. + cb(null, fakeDataRetrievalInfo, { completedHash: null }); + }); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + dataStore({}, null, request, 0, null, {}, log, () => { + cbCount++; + setImmediate(() => { + assert.strictEqual(cbCount, 1); + done(); + }); + }); + }); + + it('cb is called exactly once when stream errors after data.put (no double callback)', done => { + batchDeleteSucceeds(); + let cbCount = 0; + let capturedStream; + putStub.callsFake((cipher, stream, size, ctx, backend, log2, cb) => { + capturedStream = stream; + cb(null, fakeDataRetrievalInfo, { completedHash: null }); + }); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + dataStore({}, null, request, 0, null, {}, log, () => { + cbCount++; + setImmediate(() => { + assert.strictEqual(cbCount, 1); + done(); + }); + }); + process.nextTick(() => capturedStream.emit('error', errors.InternalError)); + }); + }); + + describe('batchDelete failure paths', () => { + it('validateChecksum fails and batchDelete fails: cb called with checksum error, not delete error', done => { + batchDeleteStub.callsFake((keys, a, b, log2, cb) => cb(errors.InternalError)); + putSucceeds(); + const request = makeStream({ + 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', + 'x-amz-checksum-crc32': 'AAAAAA==', + }, 'hello world'); + dataStore({}, null, request, 0, null, {}, log, err => { + assert(err.is.BadDigest); + done(); + }); + }); + + it('content-md5 mismatch and batchDelete fails: cb called with BadDigest, not delete error', done => { + batchDeleteStub.callsFake((keys, a, b, log2, cb) => cb(errors.InternalError)); + putSucceeds('correct-md5'); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + request.contentMD5 = 'wrong-md5'; + dataStore({}, null, request, 0, null, {}, log, err => { + assert.deepStrictEqual(err, errors.BadDigest); + done(); + }); + }); + + it('checksumedStream emits error after data.put and batchDelete fails: ' + + 'cb called with stream error, not delete error', + done => { + batchDeleteStub.callsFake((keys, a, b, log2, cb) => cb(errors.BadRequest)); + let capturedStream; + putStub.callsFake((cipher, stream, size, ctx, backend, log2, cb) => { + capturedStream = stream; + cb(null, fakeDataRetrievalInfo, { completedHash: null }); + }); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + dataStore({}, null, request, 0, null, {}, log, err => { + assert.deepStrictEqual(err, errors.InternalError); + done(); + }); + process.nextTick(() => capturedStream.emit('error', errors.InternalError)); + }); + }); +}); From 888ea4333d5b1ed6af940cdc5ab6bf945eebb595 Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Mon, 16 Mar 2026 20:49:27 +0100 Subject: [PATCH 7/8] CLDSRV-863: PutObject/UploadPart checksum service tests --- .../test/checksumPutObjectUploadPart.js | 610 ++++++++++++++++++ 1 file changed, 610 insertions(+) create mode 100644 tests/functional/raw-node/test/checksumPutObjectUploadPart.js diff --git a/tests/functional/raw-node/test/checksumPutObjectUploadPart.js b/tests/functional/raw-node/test/checksumPutObjectUploadPart.js new file mode 100644 index 0000000000..132d8b1a60 --- /dev/null +++ b/tests/functional/raw-node/test/checksumPutObjectUploadPart.js @@ -0,0 +1,610 @@ +const assert = require('assert'); +const crypto = require('crypto'); +const async = require('async'); + +const { makeS3Request } = require('../utils/makeRequest'); +const HttpRequestAuthV4 = require('../utils/HttpRequestAuthV4'); + +const bucket = 'checksumrejectionbucket'; +const objectKey = 'key'; +const objData = Buffer.alloc(1, 'a'); +const objDataSha256Hex = crypto.createHash('sha256').update(objData).digest('hex'); + +const authCredentials = { + accessKey: 'accessKey1', + secretKey: 'verySecretKey1', +}; + +const itSkipIfAWS = process.env.AWS_ON_AIR ? it.skip : it; + +const algos = [ + { name: 'crc32', wrongDigest: 'AAAAAA==' }, + { name: 'crc32c', wrongDigest: 'AAAAAA==' }, + { name: 'crc64nvme', wrongDigest: 'AAAAAAAAAAA=' }, + { name: 'sha1', wrongDigest: 'AAAAAAAAAAAAAAAAAAAAAAAAAAA=' }, + { name: 'sha256', wrongDigest: 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=' }, +]; + +// Build the raw chunked body for STREAMING-UNSIGNED-PAYLOAD-TRAILER +function buildTrailerBody(algoName, wrongDigest) { + const chunkSize = objData.length.toString(16); + return `${chunkSize}\r\n${objData.toString()}\r\n0\r\nx-amz-checksum-${algoName}:${wrongDigest}\r\n`; +} + +function doPutRequest(url, headers, body, callback) { + const req = new HttpRequestAuthV4( + url, + Object.assign({ method: 'PUT', headers }, authCredentials), + res => { + let data = ''; + res.on('data', chunk => { data += chunk; }); + res.on('end', () => callback(null, { statusCode: res.statusCode, body: data })); + } + ); + req.on('error', callback); + req.write(body); + req.end(); +} + +const protocols = [ + { + name: 'UNSIGNED-PAYLOAD', + buildHeaders: algo => ({ + 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', + 'content-length': objData.length, + [`x-amz-checksum-${algo.name}`]: algo.wrongDigest, + }), + buildBody: () => objData, + }, + { + name: 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD', + buildHeaders: algo => ({ + // No x-amz-content-sha256: HttpRequestAuthV4 defaults to + // STREAMING-AWS4-HMAC-SHA256-PAYLOAD and handles chunk signing. + 'content-length': objData.length, + [`x-amz-checksum-${algo.name}`]: algo.wrongDigest, + }), + buildBody: () => objData, + }, + { + name: 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + buildHeaders: algo => { + const body = buildTrailerBody(algo.name, algo.wrongDigest); + return { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': `x-amz-checksum-${algo.name}`, + 'x-amz-decoded-content-length': objData.length, + 'content-length': body.length, + }; + }, + buildBody: algo => buildTrailerBody(algo.name, algo.wrongDigest), + }, + { + name: 'valid x-amz-content-sha256', + buildHeaders: algo => ({ + 'x-amz-content-sha256': objDataSha256Hex, + 'content-length': objData.length, + [`x-amz-checksum-${algo.name}`]: algo.wrongDigest, + }), + buildBody: () => objData, + }, +]; + +function assertBadDigest(err, res, done) { + assert.ifError(err); + assert.strictEqual(res.statusCode, 400, `expected 400, got ${res.statusCode}: ${res.body}`); + assert(res.body.includes('BadDigest'), `missing BadDigest in "${res.body}"`); + done(); +} + +// Constants for protocol scenario tests + +const trailerContent = Buffer.from('trailer content'); // 15 bytes, hex 'f' +const trailerContentSha256 = '4x74k2oA6j6knXzpDwogNkS6E3MM49tPpJMjfD+ES68='; +// md5("trailer content") in base64 — used in testS3PutTrailerWithContentMD5 +const trailerContentMd5B64 = crypto.createHash('md5').update(trailerContent).digest('base64'); + +const testContent2 = Buffer.from('test content'); +const testContent2Sha256Hex = crypto.createHash('sha256').update(testContent2).digest('hex'); +const testContent2Sha256B64 = crypto.createHash('sha256').update(testContent2).digest('base64'); + +// Build a STREAMING-UNSIGNED-PAYLOAD-TRAILER body with "trailer content" as the +// data and a sha256 trailer. Uses the '\n\r\n\r\n\r\n' ending that AWS SDK sends +// (TrailingChecksumTransform strips the trailing \n from the parsed trailer line). +function buildOkTrailerBody() { + const hexLen = trailerContent.length.toString(16); // 'f' + return `${hexLen}\r\ntrailer content\r\n0\r\nx-amz-checksum-sha256:${trailerContentSha256}\n\r\n\r\n\r\n`; +} + +// Assert that the response has the given HTTP status code and (optionally) +// that the body contains the expected error code string. +// Returns a (err, res, done) callback suitable for use with doPutRequest. +function assertStatus(expectedStatus, expectedCode, expectedMessage) { + return (err, res, done) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, expectedStatus, + `expected ${expectedStatus}, got ${res.statusCode}: ${res.body}`); + if (expectedCode) { + assert(res.body.includes(expectedCode), + `expected "${expectedCode}" in body: "${res.body}"`); + } + if (expectedMessage) { + assert(res.body.includes(expectedMessage), + `expected "${expectedMessage}" in body: "${res.body}"`); + } + done(); + }; +} + +const msgMalformedTrailer = 'The request contained trailing data that was not well-formed' + + ' or did not conform to our published schema.'; +const msgSdkMissingTrailer = 'x-amz-sdk-checksum-algorithm specified, but no corresponding' + + ' x-amz-checksum-* or x-amz-trailer headers were found.'; + +// Create the 24 common protocol-scenario tests for a given URL factory. +// urlFn() is called lazily at test runtime so that uploadId is available. +function makeScenarioTests(urlFn) { + itSkipIfAWS( + 'testS3PutNoChecksum: signed sha256 in x-amz-content-sha256, no x-amz-checksum header -> 200 OK', + done => { + doPutRequest(urlFn(), { + 'x-amz-content-sha256': testContent2Sha256Hex, + 'content-length': testContent2.length, + }, testContent2, (err, res) => assertStatus(200)(err, res, done)); + }); + + itSkipIfAWS( + 'testS3PutChecksum: correct sha256 checksum with x-amz-sdk-checksum-algorithm -> 200 OK', + done => { + doPutRequest(urlFn(), { + 'x-amz-content-sha256': testContent2Sha256Hex, + 'x-amz-sdk-checksum-algorithm': 'SHA256', + 'x-amz-checksum-sha256': testContent2Sha256B64, + 'content-length': testContent2.length, + }, testContent2, (err, res) => assertStatus(200)(err, res, done)); + }); + + itSkipIfAWS( + 'testS3PutChecksumKo: wrong sha256 checksum with x-amz-sdk-checksum-algorithm -> 400 BadDigest', + done => { + doPutRequest(urlFn(), { + 'x-amz-content-sha256': testContent2Sha256Hex, + 'x-amz-sdk-checksum-algorithm': 'SHA256', + 'x-amz-checksum-sha256': 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=', + 'content-length': testContent2.length, + }, testContent2, (err, res) => assertStatus(400, 'BadDigest', + 'The SHA256 you specified did not match the calculated checksum.')(err, res, done)); + }); + + itSkipIfAWS( + 'testS3PutChecksumUnsignedPayload: UNSIGNED-PAYLOAD with correct sha256 checksum -> 200 OK', + done => { + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', + 'x-amz-sdk-checksum-algorithm': 'SHA256', + 'x-amz-checksum-sha256': testContent2Sha256B64, + 'content-length': testContent2.length, + }, testContent2, (err, res) => assertStatus(200)(err, res, done)); + }); + + itSkipIfAWS( + 'testS3PutTrailerNoBody: TRAILER with empty body -> 400 IncompleteBody', + done => { + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha256', + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': 0, + }, Buffer.alloc(0), (err, res) => assertStatus(400, 'IncompleteBody', + 'The request body terminated unexpectedly')(err, res, done)); + }); + + itSkipIfAWS( + 'testS3PutTrailerOk: TRAILER with correct sha256 checksum -> 200 OK', + done => { + const body = buildOkTrailerBody(); + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha256', + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(200)(err, res, done)); + }); + + itSkipIfAWS( + 'testS3PutTrailerBadDecodedLen: wrong x-amz-decoded-content-length (7 but actual is 32) -> 500 InternalError', + done => { + // Two chunks of 16 bytes each with a valid crc64nvme trailer. + const body = + '10\r\n0123456789abcdef\r\n' + + '10\r\n0123456789abcdef\r\n' + + '0\r\nx-amz-checksum-crc64nvme:skQv82y5rgE=\r\n\r\n\r\n'; + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-crc64nvme', + 'x-amz-decoded-content-length': 7, // wrong: actual content is 32 bytes + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(500, 'InternalError', + 'We encountered an internal error. Please try again.')(err, res, done)); + }); + + itSkipIfAWS( + 'testS3PutTrailerAlgoMismatch: x-amz-trailer says sha1 but body trailer has sha256 ' + + ' -> 400 MalformedTrailerError', + done => { + // Header announces sha1 but the actual trailer line carries sha256. + const body = + `f\r\ntrailer content\r\n0\r\nx-amz-checksum-sha256:${trailerContentSha256}\n\r\n\r\n\r\n`; + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha1', + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(400, 'MalformedTrailerError', + msgMalformedTrailer)(err, res, done)); + }); + + itSkipIfAWS( + 'testS3PutTrailerBadDigest: TRAILER with wrong sha256 checksum -> 400 BadDigest', + done => { + const wrongSha256 = 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA='; + const body = + `f\r\ntrailer content\r\n0\r\nx-amz-checksum-sha256:${wrongSha256}\n\r\n\r\n\r\n`; + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha256', + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(400, 'BadDigest', + 'The SHA256 you specified did not match the calculated checksum.')(err, res, done)); + }); + + itSkipIfAWS( + 'testS3PutTrailerAndChecksum: x-amz-trailer + x-amz-checksum-crc32 header -> 400 InvalidRequest', + done => { + const body = buildOkTrailerBody(); + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha256', + 'x-amz-checksum-crc32': 'H+Yzmw==', // crc32("trailer content") + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(400, 'InvalidRequest', + 'Expecting a single x-amz-checksum- header')(err, res, done)); + }); + + itSkipIfAWS( + 'testS3PutTrailerNoTrailerInHeader: no x-amz-trailer header but body has trailer -> 400 MalformedTrailerError', + done => { + const body = buildOkTrailerBody(); + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + // no x-amz-trailer header + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(400, 'MalformedTrailerError', + msgMalformedTrailer)(err, res, done)); + }); + + itSkipIfAWS( + 'testS3PutTrailerContentLength: TRAILER with explicit Content-Length -> 200 OK', + done => { + const body = buildOkTrailerBody(); + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha256', + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(200)(err, res, done)); + }); + + itSkipIfAWS( + 'testS3PutTrailerSDK: TRAILER with matching x-amz-sdk-checksum-algorithm:SHA256 -> 200 OK', + done => { + const body = buildOkTrailerBody(); + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha256', + 'x-amz-sdk-checksum-algorithm': 'SHA256', + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(200)(err, res, done)); + }); + + itSkipIfAWS( + 'testS3PutTrailerSDKMismatch: x-amz-sdk-checksum-algorithm:SHA1 but x-amz-trailer is sha256 ' + + '-> 400 InvalidRequest', + done => { + const body = buildOkTrailerBody(); + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha256', + 'x-amz-sdk-checksum-algorithm': 'SHA1', // mismatch + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(400, 'InvalidRequest', + 'Value for x-amz-sdk-checksum-algorithm header is invalid.')(err, res, done)); + }); + + itSkipIfAWS( + 'testS3PutTrailerUnknownTrailerAlgo: x-amz-trailer:x-amz-checksum-sha3 (unknown) -> 400 InvalidRequest', + done => { + const body = buildOkTrailerBody(); + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha3', + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(400, 'InvalidRequest', + 'The value specified in the x-amz-trailer header is not supported')(err, res, done)); + }); + + itSkipIfAWS( + 'testS3PutTrailerInvalidTrailerHeader: x-amz-trailer with non-checksum value -> 400 InvalidRequest', + done => { + const body = buildOkTrailerBody(); + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'AAAAAAAAAAAAAAAAAAAAA', + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(400, 'InvalidRequest', + 'The value specified in the x-amz-trailer header is not supported')(err, res, done)); + }); + + itSkipIfAWS( + 'testS3PutTrailerInvalidTrailerBody: trailer body has invalid base64 checksum value -> 400 InvalidRequest', + done => { + const body = 'f\r\ntrailer content\r\n0\r\nx-amz-checksum-sha256:BAD\n\r\n\r\n\r\n'; + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha256', + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(400, 'InvalidRequest', + 'Value for x-amz-checksum-sha256 trailing header is invalid.')(err, res, done)); + }); + + itSkipIfAWS( + 'testS3PutTrailerSDKMissingTrailer: x-amz-sdk-checksum-algorithm without x-amz-trailer -> 400 InvalidRequest', + done => { + const body = buildOkTrailerBody(); + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + // no x-amz-trailer + 'x-amz-sdk-checksum-algorithm': 'SHA256', + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(400, 'InvalidRequest', + msgSdkMissingTrailer)(err, res, done)); + }); + + itSkipIfAWS( + 'testS3PutTrailerNoTrailerInBody: x-amz-trailer header but body has no trailer -> 400 MalformedTrailerError', + done => { + // Body ends with "0\r\n\r\n" — empty trailer section, no checksum line. + const body = 'f\r\ntrailer content\r\n0\r\n\r\n'; + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha256', + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(400, 'MalformedTrailerError', + msgMalformedTrailer)(err, res, done)); + }); + + itSkipIfAWS( + 'testS3PutTrailerNoTrailerInBodyAndHeader: no x-amz-trailer, no body trailer -> 200 OK', + done => { + // No x-amz-trailer header; body just has chunked data with no trailer. + const body = 'f\r\ntrailer content\r\n0\r\n\r\n'; + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + // no x-amz-trailer + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(200)(err, res, done)); + }); + + itSkipIfAWS( + 'testS3PutTrailerDataAfterCRLF: data after final CRLF is ignored -> 200 OK', + done => { + // No x-amz-trailer; after the terminating CRLF there is extra data. + // TrailingChecksumTransform discards everything after streamClosed=true. + const body = 'f\r\ntrailer content\r\n0\r\n\r\nRANDOM DATA IGNORED'; + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + // no x-amz-trailer + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(200)(err, res, done)); + }); + + itSkipIfAWS( + 'testS3PutTrailerWithContentMD5: TRAILER + correct Content-MD5 header -> 200 OK', + done => { + const body = buildOkTrailerBody(); + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha256', + 'x-amz-sdk-checksum-algorithm': 'SHA256', + 'content-md5': trailerContentMd5B64, + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(200)(err, res, done)); + }); + + itSkipIfAWS( + 'testS3PutTrailerWithWhitespace: trailer line with whitespace around name and value -> 200 OK', + done => { + // TrailingChecksumTransform trims both name and value, so whitespace is accepted. + const body = + `f\r\ntrailer content\r\n0\r\n x-amz-checksum-sha256 : ${trailerContentSha256} \n\r\n\r\n\r\n`; + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha256', + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(200)(err, res, done)); + }); +} + +describe('PutObject: bad checksum is rejected', () => { + before(done => { + makeS3Request({ method: 'PUT', authCredentials, bucket }, err => { + assert.ifError(err); + done(); + }); + }); + + after(done => { + // Delete the object key first (defensive: clears any state left by a previous run). + makeS3Request({ method: 'DELETE', authCredentials, bucket, objectKey }, () => { + makeS3Request({ method: 'DELETE', authCredentials, bucket }, err => { + assert.ifError(err); + done(); + }); + }); + }); + + for (const protocol of protocols) { + for (const algo of algos) { + itSkipIfAWS( + `${protocol.name} with wrong x-amz-checksum-${algo.name}: returns 400 BadDigest`, + done => { + const url = `http://localhost:8000/${bucket}/${objectKey}`; + doPutRequest(url, protocol.buildHeaders(algo), protocol.buildBody(algo), + (err, res) => assertBadDigest(err, res, done)); + } + ); + } + } +}); + +describe('UploadPart: bad checksum is rejected', () => { + let uploadId; + + before(done => { + async.series([ + next => makeS3Request({ method: 'PUT', authCredentials, bucket }, next), + next => makeS3Request({ + method: 'POST', + authCredentials, + bucket, + objectKey, + queryObj: { uploads: '' }, + }, (err, res) => { + if (err) { return next(err); } + const match = res.body.match(/([^<]+)<\/UploadId>/); + assert(match, `missing UploadId in response: ${res.body}`); + uploadId = match[1]; + return next(); + }), + ], err => { + assert.ifError(err); + done(); + }); + }); + + after(done => { + async.series([ + next => makeS3Request({ + method: 'DELETE', + authCredentials, + bucket, + objectKey, + queryObj: { uploadId }, + }, next), + // Delete the object key first (defensive: clears any state left by a previous run). + next => makeS3Request({ method: 'DELETE', authCredentials, bucket, objectKey }, () => next()), + next => makeS3Request({ method: 'DELETE', authCredentials, bucket }, next), + ], err => { + assert.ifError(err); + done(); + }); + }); + + for (const protocol of protocols) { + for (const algo of algos) { + itSkipIfAWS( + `${protocol.name} with wrong x-amz-checksum-${algo.name}: returns 400 BadDigest`, + done => { + const url = `http://localhost:8000/${bucket}/${objectKey}` + + `?partNumber=1&uploadId=${uploadId}`; + doPutRequest(url, protocol.buildHeaders(algo), protocol.buildBody(algo), + (err, res) => assertBadDigest(err, res, done)); + } + ); + } + } +}); + +describe('PutObject: trailer and checksum protocol scenarios', () => { + before(done => { + makeS3Request({ method: 'PUT', authCredentials, bucket }, err => { + assert.ifError(err); + done(); + }); + }); + + after(done => { + // Some scenario tests store objects; delete the object key before removing the bucket. + makeS3Request({ method: 'DELETE', authCredentials, bucket, objectKey }, () => { + makeS3Request({ method: 'DELETE', authCredentials, bucket }, err => { + assert.ifError(err); + done(); + }); + }); + }); + + makeScenarioTests(() => `http://localhost:8000/${bucket}/${objectKey}`); + +}); + +describe('UploadPart: trailer and checksum protocol scenarios', () => { + let uploadId2; + + before(done => { + async.series([ + next => makeS3Request({ method: 'PUT', authCredentials, bucket }, next), + next => makeS3Request({ + method: 'POST', + authCredentials, + bucket, + objectKey, + queryObj: { uploads: '' }, + }, (err, res) => { + if (err) { return next(err); } + const match = res.body.match(/([^<]+)<\/UploadId>/); + assert(match, `missing UploadId in response: ${res.body}`); + uploadId2 = match[1]; + return next(); + }), + ], err => { + assert.ifError(err); + done(); + }); + }); + + after(done => { + async.series([ + next => makeS3Request({ + method: 'DELETE', + authCredentials, + bucket, + objectKey, + queryObj: { uploadId: uploadId2 }, + }, next), + // Delete the object key first (defensive: clears any state left by a previous run). + next => makeS3Request({ method: 'DELETE', authCredentials, bucket, objectKey }, () => next()), + next => makeS3Request({ method: 'DELETE', authCredentials, bucket }, next), + ], err => { + assert.ifError(err); + done(); + }); + }); + + makeScenarioTests( + () => `http://localhost:8000/${bucket}/${objectKey}?partNumber=1&uploadId=${uploadId2}` + ); +}); From 85e28c8332c5b7eded0bf97383e711b9d7d65c95 Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Tue, 17 Mar 2026 15:59:44 +0100 Subject: [PATCH 8/8] CLDSRV-872: store checksum in metadata for objects uploaded with PutObject --- .../apiUtils/object/createAndStoreObject.js | 7 +- lib/api/apiUtils/object/storeObject.js | 10 ++- lib/services.js | 6 +- .../raw-node/test/routes/routeMetadata.js | 82 ++++++++++++++++++- tests/unit/api/objectPut.js | 47 +++++++++++ 5 files changed, 143 insertions(+), 9 deletions(-) diff --git a/lib/api/apiUtils/object/createAndStoreObject.js b/lib/api/apiUtils/object/createAndStoreObject.js index 542b9f5296..d1b17b0e44 100644 --- a/lib/api/apiUtils/object/createAndStoreObject.js +++ b/lib/api/apiUtils/object/createAndStoreObject.js @@ -217,7 +217,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo, if (size === 0) { if (!dontSkipBackend[locationType]) { metadataStoreParams.contentMD5 = constants.emptyFileMd5; - return next(null, null, null); + return next(null, null, null, null); } // Handle mdOnlyHeader as a metadata only operation. If @@ -243,14 +243,14 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo, dataStoreVersionId: versionId, dataStoreMD5: _md5, }; - return next(null, dataGetInfo, _md5); + return next(null, dataGetInfo, _md5, null); } } return dataStore(objectKeyContext, cipherBundle, request, size, streamingV4Params, backendInfo, log, next); }, - function processDataResult(dataGetInfo, calculatedHash, next) { + function processDataResult(dataGetInfo, calculatedHash, checksum, next) { if (dataGetInfo === null || dataGetInfo === undefined) { return next(null, null); } @@ -275,6 +275,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo, dataGetInfoArr[0].size = mdOnlySize; } metadataStoreParams.contentMD5 = calculatedHash; + metadataStoreParams.checksum = checksum; return next(null, dataGetInfoArr); }, function getVersioningInfo(infoArr, next) { diff --git a/lib/api/apiUtils/object/storeObject.js b/lib/api/apiUtils/object/storeObject.js index 68a5312c4f..7963e97aa5 100644 --- a/lib/api/apiUtils/object/storeObject.js +++ b/lib/api/apiUtils/object/storeObject.js @@ -17,7 +17,7 @@ const { arsenalErrorFromChecksumError } = require('../../apiUtils/integrity/vali * @return {function} - calls callback with arguments: * error, dataRetrievalInfo, and completedHash (if any) */ -function checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo, log, cb) { +function checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo, checksumStream, log, cb) { const contentMD5 = stream.contentMD5; const completedHash = hashedStream.completedHash; if (contentMD5 && completedHash && contentMD5 !== completedHash) { @@ -37,7 +37,10 @@ function checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo, log, cb) { return cb(errors.BadDigest); }); } - return cb(null, dataRetrievalInfo, completedHash); + const checksum = checksumStream.digest + ? { algorithm: checksumStream.algoName, value: checksumStream.digest, type: 'FULL_OBJECT' } + : null; + return cb(null, dataRetrievalInfo, completedHash, checksum); } /** @@ -107,7 +110,8 @@ function dataStore(objectContext, cipherBundle, stream, size, return cbOnce(arsenalErrorFromChecksumError(checksumErr)); }); } - return checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo, log, cbOnce); + return checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo, + checksumedStream.stream, log, cbOnce); }; // ChecksumTransform._flush computes the digest asynchronously for diff --git a/lib/services.js b/lib/services.js index f2980f1fc5..43b41c9548 100644 --- a/lib/services.js +++ b/lib/services.js @@ -6,6 +6,7 @@ const { errors, s3middleware } = require('arsenal'); const ObjectMD = require('arsenal').models.ObjectMD; const BucketInfo = require('arsenal').models.BucketInfo; const ObjectMDArchive = require('arsenal').models.ObjectMDArchive; +const ObjectMDChecksum = require('arsenal').models.ObjectMDChecksum; const { versioning } = require('arsenal'); const acl = require('./metadata/acl'); const constants = require('../constants'); @@ -102,7 +103,7 @@ const services = { * @return {function} executes callback with err or ETag as arguments */ metadataStoreObject(bucketName, dataGetInfo, cipherBundle, params, cb) { - const { objectKey, authInfo, size, contentMD5, metaHeaders, + const { objectKey, authInfo, size, contentMD5, checksum, metaHeaders, contentType, cacheControl, contentDisposition, contentEncoding, expires, multipart, headers, overrideMetadata, log, lastModifiedDate, versioning, versionId, uploadId, @@ -138,6 +139,9 @@ const services = { // CreationTime needs to be carried over so that it remains static .setCreationTime(creationTime) .setOriginOp(originOp); + if (checksum) { + md.setChecksum(new ObjectMDChecksum(checksum.algorithm, checksum.value, checksum.type)); + } // Sending in last modified date in object put copy since need // to return the exact date in the response if (lastModifiedDate) { diff --git a/tests/functional/raw-node/test/routes/routeMetadata.js b/tests/functional/raw-node/test/routes/routeMetadata.js index 81b24a2bfa..2c8f2c1a12 100644 --- a/tests/functional/raw-node/test/routes/routeMetadata.js +++ b/tests/functional/raw-node/test/routes/routeMetadata.js @@ -1,7 +1,10 @@ const assert = require('assert'); +const crypto = require('crypto'); +const { CrtCrc64Nvme } = require('@aws-sdk/crc64-nvme-crt'); const http = require('http'); -const { CreateBucketCommand, - PutObjectCommand, +const { CreateBucketCommand, + PutObjectCommand, + DeleteObjectCommand, DeleteBucketCommand } = require('@aws-sdk/client-s3'); const { makeRequest } = require('../../utils/makeRequest'); @@ -144,3 +147,78 @@ describe('metadata routes with metadata', () => { }); }); }); + +describe('checksum stored in object metadata after PutObject', () => { + const bucketUtil = new BucketUtility('default', { signatureVersion: 'v4' }); + const s3 = bucketUtil.s3; + + const bucket = 'bucket1'; + const objectBody = 'hello checksum'; + const sha256Key = 'object-with-sha256-checksum'; + const defaultKey = 'object-with-default-checksum'; + + let expectedCrc64nvme; + + before(async function () { + if (!process.env.S3_END_TO_END) { + this.skip(); + } + const crc = new CrtCrc64Nvme(); + crc.update(Buffer.from(objectBody)); + expectedCrc64nvme = Buffer.from(await crc.digest()).toString('base64'); + + const sha256Value = crypto.createHash('sha256').update(objectBody).digest('base64'); + await s3.send(new PutObjectCommand({ + Bucket: bucket, + Key: sha256Key, + Body: objectBody, + ChecksumSHA256: sha256Value, + })); + await s3.send(new PutObjectCommand({ + Bucket: bucket, + Key: defaultKey, + Body: objectBody, + })); + }); + + after(async () => { + if (!process.env.S3_END_TO_END) { + return; + } + await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: sha256Key })); + await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: defaultKey })); + }); + + it('stores sha256 checksum in metadata when x-amz-checksum-sha256 is provided', done => { + const expectedValue = crypto.createHash('sha256').update(objectBody).digest('base64'); + makeMetadataRequest({ + method: 'GET', + authCredentials: metadataAuthCredentials, + path: `/_/metadata/default/bucket/${bucket}/${sha256Key}`, + }, (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 200); + const md = JSON.parse(res.body); + assert.strictEqual(md.checksum.checksumAlgorithm, 'sha256'); + assert.strictEqual(md.checksum.checksumValue, expectedValue); + assert.strictEqual(md.checksum.checksumType, 'FULL_OBJECT'); + return done(); + }); + }); + + it('stores crc64nvme checksum in metadata when no checksum header is provided', done => { + makeMetadataRequest({ + method: 'GET', + authCredentials: metadataAuthCredentials, + path: `/_/metadata/default/bucket/${bucket}/${defaultKey}`, + }, (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 200); + const md = JSON.parse(res.body); + assert.strictEqual(md.checksum.checksumAlgorithm, 'crc64nvme'); + assert.strictEqual(md.checksum.checksumValue, expectedCrc64nvme); + assert.strictEqual(md.checksum.checksumType, 'FULL_OBJECT'); + return done(); + }); + }); +}); diff --git a/tests/unit/api/objectPut.js b/tests/unit/api/objectPut.js index 3f4c458d60..4e219c53e7 100644 --- a/tests/unit/api/objectPut.js +++ b/tests/unit/api/objectPut.js @@ -1,5 +1,6 @@ const assert = require('assert'); const async = require('async'); +const crypto = require('crypto'); const moment = require('moment'); const { s3middleware, storage, versioning } = require('arsenal'); const sinon = require('sinon'); @@ -865,6 +866,52 @@ describe('objectPut API', () => { }); }); }); + + it('should store sha256 checksum in metadata when x-amz-checksum-sha256 header is provided', done => { + const sha256Value = crypto.createHash('sha256').update(postBody).digest('base64'); + const request = new DummyRequest({ + bucketName, + namespace, + objectKey: objectName, + headers: { + host: `${bucketName}.s3.amazonaws.com`, + 'x-amz-checksum-sha256': sha256Value, + }, + url: '/', + }, postBody); + + bucketPut(authInfo, testPutBucketRequest, log, err => { + assert.ifError(err); + objectPut(authInfo, request, undefined, log, err => { + assert.ifError(err); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { + assert.ifError(err); + assert(md.checksum, 'checksum should be set in metadata'); + assert.strictEqual(md.checksum.checksumAlgorithm, 'sha256'); + assert.strictEqual(md.checksum.checksumValue, sha256Value); + assert.strictEqual(md.checksum.checksumType, 'FULL_OBJECT'); + done(); + }); + }); + }); + }); + + it('should store crc64nvme checksum in metadata when no checksum header is provided', done => { + bucketPut(authInfo, testPutBucketRequest, log, err => { + assert.ifError(err); + objectPut(authInfo, testPutObjectRequest, undefined, log, err => { + assert.ifError(err); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { + assert.ifError(err); + assert(md.checksum, 'checksum should be set in metadata'); + assert.strictEqual(md.checksum.checksumAlgorithm, 'crc64nvme'); + assert(md.checksum.checksumValue, 'checksumValue should be set'); + assert.strictEqual(md.checksum.checksumType, 'FULL_OBJECT'); + done(); + }); + }); + }); + }); }); describe('objectPut API with versioning', () => {