diff --git a/lib/api/apiUtils/integrity/validateChecksums.js b/lib/api/apiUtils/integrity/validateChecksums.js index 9643a71258..b6fed91063 100644 --- a/lib/api/apiUtils/integrity/validateChecksums.js +++ b/lib/api/apiUtils/integrity/validateChecksums.js @@ -2,7 +2,21 @@ const crypto = require('crypto'); const { Crc32 } = require('@aws-crypto/crc32'); const { Crc32c } = require('@aws-crypto/crc32c'); const { CrtCrc64Nvme } = require('@aws-sdk/crc64-nvme-crt'); -const { errors: ArsenalErrors } = require('arsenal'); +const { errors: ArsenalErrors, errorInstances } = require('arsenal'); + +const errAlgoNotSupported = errorInstances.InvalidRequest.customizeDescription( + 'The algorithm type you specified in x-amz-checksum- header is invalid.'); +const errAlgoNotSupportedSDK = errorInstances.InvalidRequest.customizeDescription( + 'Value for x-amz-sdk-checksum-algorithm header is invalid.'); +const errMissingCorresponding = errorInstances.InvalidRequest.customizeDescription( + 'x-amz-sdk-checksum-algorithm specified, but no corresponding x-amz-checksum-* ' + + 'or x-amz-trailer headers were found.'); +const errMultipleChecksumTypes = errorInstances.InvalidRequest.customizeDescription( + 'Expecting a single x-amz-checksum- header. Multiple checksum Types are not allowed.'); +const errTrailerAndChecksum = errorInstances.InvalidRequest.customizeDescription( + 'Expecting a single x-amz-checksum- header'); +const errTrailerNotSupported = errorInstances.InvalidRequest.customizeDescription( + 'The value specified in the x-amz-trailer header is not supported'); const { config } = require('../../../Config'); const checksumedMethods = Object.freeze({ @@ -37,6 +51,12 @@ const ChecksumError = Object.freeze({ MultipleChecksumTypes: 'MultipleChecksumTypes', MissingCorresponding: 'MissingCorresponding', MalformedChecksum: 'MalformedChecksum', + TrailerAlgoMismatch: 'TrailerAlgoMismatch', + TrailerChecksumMalformed: 'TrailerChecksumMalformed', + TrailerMissing: 'TrailerMissing', + TrailerUnexpected: 'TrailerUnexpected', + TrailerAndChecksum: 'TrailerAndChecksum', + TrailerNotSupported: 'TrailerNotSupported', }); const base64Regex = /^[A-Za-z0-9+/]*={0,2}$/; @@ -56,35 +76,51 @@ const algorithms = Object.freeze({ const result = await crc.digest(); return Buffer.from(result).toString('base64'); }, + digestFromHash: async hash => { + const result = await hash.digest(); + return Buffer.from(result).toString('base64'); + }, isValidDigest: expected => typeof expected === 'string' && expected.length === 12 && base64Regex.test(expected), + createHash: () => new CrtCrc64Nvme() }, crc32: { digest: data => { const input = Buffer.isBuffer(data) ? data : Buffer.from(data); return uint32ToBase64(new Crc32().update(input).digest() >>> 0); // >>> 0 coerce number to uint32 }, + digestFromHash: hash => { + const result = hash.digest(); + return uint32ToBase64(result >>> 0); + }, isValidDigest: expected => typeof expected === 'string' && expected.length === 8 && base64Regex.test(expected), + createHash: () => new Crc32() }, crc32c: { digest: data => { const input = Buffer.isBuffer(data) ? data : Buffer.from(data); return uint32ToBase64(new Crc32c().update(input).digest() >>> 0); // >>> 0 coerce number to uint32 }, + digestFromHash: hash => uint32ToBase64(hash.digest() >>> 0), isValidDigest: expected => typeof expected === 'string' && expected.length === 8 && base64Regex.test(expected), + createHash: () => new Crc32c() }, sha1: { digest: data => { const input = Buffer.isBuffer(data) ? data : Buffer.from(data); return crypto.createHash('sha1').update(input).digest('base64'); }, + digestFromHash: hash => hash.digest('base64'), isValidDigest: expected => typeof expected === 'string' && expected.length === 28 && base64Regex.test(expected), + createHash: () => crypto.createHash('sha1') }, sha256: { digest: data => { const input = Buffer.isBuffer(data) ? data : Buffer.from(data); return crypto.createHash('sha256').update(input).digest('base64'); }, + digestFromHash: hash => hash.digest('base64'), isValidDigest: expected => typeof expected === 'string' && expected.length === 44 && base64Regex.test(expected), + createHash: () => crypto.createHash('sha256') } }); @@ -132,7 +168,7 @@ async function validateXAmzChecksums(headers, body) { return { error: ChecksumError.AlgoNotSupportedSDK, details: { algorithm: sdkAlgo } }; } - // If AWS there is a mismatch, AWS returns the same error as if the algo was invalid. + // If there is a mismatch, AWS returns the same error as if the algo was invalid. if (sdkLowerAlgo !== algo) { return { error: ChecksumError.AlgoNotSupportedSDK, details: { algorithm: sdkAlgo } }; } @@ -141,6 +177,94 @@ async function validateXAmzChecksums(headers, body) { return null; } +function getChecksumDataFromHeaders(headers) { + const checkSdk = algo => { + if (!('x-amz-sdk-checksum-algorithm' in headers)) { + return null; + } + + const sdkAlgo = headers['x-amz-sdk-checksum-algorithm']; + if (typeof sdkAlgo !== 'string') { + return { error: ChecksumError.AlgoNotSupportedSDK, details: { algorithm: sdkAlgo } }; + } + + const sdkLowerAlgo = sdkAlgo.toLowerCase(); + if (!(sdkLowerAlgo in algorithms)) { + return { error: ChecksumError.AlgoNotSupportedSDK, details: { algorithm: sdkAlgo } }; + } + + // If there is a mismatch, AWS returns the same error as if the algo was invalid. + if (sdkLowerAlgo !== algo) { + return { error: ChecksumError.AlgoNotSupportedSDK, details: { algorithm: sdkAlgo } }; + } + + return null; + }; + + const checksumHeaders = Object.keys(headers).filter(header => header.startsWith('x-amz-checksum-')); + const xAmzChecksumCnt = checksumHeaders.length; + if (xAmzChecksumCnt > 1) { + return { error: ChecksumError.MultipleChecksumTypes, details: { algorithms: checksumHeaders } }; + } + + if (xAmzChecksumCnt === 0 && !('x-amz-trailer' in headers) && 'x-amz-sdk-checksum-algorithm' in headers) { + return { + error: ChecksumError.MissingCorresponding, + details: { expected: headers['x-amz-sdk-checksum-algorithm'] } + }; + } + + if ('x-amz-trailer' in headers) { + if (xAmzChecksumCnt !== 0) { + return { + error: ChecksumError.TrailerAndChecksum, + details: { trailer: headers['x-amz-trailer'], checksum: checksumHeaders }, + }; + } + + const trailer = headers['x-amz-trailer']; + if (!trailer.startsWith('x-amz-checksum-')) { + return { error: ChecksumError.TrailerNotSupported, details: { value: trailer } }; + } + + const trailerAlgo = trailer.slice('x-amz-checksum-'.length); + if (!(trailerAlgo in algorithms)) { + return { error: ChecksumError.TrailerNotSupported, details: { value: trailer } }; + } + + const err = checkSdk(trailerAlgo); + if (err) { + return err; + } + + return { algorithm: trailerAlgo, isTrailer: true, expected: undefined }; + } + + if (xAmzChecksumCnt === 0) { + // There was no x-amz-checksum- or x-amz-trailer return crc64nvme. + // The calculated crc64nvme will be stored in the object metadata. + return { algorithm: 'crc64nvme', isTrailer: false, expected: undefined }; + } + + // No x-amz-sdk-checksum-algorithm we expect one x-amz-checksum-[crc64nvme, crc32, crc32C, sha1, sha256]. + const algo = checksumHeaders[0].slice('x-amz-checksum-'.length); + if (!(algo in algorithms)) { + return { error: ChecksumError.AlgoNotSupported, details: { algorithm: algo } }; + } + + const expected = headers[`x-amz-checksum-${algo}`]; + if (!algorithms[algo].isValidDigest(expected)) { + return { error: ChecksumError.MalformedChecksum, details: { algorithm: algo, expected } }; + } + + const err = checkSdk(algo); + if (err) { + return err; + } + + return { algorithm: algo, isTrailer: false, expected }; +} + /** * validateChecksumsNoChunking - Validate the checksums of a request. * @param {object} headers - http headers @@ -183,53 +307,59 @@ async function validateChecksumsNoChunking(headers, body) { return err; } -async function defaultValidationFunc(request, body, log) { - const err = await validateChecksumsNoChunking(request.headers, body); - if (!err) { - return null; - } - - if (err.error !== ChecksumError.MissingChecksum) { - log.debug('failed checksum validation', { method: request.apiMethod }, err); - } - +function arsenalErrorFromChecksumError(err) { switch (err.error) { case ChecksumError.MissingChecksum: return null; case ChecksumError.XAmzMismatch: { const algoUpper = err.details.algorithm.toUpperCase(); - return ArsenalErrors.BadDigest.customizeDescription( - `The ${algoUpper} you specified did not match the calculated checksum.` - ); + return errorInstances.BadDigest.customizeDescription( + `The ${algoUpper} you specified did not match the calculated checksum.`); } case ChecksumError.AlgoNotSupported: - return ArsenalErrors.InvalidRequest.customizeDescription( - 'The algorithm type you specified in x-amz-checksum- header is invalid.' - ); + return errAlgoNotSupported; case ChecksumError.AlgoNotSupportedSDK: - return ArsenalErrors.InvalidRequest.customizeDescription( - 'Value for x-amz-sdk-checksum-algorithm header is invalid.' - ); + return errAlgoNotSupportedSDK; case ChecksumError.MissingCorresponding: - return ArsenalErrors.InvalidRequest.customizeDescription( - 'x-amz-sdk-checksum-algorithm specified, but no corresponding x-amz-checksum-* ' + - 'or x-amz-trailer headers were found.' - ); + return errMissingCorresponding; case ChecksumError.MultipleChecksumTypes: - return ArsenalErrors.InvalidRequest.customizeDescription( - 'Expecting a single x-amz-checksum- header. Multiple checksum Types are not allowed.' - ); + return errMultipleChecksumTypes; case ChecksumError.MalformedChecksum: - return ArsenalErrors.InvalidRequest.customizeDescription( - `Value for x-amz-checksum-${err.details.algorithm} header is invalid.` - ); + return errorInstances.InvalidRequest.customizeDescription( + `Value for x-amz-checksum-${err.details.algorithm} header is invalid.`); case ChecksumError.MD5Invalid: return ArsenalErrors.InvalidDigest; + case ChecksumError.TrailerAlgoMismatch: + return ArsenalErrors.MalformedTrailerError; + case ChecksumError.TrailerMissing: + return ArsenalErrors.MalformedTrailerError; + case ChecksumError.TrailerUnexpected: + return ArsenalErrors.MalformedTrailerError; + case ChecksumError.TrailerChecksumMalformed: + return errorInstances.InvalidRequest.customizeDescription( + `Value for x-amz-checksum-${err.details.algorithm} trailing header is invalid.`); + case ChecksumError.TrailerAndChecksum: + return errTrailerAndChecksum; + case ChecksumError.TrailerNotSupported: + return errTrailerNotSupported; default: return ArsenalErrors.BadDigest; } } +async function defaultValidationFunc(request, body, log) { + const err = await validateChecksumsNoChunking(request.headers, body); + if (!err) { + return null; + } + + if (err.error !== ChecksumError.MissingChecksum) { + log.debug('failed checksum validation', { method: request.apiMethod }, err); + } + + return arsenalErrorFromChecksumError(err); +} + /** * validateMethodChecksumsNoChunking - Validate the checksums of a request. * @param {object} request - http request @@ -253,5 +383,8 @@ module.exports = { ChecksumError, validateChecksumsNoChunking, validateMethodChecksumNoChunking, + getChecksumDataFromHeaders, + arsenalErrorFromChecksumError, + algorithms, checksumedMethods, }; diff --git a/lib/api/apiUtils/object/prepareStream.js b/lib/api/apiUtils/object/prepareStream.js index be3f6c69ca..dda1276bd9 100644 --- a/lib/api/apiUtils/object/prepareStream.js +++ b/lib/api/apiUtils/object/prepareStream.js @@ -1,50 +1,114 @@ const V4Transform = require('../../../auth/streamingV4/V4Transform'); const TrailingChecksumTransform = require('../../../auth/streamingV4/trailingChecksumTransform'); +const ChecksumTransform = require('../../../auth/streamingV4/ChecksumTransform'); +const { + getChecksumDataFromHeaders, + arsenalErrorFromChecksumError, +} = require('../../apiUtils/integrity/validateChecksums'); +const { errors, errorInstances, jsutil } = require('arsenal'); +const { unsupportedSignatureChecksums } = require('../../../../constants'); /** - * Prepares the stream if the chunks are sent in a v4 Auth request - * @param {object} stream - stream containing the data - * @param {object | null } streamingV4Params - if v4 auth, object containing - * accessKey, signatureFromRequest, region, scopeDate, timestamp, and - * credentialScope (to be used for streaming v4 auth if applicable) - * @param {RequestLogger} log - the current request logger - * @param {function} errCb - callback called if an error occurs - * @return {object|null} - V4Transform object if v4 Auth request, or - * the original stream, or null if the request has no V4 params but - * the type of request requires them + * Prepares the request stream for data storage by wrapping it in the + * appropriate transform pipeline based on the x-amz-content-sha256 header. + * Always returns a ChecksumTransform as the final stream. + * If no checksum was sent by the client a CRC64NVME ChecksumTransform is returned. + * + * @param {object} request - incoming HTTP request with headers and body stream + * @param {object|null} streamingV4Params - v4 streaming auth params (accessKey, + * signatureFromRequest, region, scopeDate, timestamp, credentialScope), or + * null/undefined for non-v4 requests + * @param {RequestLogger} log - request logger + * @param {function} errCb - error callback invoked if a stream error occurs + * @return {{ error: Arsenal.Error|null, stream: ChecksumTransform|null }} + * error is set and stream is null if the request headers are invalid; + * otherwise error is null and stream is the ChecksumTransform to read from */ -function prepareStream(stream, streamingV4Params, log, errCb) { - if (stream.headers['x-amz-content-sha256'] === - 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD') { - if (typeof streamingV4Params !== 'object') { - // this might happen if the user provided a valid V2 - // Authentication header, while the chunked upload method - // requires V4: in such case we don't get any V4 params - // and we should return an error to the client. - return null; - } - const v4Transform = new V4Transform(streamingV4Params, log, errCb); - stream.pipe(v4Transform); - v4Transform.headers = stream.headers; - return v4Transform; - } - return stream; -} +function prepareStream(request, streamingV4Params, log, errCb) { + const xAmzContentSHA256 = request.headers['x-amz-content-sha256']; -function stripTrailingChecksumStream(stream, log, errCb) { - // don't do anything if we are not in the correct integrity check mode - if (stream.headers['x-amz-content-sha256'] !== 'STREAMING-UNSIGNED-PAYLOAD-TRAILER') { - return stream; + const checksumAlgo = getChecksumDataFromHeaders(request.headers); + if (checksumAlgo.error) { + log.debug('prepareStream invalid checksum headers', checksumAlgo); + return { error: arsenalErrorFromChecksumError(checksumAlgo), stream: null }; } - const trailingChecksumTransform = new TrailingChecksumTransform(log); - trailingChecksumTransform.on('error', errCb); - stream.pipe(trailingChecksumTransform); - trailingChecksumTransform.headers = stream.headers; - return trailingChecksumTransform; + let stream = request; + switch (xAmzContentSHA256) { + case 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD': { + if (streamingV4Params === null || typeof streamingV4Params !== 'object') { + // this might happen if the user provided a valid V2 + // Authentication header, while the chunked upload method + // requires V4: in such case we don't get any V4 params + // and we should return an error to the client. + log.error('missing v4 streaming params for chunked upload', { + method: 'prepareStream', + streamingV4ParamsType: typeof streamingV4Params, + streamingV4Params, + }); + return { error: errors.InvalidArgument, stream: null }; + } + // Use a once-guard so that if both V4Transform and ChecksumTransform + // independently error, errCb is only called once. + const onStreamError = jsutil.once(errCb); + const v4Transform = new V4Transform(streamingV4Params, log, onStreamError); + request.pipe(v4Transform); + v4Transform.headers = request.headers; + stream = v4Transform; + + const checksumedStream = new ChecksumTransform( + checksumAlgo.algorithm, + checksumAlgo.expected, + checksumAlgo.isTrailer, + log, + ); + checksumedStream.on('error', onStreamError); + stream.pipe(checksumedStream); + return { error: null, stream: checksumedStream }; + } + case 'STREAMING-UNSIGNED-PAYLOAD-TRAILER': { + // Use a once-guard so that auto-destroying both piped streams + // when one errors does not result in errCb being called twice. + const onStreamError = jsutil.once(errCb); + const trailingChecksumTransform = new TrailingChecksumTransform(log); + trailingChecksumTransform.on('error', onStreamError); + request.pipe(trailingChecksumTransform); + trailingChecksumTransform.headers = request.headers; + stream = trailingChecksumTransform; + + const checksumedStream = new ChecksumTransform( + checksumAlgo.algorithm, + checksumAlgo.expected, + checksumAlgo.isTrailer, + log, + ); + checksumedStream.on('error', onStreamError); + trailingChecksumTransform.on('trailer', (name, value) => { + checksumedStream.setExpectedChecksum(name, value); + }); + stream.pipe(checksumedStream); + return { error: null, stream: checksumedStream }; + } + case 'UNSIGNED-PAYLOAD': // Fallthrough + default: { + if (unsupportedSignatureChecksums.has(xAmzContentSHA256)) { + return { + error: errorInstances.BadRequest.customizeDescription(`${xAmzContentSHA256} is not supported`), + stream: null, + }; + } + + const checksumedStream = new ChecksumTransform( + checksumAlgo.algorithm, + checksumAlgo.expected, + checksumAlgo.isTrailer, + log, + ); + checksumedStream.on('error', errCb); + stream.pipe(checksumedStream); + return { error: null, stream: checksumedStream }; + } + } } -module.exports = { - prepareStream, - stripTrailingChecksumStream, -}; +module.exports = { prepareStream }; diff --git a/lib/api/apiUtils/object/storeObject.js b/lib/api/apiUtils/object/storeObject.js index 8beea03ecb..68a5312c4f 100644 --- a/lib/api/apiUtils/object/storeObject.js +++ b/lib/api/apiUtils/object/storeObject.js @@ -1,7 +1,8 @@ const { errors, jsutil } = require('arsenal'); const { data } = require('../../../data/wrapper'); -const { prepareStream, stripTrailingChecksumStream } = require('./prepareStream'); +const { prepareStream } = require('./prepareStream'); +const { arsenalErrorFromChecksumError } = require('../../apiUtils/integrity/validateChecksums'); /** * Check that `hashedStream.completedHash` matches header `stream.contentMD5` @@ -58,31 +59,67 @@ function checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo, log, cb) { function dataStore(objectContext, cipherBundle, stream, size, streamingV4Params, backendInfo, log, cb) { const cbOnce = jsutil.once(cb); - const dataStreamTmp = prepareStream(stream, streamingV4Params, log, cbOnce); - if (!dataStreamTmp) { - return process.nextTick(() => cb(errors.InvalidArgument)); + + // errCb is delegated through a mutable reference so it can be upgraded to + // include batchDelete once data.put has actually stored data. + let onStreamError = cbOnce; + const errCb = err => onStreamError(err); + + const checksumedStream = prepareStream(stream, streamingV4Params, log, errCb); + if (checksumedStream.error) { + log.debug('dataStore failed to prepare stream', checksumedStream); + return process.nextTick(() => cbOnce(checksumedStream.error)); } - const dataStream = stripTrailingChecksumStream(dataStreamTmp, log, cbOnce); return data.put( - cipherBundle, dataStream, size, objectContext, backendInfo, log, + cipherBundle, checksumedStream.stream, size, objectContext, backendInfo, log, (err, dataRetrievalInfo, hashedStream) => { if (err) { - log.error('error in datastore', { - error: err, - }); + log.error('error in datastore', { error: err }); return cbOnce(err); } if (!dataRetrievalInfo) { - log.fatal('data put returned neither an error nor a key', { - method: 'storeObject::dataStore', - }); + log.fatal('data put returned neither an error nor a key', { method: 'storeObject::dataStore' }); return cbOnce(errors.InternalError); } - log.trace('dataStore: backend stored key', { - dataRetrievalInfo, - }); - return checkHashMatchMD5(stream, hashedStream, - dataRetrievalInfo, log, cbOnce); + log.trace('dataStore: backend stored key', { dataRetrievalInfo }); + + // Data is now stored. Upgrade the error handler so that any stream + // error from this point on cleans up the stored data first. + onStreamError = streamErr => { + log.error('checksum stream error after data.put', { error: streamErr }); + return data.batchDelete([dataRetrievalInfo], null, null, log, deleteErr => { + if (deleteErr) { + log.error('dataStore failed to delete old data', { error: deleteErr }); + } + cbOnce(streamErr); + }); + }; + + const doValidate = () => { + const checksumErr = checksumedStream.stream.validateChecksum(); + if (checksumErr) { + log.debug('failed checksum validation stream', checksumErr); + return data.batchDelete([dataRetrievalInfo], null, null, log, deleteErr => { + if (deleteErr) { + // Failure of batch delete is only logged. + log.error('dataStore failed to delete old data', { error: deleteErr }); + } + return cbOnce(arsenalErrorFromChecksumError(checksumErr)); + }); + } + return checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo, log, cbOnce); + }; + + // ChecksumTransform._flush computes the digest asynchronously for + // some algorithms (e.g. crc64nvme). writableFinished is true once + // _flush has called its callback, guaranteeing this.digest is set. + // Stream piping ordering means this is virtually always true here, + // but we wait for 'finish' explicitly to be safe. + if (checksumedStream.stream.writableFinished) { + return doValidate(); + } + checksumedStream.stream.once('finish', doValidate); + return undefined; }); } diff --git a/lib/auth/streamingV4/ChecksumTransform.js b/lib/auth/streamingV4/ChecksumTransform.js new file mode 100644 index 0000000000..4e066941d6 --- /dev/null +++ b/lib/auth/streamingV4/ChecksumTransform.js @@ -0,0 +1,94 @@ +const { errors } = require('arsenal'); +const { algorithms, ChecksumError } = require('../../api/apiUtils/integrity/validateChecksums'); +const { Transform } = require('stream'); + +class ChecksumTransform extends Transform { + constructor(algoName, expectedDigest, isTrailer, log) { + super({}); + this.log = log; + this.algoName = algoName; + this.algo = algorithms[algoName]; + this.hash = this.algo.createHash(); + this.digest = undefined; + this.expectedDigest = expectedDigest; + this.isTrailer = isTrailer; + this.trailerChecksumName = undefined; + this.trailerChecksumValue = undefined; + } + + setExpectedChecksum(name, value) { + this.trailerChecksumName = name; + this.trailerChecksumValue = value; + } + + validateChecksum() { + if (this.isTrailer) { + // x-amz-trailer in headers but no trailer in body. + if (this.trailerChecksumValue === undefined) { + return { + error: ChecksumError.TrailerMissing, + details: { expectedTrailer: `x-amz-checksum-${this.algoName}` }, + }; + } + + if (this.trailerChecksumName !== `x-amz-checksum-${this.algoName}`) { + return { error: ChecksumError.TrailerAlgoMismatch, details: { algorithm: this.algoName } }; + } + + const expected = this.trailerChecksumValue; + if (!this.algo.isValidDigest(expected)) { + return { + error: ChecksumError.TrailerChecksumMalformed, + details: { algorithm: this.algoName, expected }, + }; + } + + if (this.digest !== this.trailerChecksumValue) { + return { + error: ChecksumError.XAmzMismatch, + details: { algorithm: this.algoName, calculated: this.digest, expected }, + }; + } + + return null; + } + + if (this.trailerChecksumValue !== undefined) { + // Trailer found in the body but no x-amz-trailer in the headers. + return { + error: ChecksumError.TrailerUnexpected, + details: { name: this.trailerChecksumName, val: this.trailerChecksumValue }, + }; + } + + if (this.expectedDigest) { + if (this.digest !== this.expectedDigest) { + return { + error: ChecksumError.XAmzMismatch, + details: { algorithm: this.algoName, calculated: this.digest, expected: this.expectedDigest }, + }; + } + } + + return null; + } + + _flush(callback) { + Promise.resolve(this.algo.digestFromHash(this.hash)) + .then(digest => { + this.digest = digest; + callback(); + }, err => { + this.log.error('failed to compute checksum digest', { error: err, algorithm: this.algoName }); + callback(errors.InternalError); + }); + } + + _transform(chunk, encoding, callback) { + const input = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + this.hash.update(input, encoding); + callback(null, input); + } +} + +module.exports = ChecksumTransform; diff --git a/lib/auth/streamingV4/trailingChecksumTransform.js b/lib/auth/streamingV4/trailingChecksumTransform.js index 48870c80e7..35a1fa5c14 100644 --- a/lib/auth/streamingV4/trailingChecksumTransform.js +++ b/lib/auth/streamingV4/trailingChecksumTransform.js @@ -1,12 +1,16 @@ const { Transform } = require('stream'); -const { errors } = require('arsenal'); +const { errors, errorInstances } = require('arsenal'); const { maximumAllowedPartSize } = require('../../../constants'); +const incompleteBodyError = errorInstances.IncompleteBody.customizeDescription( + 'The request body terminated unexpectedly'); + /** - * This class is designed to handle the chunks sent in a streaming - * unsigned playload trailer request. In this iteration, we are not checking - * the checksums, but we are removing them from the stream. - * S3C-9732 will deal with checksum verification. + * This class handles the chunked-upload body format used by + * STREAMING-UNSIGNED-PAYLOAD-TRAILER requests. It strips the chunk-size + * headers and trailing checksum trailer from the stream, forwarding only + * the raw object data. The trailer name and value are emitted via a + * 'trailer' event so that ChecksumTransform can validate the checksum. */ class TrailingChecksumTransform extends Transform { /** @@ -20,6 +24,10 @@ class TrailingChecksumTransform extends Transform { this.bytesToDiscard = 0; // when trailing \r\n are present, we discard them but they can be in different chunks this.bytesToRead = 0; // when a chunk is advertised, the size is put here and we forward all bytes this.streamClosed = false; + this.readingTrailer = false; + this.trailerBuffer = Buffer.alloc(0); + this.trailerName = null; + this.trailerValue = null; } /** @@ -30,9 +38,16 @@ class TrailingChecksumTransform extends Transform { * @return {function} executes callback with err if applicable */ _flush(callback) { - if (!this.streamClosed) { + if (!this.streamClosed && this.readingTrailer && this.trailerBuffer.length === 0) { + // Nothing came after "0\r\n", don't fail. + // If the x-amz-trailer header was present then the trailer is required and ChecksumTransform will fail. + return callback(); + } else if (!this.streamClosed && this.readingTrailer && this.trailerBuffer.length !== 0) { + this.log.error('stream ended without trailer "\r\n"'); + return callback(incompleteBodyError); + } else if (!this.streamClosed && !this.readingTrailer) { this.log.error('stream ended without closing chunked encoding'); - return callback(errors.InvalidArgument); + return callback(incompleteBodyError); } return callback(); } @@ -66,6 +81,49 @@ class TrailingChecksumTransform extends Transform { continue; } + // after the 0-size chunk, read the trailer line (e.g. "x-amz-checksum-crc32:YABb/g==") + if (this.readingTrailer) { + const combined = Buffer.concat([this.trailerBuffer, chunk]); + const lineBreakIndex = combined.indexOf('\r\n'); + if (lineBreakIndex === -1) { + if (combined.byteLength > 1024) { + this.log.error('trailer line too long'); + return callback(errors.MalformedTrailerError); + } + // The trailer is not complete yet, continue. + this.trailerBuffer = combined; + return callback(); + } + this.trailerBuffer = Buffer.alloc(0); + const fullTrailer = combined.subarray(0, lineBreakIndex); + if (fullTrailer.length === 0) { + // The trailer is empty, stop reading. + this.readingTrailer = false; + this.streamClosed = true; + return callback(); + } + let trailerLine = fullTrailer.toString(); + // Some clients terminate the trailer with \n\r\n instead of + // just \r\n, producing a trailing \n in the parsed line. + if (trailerLine.endsWith('\n')) { + trailerLine = trailerLine.slice(0, -1); + } + const colonIndex = trailerLine.indexOf(':'); + if (colonIndex > 0) { + this.trailerName = trailerLine.slice(0, colonIndex).trim(); + this.trailerValue = trailerLine.slice(colonIndex + 1).trim(); + this.emit('trailer', this.trailerName, this.trailerValue); + } else { + this.log.error('incomplete trailer missing ":"', { trailerLine }); + return callback(incompleteBodyError); + } + this.readingTrailer = false; + this.streamClosed = true; + // The trailer \r\n is the last bytes of the stream per the AWS + // chunked upload format, so any remaining bytes are discarded. + return callback(); + } + // we are now looking for the chunk size field // no need to look further than 10 bytes since the field cannot be bigger: the max // chunk size is 5GB (see constants.maximumAllowedPartSize) @@ -100,9 +158,9 @@ class TrailingChecksumTransform extends Transform { } this.chunkSizeBuffer = Buffer.alloc(0); if (dataSize === 0) { - // TODO: check if the checksum is correct (S3C-9732) - // last chunk, no more data to read, the stream is closed - this.streamClosed = true; + // last chunk, no more data to read; enter trailer-reading mode + // bytesToDiscard = 2 below will consume the \r\n after "0" + this.readingTrailer = true; } if (dataSize > maximumAllowedPartSize) { this.log.error('chunk size too big', { dataSize }); diff --git a/lib/services.js b/lib/services.js index cbf8740170..f2980f1fc5 100644 --- a/lib/services.js +++ b/lib/services.js @@ -682,6 +682,16 @@ const services = { return metadata.getObjectMD(mpuBucketName, mpuOverviewKey, {}, log, (err, res) => { if (err) { + if (err.is && err.is.NoSuchKey) { + // The overview key no longer exists, meaning completeMultipartUpload + // already ran to completion and cleaned up the MPU bucket. + // This is a race condition: objectPutPart checked for old + // part locations after completeMultipartUpload deleted the overview. + // Returning true (complete in progress) prevents objectPutPart + // from deleting part data that may have already been committed + // as the final object. + return cb(null, true); + } log.error('error getting the overview object from mpu bucket', { error: err, method: 'services.isCompleteMPUInProgress', @@ -737,7 +747,7 @@ const services = { metadata.getObjectMD(mpuBucket.getName(), mpuOverviewKey, {}, log, (err, storedMetadata) => { if (err) { - if (err.is.NoSuchKey) { + if (err.is && err.is.NoSuchKey) { return cb(errors.NoSuchUpload); } log.error('error from metadata', { error: err }); diff --git a/package.json b/package.json index 18dbec49d8..eaee9d3d60 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "@azure/storage-blob": "^12.28.0", "@hapi/joi": "^17.1.1", "@smithy/node-http-handler": "^3.0.0", - "arsenal": "git+https://github.com/scality/Arsenal#8.3.5", + "arsenal": "git+https://github.com/scality/Arsenal#8.3.6", "async": "2.6.4", "bucketclient": "scality/bucketclient#8.2.7", "bufferutil": "^4.0.8", diff --git a/tests/functional/raw-node/test/checksumPutObjectUploadPart.js b/tests/functional/raw-node/test/checksumPutObjectUploadPart.js new file mode 100644 index 0000000000..cbb11e8b71 --- /dev/null +++ b/tests/functional/raw-node/test/checksumPutObjectUploadPart.js @@ -0,0 +1,608 @@ +const assert = require('assert'); +const crypto = require('crypto'); +const async = require('async'); + +const { makeS3Request } = require('../utils/makeRequest'); +const HttpRequestAuthV4 = require('../utils/HttpRequestAuthV4'); + +const bucket = 'checksumrejectionbucket'; +const objectKey = 'key'; +const objData = Buffer.alloc(1, 'a'); +const objDataSha256Hex = crypto.createHash('sha256').update(objData).digest('hex'); + +const authCredentials = { + accessKey: 'accessKey1', + secretKey: 'verySecretKey1', +}; + +const itSkipIfAWS = process.env.AWS_ON_AIR ? it.skip : it; + +const algos = [ + { name: 'crc32', wrongDigest: 'AAAAAA==' }, + { name: 'crc32c', wrongDigest: 'AAAAAA==' }, + { name: 'crc64nvme', wrongDigest: 'AAAAAAAAAAA=' }, + { name: 'sha1', wrongDigest: 'AAAAAAAAAAAAAAAAAAAAAAAAAAA=' }, + { name: 'sha256', wrongDigest: 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=' }, +]; + +// Build the raw chunked body for STREAMING-UNSIGNED-PAYLOAD-TRAILER +function buildTrailerBody(algoName, wrongDigest) { + const chunkSize = objData.length.toString(16); + return `${chunkSize}\r\n${objData.toString()}\r\n0\r\nx-amz-checksum-${algoName}:${wrongDigest}\r\n`; +} + +function doPutRequest(url, headers, body, callback) { + const req = new HttpRequestAuthV4( + url, + Object.assign({ method: 'PUT', headers }, authCredentials), + res => { + let data = ''; + res.on('data', chunk => { data += chunk; }); + res.on('end', () => callback(null, { statusCode: res.statusCode, body: data })); + } + ); + req.on('error', callback); + req.write(body); + req.end(); +} + +const protocols = [ + { + name: 'UNSIGNED-PAYLOAD', + buildHeaders: algo => ({ + 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', + 'content-length': objData.length, + [`x-amz-checksum-${algo.name}`]: algo.wrongDigest, + }), + buildBody: () => objData, + }, + { + name: 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD', + buildHeaders: algo => ({ + // No x-amz-content-sha256: HttpRequestAuthV4 defaults to + // STREAMING-AWS4-HMAC-SHA256-PAYLOAD and handles chunk signing. + 'content-length': objData.length, + [`x-amz-checksum-${algo.name}`]: algo.wrongDigest, + }), + buildBody: () => objData, + }, + { + name: 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + buildHeaders: algo => { + const body = buildTrailerBody(algo.name, algo.wrongDigest); + return { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': `x-amz-checksum-${algo.name}`, + 'x-amz-decoded-content-length': objData.length, + 'content-length': body.length, + }; + }, + buildBody: algo => buildTrailerBody(algo.name, algo.wrongDigest), + }, + { + name: 'valid x-amz-content-sha256', + buildHeaders: algo => ({ + 'x-amz-content-sha256': objDataSha256Hex, + 'content-length': objData.length, + [`x-amz-checksum-${algo.name}`]: algo.wrongDigest, + }), + buildBody: () => objData, + }, +]; + +function assertBadDigest(err, res, done) { + assert.ifError(err); + assert.strictEqual(res.statusCode, 400, `expected 400, got ${res.statusCode}: ${res.body}`); + assert(res.body.includes('BadDigest'), `missing BadDigest in "${res.body}"`); + done(); +} + +// Constants for protocol scenario tests + +const trailerContent = Buffer.from('trailer content'); // 15 bytes, hex 'f' +const trailerContentSha256 = '4x74k2oA6j6knXzpDwogNkS6E3MM49tPpJMjfD+ES68='; +// md5("trailer content") in base64 — used in testS3PutTrailerWithContentMD5 +const trailerContentMd5B64 = crypto.createHash('md5').update(trailerContent).digest('base64'); + +const testContent2 = Buffer.from('test content'); +const testContent2Sha256Hex = crypto.createHash('sha256').update(testContent2).digest('hex'); +const testContent2Sha256B64 = crypto.createHash('sha256').update(testContent2).digest('base64'); + +// Build a STREAMING-UNSIGNED-PAYLOAD-TRAILER body with "trailer content" as the +// data and a sha256 trailer. Uses the '\n\r\n\r\n\r\n' ending that AWS SDK sends +// (TrailingChecksumTransform strips the trailing \n from the parsed trailer line). +function buildOkTrailerBody() { + const hexLen = trailerContent.length.toString(16); // 'f' + return `${hexLen}\r\ntrailer content\r\n0\r\nx-amz-checksum-sha256:${trailerContentSha256}\n\r\n\r\n\r\n`; +} + +// Assert that the response has the given HTTP status code and (optionally) +// that the body contains the expected error code string. +// Returns a (err, res, done) callback suitable for use with doPutRequest. +function assertStatus(expectedStatus, expectedCode, expectedMessage) { + return (err, res, done) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, expectedStatus, + `expected ${expectedStatus}, got ${res.statusCode}: ${res.body}`); + if (expectedCode) { + assert(res.body.includes(expectedCode), + `expected "${expectedCode}" in body: "${res.body}"`); + } + if (expectedMessage) { + assert(res.body.includes(expectedMessage), + `expected "${expectedMessage}" in body: "${res.body}"`); + } + done(); + }; +} + +const msgMalformedTrailer = 'The request contained trailing data that was not well-formed' + + ' or did not conform to our published schema.'; +const msgSdkMissingTrailer = 'x-amz-sdk-checksum-algorithm specified, but no corresponding' + + ' x-amz-checksum-* or x-amz-trailer headers were found.'; + +// Create the 24 common protocol-scenario tests for a given URL factory. +// urlFn() is called lazily at test runtime so that uploadId is available. +function makeScenarioTests(urlFn) { + itSkipIfAWS( + 'should return 200 for signed sha256 in x-amz-content-sha256, no x-amz-checksum header', + done => { + doPutRequest(urlFn(), { + 'x-amz-content-sha256': testContent2Sha256Hex, + 'content-length': testContent2.length, + }, testContent2, (err, res) => assertStatus(200)(err, res, done)); + }); + + itSkipIfAWS( + 'should return 200 for correct sha256 checksum with x-amz-sdk-checksum-algorithm', + done => { + doPutRequest(urlFn(), { + 'x-amz-content-sha256': testContent2Sha256Hex, + 'x-amz-sdk-checksum-algorithm': 'SHA256', + 'x-amz-checksum-sha256': testContent2Sha256B64, + 'content-length': testContent2.length, + }, testContent2, (err, res) => assertStatus(200)(err, res, done)); + }); + + itSkipIfAWS( + 'should return 400 BadDigest for wrong sha256 checksum with x-amz-sdk-checksum-algorithm', + done => { + doPutRequest(urlFn(), { + 'x-amz-content-sha256': testContent2Sha256Hex, + 'x-amz-sdk-checksum-algorithm': 'SHA256', + 'x-amz-checksum-sha256': 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=', + 'content-length': testContent2.length, + }, testContent2, (err, res) => assertStatus(400, 'BadDigest', + 'The SHA256 you specified did not match the calculated checksum.')(err, res, done)); + }); + + itSkipIfAWS( + 'should return 200 for UNSIGNED-PAYLOAD with correct sha256 checksum', + done => { + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', + 'x-amz-sdk-checksum-algorithm': 'SHA256', + 'x-amz-checksum-sha256': testContent2Sha256B64, + 'content-length': testContent2.length, + }, testContent2, (err, res) => assertStatus(200)(err, res, done)); + }); + + itSkipIfAWS( + 'should return 400 IncompleteBody for TRAILER with empty body', + done => { + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha256', + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': 0, + }, Buffer.alloc(0), (err, res) => assertStatus(400, 'IncompleteBody', + 'The request body terminated unexpectedly')(err, res, done)); + }); + + itSkipIfAWS( + 'should return 200 for TRAILER with correct sha256 checksum', + done => { + const body = buildOkTrailerBody(); + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha256', + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(200)(err, res, done)); + }); + + itSkipIfAWS( + 'should return 500 InternalError for wrong x-amz-decoded-content-length', + done => { + // Two chunks of 16 bytes each with a valid crc64nvme trailer. + const body = + '10\r\n0123456789abcdef\r\n' + + '10\r\n0123456789abcdef\r\n' + + '0\r\nx-amz-checksum-crc64nvme:skQv82y5rgE=\r\n\r\n\r\n'; + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-crc64nvme', + 'x-amz-decoded-content-length': 7, // wrong: actual content is 32 bytes + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(500, 'InternalError', + 'We encountered an internal error. Please try again.')(err, res, done)); + }); + + itSkipIfAWS( + 'should return 400 MalformedTrailerError when x-amz-trailer says sha1 but body trailer has sha256', + done => { + // Header announces sha1 but the actual trailer line carries sha256. + const body = + `f\r\ntrailer content\r\n0\r\nx-amz-checksum-sha256:${trailerContentSha256}\n\r\n\r\n\r\n`; + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha1', + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(400, 'MalformedTrailerError', + msgMalformedTrailer)(err, res, done)); + }); + + itSkipIfAWS( + 'should return 400 BadDigest for TRAILER with wrong sha256 checksum', + done => { + const wrongSha256 = 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA='; + const body = + `f\r\ntrailer content\r\n0\r\nx-amz-checksum-sha256:${wrongSha256}\n\r\n\r\n\r\n`; + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha256', + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(400, 'BadDigest', + 'The SHA256 you specified did not match the calculated checksum.')(err, res, done)); + }); + + itSkipIfAWS( + 'should return 400 InvalidRequest for x-amz-trailer + x-amz-checksum-crc32 header', + done => { + const body = buildOkTrailerBody(); + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha256', + 'x-amz-checksum-crc32': 'H+Yzmw==', // crc32("trailer content") + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(400, 'InvalidRequest', + 'Expecting a single x-amz-checksum- header')(err, res, done)); + }); + + itSkipIfAWS( + 'should return 400 MalformedTrailerError when no x-amz-trailer header but body has trailer', + done => { + const body = buildOkTrailerBody(); + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + // no x-amz-trailer header + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(400, 'MalformedTrailerError', + msgMalformedTrailer)(err, res, done)); + }); + + itSkipIfAWS( + 'should return 200 for TRAILER with explicit Content-Length', + done => { + const body = buildOkTrailerBody(); + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha256', + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(200)(err, res, done)); + }); + + itSkipIfAWS( + 'should return 200 for TRAILER with matching x-amz-sdk-checksum-algorithm:SHA256', + done => { + const body = buildOkTrailerBody(); + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha256', + 'x-amz-sdk-checksum-algorithm': 'SHA256', + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(200)(err, res, done)); + }); + + itSkipIfAWS( + 'should return 400 InvalidRequest when x-amz-sdk-checksum-algorithm:SHA1 but x-amz-trailer is sha256', + done => { + const body = buildOkTrailerBody(); + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha256', + 'x-amz-sdk-checksum-algorithm': 'SHA1', // mismatch + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(400, 'InvalidRequest', + 'Value for x-amz-sdk-checksum-algorithm header is invalid.')(err, res, done)); + }); + + itSkipIfAWS( + 'should return 400 InvalidRequest for x-amz-trailer:x-amz-checksum-sha3 (unknown algo)', + done => { + const body = buildOkTrailerBody(); + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha3', + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(400, 'InvalidRequest', + 'The value specified in the x-amz-trailer header is not supported')(err, res, done)); + }); + + itSkipIfAWS( + 'should return 400 InvalidRequest for x-amz-trailer with non-checksum value', + done => { + const body = buildOkTrailerBody(); + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'AAAAAAAAAAAAAAAAAAAAA', + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(400, 'InvalidRequest', + 'The value specified in the x-amz-trailer header is not supported')(err, res, done)); + }); + + itSkipIfAWS( + 'should return 400 InvalidRequest for trailer body with invalid base64 checksum value', + done => { + const body = 'f\r\ntrailer content\r\n0\r\nx-amz-checksum-sha256:BAD\n\r\n\r\n\r\n'; + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha256', + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(400, 'InvalidRequest', + 'Value for x-amz-checksum-sha256 trailing header is invalid.')(err, res, done)); + }); + + itSkipIfAWS( + 'should return 400 InvalidRequest for x-amz-sdk-checksum-algorithm without x-amz-trailer', + done => { + const body = buildOkTrailerBody(); + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + // no x-amz-trailer + 'x-amz-sdk-checksum-algorithm': 'SHA256', + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(400, 'InvalidRequest', + msgSdkMissingTrailer)(err, res, done)); + }); + + itSkipIfAWS( + 'should return 400 MalformedTrailerError when x-amz-trailer header present but body has no trailer', + done => { + // Body ends with "0\r\n\r\n" — empty trailer section, no checksum line. + const body = 'f\r\ntrailer content\r\n0\r\n\r\n'; + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha256', + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(400, 'MalformedTrailerError', + msgMalformedTrailer)(err, res, done)); + }); + + itSkipIfAWS( + 'should return 200 when no x-amz-trailer and no body trailer', + done => { + // No x-amz-trailer header; body just has chunked data with no trailer. + const body = 'f\r\ntrailer content\r\n0\r\n\r\n'; + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + // no x-amz-trailer + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(200)(err, res, done)); + }); + + itSkipIfAWS( + 'should return 200 and ignore data after final CRLF', + done => { + // No x-amz-trailer; after the terminating CRLF there is extra data. + // TrailingChecksumTransform discards everything after streamClosed=true. + const body = 'f\r\ntrailer content\r\n0\r\n\r\nRANDOM DATA IGNORED'; + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + // no x-amz-trailer + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(200)(err, res, done)); + }); + + itSkipIfAWS( + 'should return 200 for TRAILER with correct Content-MD5 header', + done => { + const body = buildOkTrailerBody(); + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha256', + 'x-amz-sdk-checksum-algorithm': 'SHA256', + 'content-md5': trailerContentMd5B64, + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(200)(err, res, done)); + }); + + itSkipIfAWS( + 'should return 200 for trailer line with whitespace around name and value', + done => { + // TrailingChecksumTransform trims both name and value, so whitespace is accepted. + const body = + `f\r\ntrailer content\r\n0\r\n x-amz-checksum-sha256 : ${trailerContentSha256} \n\r\n\r\n\r\n`; + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha256', + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(200)(err, res, done)); + }); +} + +describe('PutObject: bad checksum is rejected', () => { + before(done => { + makeS3Request({ method: 'PUT', authCredentials, bucket }, err => { + assert.ifError(err); + done(); + }); + }); + + after(done => { + // Delete the object key first (defensive: clears any state left by a previous run). + makeS3Request({ method: 'DELETE', authCredentials, bucket, objectKey }, () => { + makeS3Request({ method: 'DELETE', authCredentials, bucket }, err => { + assert.ifError(err); + done(); + }); + }); + }); + + for (const protocol of protocols) { + for (const algo of algos) { + itSkipIfAWS( + `${protocol.name} with wrong x-amz-checksum-${algo.name}: returns 400 BadDigest`, + done => { + const url = `http://localhost:8000/${bucket}/${objectKey}`; + doPutRequest(url, protocol.buildHeaders(algo), protocol.buildBody(algo), + (err, res) => assertBadDigest(err, res, done)); + } + ); + } + } +}); + +describe('UploadPart: bad checksum is rejected', () => { + let uploadId; + + before(done => { + async.series([ + next => makeS3Request({ method: 'PUT', authCredentials, bucket }, next), + next => makeS3Request({ + method: 'POST', + authCredentials, + bucket, + objectKey, + queryObj: { uploads: '' }, + }, (err, res) => { + if (err) { return next(err); } + const match = res.body.match(/([^<]+)<\/UploadId>/); + assert(match, `missing UploadId in response: ${res.body}`); + uploadId = match[1]; + return next(); + }), + ], err => { + assert.ifError(err); + done(); + }); + }); + + after(done => { + async.series([ + next => makeS3Request({ + method: 'DELETE', + authCredentials, + bucket, + objectKey, + queryObj: { uploadId }, + }, next), + // Delete the object key first (defensive: clears any state left by a previous run). + next => makeS3Request({ method: 'DELETE', authCredentials, bucket, objectKey }, () => next()), + next => makeS3Request({ method: 'DELETE', authCredentials, bucket }, next), + ], err => { + assert.ifError(err); + done(); + }); + }); + + for (const protocol of protocols) { + for (const algo of algos) { + itSkipIfAWS( + `${protocol.name} with wrong x-amz-checksum-${algo.name}: returns 400 BadDigest`, + done => { + const url = `http://localhost:8000/${bucket}/${objectKey}` + + `?partNumber=1&uploadId=${uploadId}`; + doPutRequest(url, protocol.buildHeaders(algo), protocol.buildBody(algo), + (err, res) => assertBadDigest(err, res, done)); + } + ); + } + } +}); + +describe('PutObject: trailer and checksum protocol scenarios', () => { + before(done => { + makeS3Request({ method: 'PUT', authCredentials, bucket }, err => { + assert.ifError(err); + done(); + }); + }); + + after(done => { + // Some scenario tests store objects; delete the object key before removing the bucket. + makeS3Request({ method: 'DELETE', authCredentials, bucket, objectKey }, () => { + makeS3Request({ method: 'DELETE', authCredentials, bucket }, err => { + assert.ifError(err); + done(); + }); + }); + }); + + makeScenarioTests(() => `http://localhost:8000/${bucket}/${objectKey}`); + +}); + +describe('UploadPart: trailer and checksum protocol scenarios', () => { + let uploadId2; + + before(done => { + async.series([ + next => makeS3Request({ method: 'PUT', authCredentials, bucket }, next), + next => makeS3Request({ + method: 'POST', + authCredentials, + bucket, + objectKey, + queryObj: { uploads: '' }, + }, (err, res) => { + if (err) { return next(err); } + const match = res.body.match(/([^<]+)<\/UploadId>/); + assert(match, `missing UploadId in response: ${res.body}`); + uploadId2 = match[1]; + return next(); + }), + ], err => { + assert.ifError(err); + done(); + }); + }); + + after(done => { + async.series([ + next => makeS3Request({ + method: 'DELETE', + authCredentials, + bucket, + objectKey, + queryObj: { uploadId: uploadId2 }, + }, next), + // Delete the object key first (defensive: clears any state left by a previous run). + next => makeS3Request({ method: 'DELETE', authCredentials, bucket, objectKey }, () => next()), + next => makeS3Request({ method: 'DELETE', authCredentials, bucket }, next), + ], err => { + assert.ifError(err); + done(); + }); + }); + + makeScenarioTests( + () => `http://localhost:8000/${bucket}/${objectKey}?partNumber=1&uploadId=${uploadId2}` + ); +}); diff --git a/tests/functional/raw-node/test/trailingChecksums.js b/tests/functional/raw-node/test/trailingChecksums.js index bad429c3c8..925d959a7f 100644 --- a/tests/functional/raw-node/test/trailingChecksums.js +++ b/tests/functional/raw-node/test/trailingChecksums.js @@ -6,10 +6,9 @@ const HttpRequestAuthV4 = require('../utils/HttpRequestAuthV4'); const bucket = 'testunsupportedchecksumsbucket'; const objectKey = 'key'; const objData = Buffer.alloc(1024, 'a'); -// note this is not the correct checksum in objDataWithTrailingChecksum const objDataWithTrailingChecksum = '10\r\n0123456789abcdef\r\n' + '10\r\n0123456789abcdef\r\n' + - '0\r\nx-amz-checksum-crc64nvme:YeIDuLa7tU0=\r\n'; + '0\r\nx-amz-checksum-crc64nvme:skQv82y5rgE=\r\n'; const objDataWithoutTrailingChecksum = '0123456789abcdef0123456789abcdef'; const config = require('../../config.json'); diff --git a/tests/unit/api/apiUtils/integrity/validateChecksums.js b/tests/unit/api/apiUtils/integrity/validateChecksums.js index a234dcfd60..d0fc9fec0a 100644 --- a/tests/unit/api/apiUtils/integrity/validateChecksums.js +++ b/tests/unit/api/apiUtils/integrity/validateChecksums.js @@ -2,8 +2,14 @@ const assert = require('assert'); const crypto = require('crypto'); const sinon = require('sinon'); -const { validateChecksumsNoChunking, ChecksumError, validateMethodChecksumNoChunking, checksumedMethods } = - require('../../../../../lib/api/apiUtils/integrity/validateChecksums'); +const { + validateChecksumsNoChunking, + ChecksumError, + validateMethodChecksumNoChunking, + checksumedMethods, + getChecksumDataFromHeaders, + arsenalErrorFromChecksumError, +} = require('../../../../../lib/api/apiUtils/integrity/validateChecksums'); const { errors: ArsenalErrors } = require('arsenal'); const { config } = require('../../../../../lib/Config'); @@ -453,3 +459,277 @@ describe('validateMethodChecksumNoChunking', () => { }); }); }); + +describe('getChecksumDataFromHeaders', () => { + // Valid-format digests (correct length and base64, content not verified by getChecksumDataFromHeaders) + const validDigests = { + crc32: 'AAAAAA==', // 8 chars + crc32c: 'AAAAAA==', // 8 chars + sha1: 'AAAAAAAAAAAAAAAAAAAAAAAAAAA=', // 28 chars + sha256: 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=', // 44 chars + crc64nvme: 'AAAAAAAAAAA=', // 12 chars + }; + + it('should return crc64nvme with isTrailer=false and expected=undefined when no headers', () => { + const result = getChecksumDataFromHeaders({}); + assert.deepStrictEqual(result, { algorithm: 'crc64nvme', isTrailer: false, expected: undefined }); + }); + + it('should return crc64nvme default when no checksum headers, no trailer, no sdk algo', () => { + const result = getChecksumDataFromHeaders({ 'content-type': 'application/octet-stream' }); + assert.deepStrictEqual(result, { algorithm: 'crc64nvme', isTrailer: false, expected: undefined }); + }); + + for (const [algo, digest] of Object.entries(validDigests)) { + it(`should return algorithm, isTrailer=false and expected for x-amz-checksum-${algo} with valid digest`, () => { + const result = getChecksumDataFromHeaders({ [`x-amz-checksum-${algo}`]: digest }); + assert.deepStrictEqual(result, { algorithm: algo, isTrailer: false, expected: digest }); + }); + } + + it('should return AlgoNotSupported error for x-amz-checksum-unknown-algo', () => { + const result = getChecksumDataFromHeaders({ 'x-amz-checksum-md4': 'AAAAAA==' }); + assert.strictEqual(result.error, ChecksumError.AlgoNotSupported); + assert.strictEqual(result.details.algorithm, 'md4'); + }); + + it('should return MalformedChecksum error for x-amz-checksum-crc32 with malformed digest (wrong length)', () => { + const result = getChecksumDataFromHeaders({ 'x-amz-checksum-crc32': 'AAAAA==' }); // 7 chars, crc32 needs 8 + assert.strictEqual(result.error, ChecksumError.MalformedChecksum); + assert.strictEqual(result.details.algorithm, 'crc32'); + }); + + it('should return MalformedChecksum error for x-amz-checksum-crc32 with malformed digest (invalid base64)', () => { + const result = getChecksumDataFromHeaders({ 'x-amz-checksum-crc32': '!!!!!!!!' }); + assert.strictEqual(result.error, ChecksumError.MalformedChecksum); + assert.strictEqual(result.details.algorithm, 'crc32'); + }); + + it('should return MultipleChecksumTypes error for two x-amz-checksum- headers', () => { + const result = getChecksumDataFromHeaders({ + 'x-amz-checksum-crc32': validDigests.crc32, + 'x-amz-checksum-sha256': validDigests.sha256, + }); + assert.strictEqual(result.error, ChecksumError.MultipleChecksumTypes); + }); + + it('should return MissingCorresponding when x-amz-sdk-checksum-algorithm has no x-amz-checksum- or x-amz-trailer', + () => { + const result = getChecksumDataFromHeaders({ 'x-amz-sdk-checksum-algorithm': 'crc32' }); + assert.strictEqual(result.error, ChecksumError.MissingCorresponding); + assert.strictEqual(result.details.expected, 'crc32'); + }); + + it('should return success for x-amz-checksum-crc32 with matching x-amz-sdk-checksum-algorithm CRC32', () => { + const result = getChecksumDataFromHeaders({ + 'x-amz-checksum-crc32': validDigests.crc32, + 'x-amz-sdk-checksum-algorithm': 'crc32', + }); + assert.deepStrictEqual(result, { algorithm: 'crc32', isTrailer: false, expected: validDigests.crc32 }); + }); + + it('should return AlgoNotSupportedSDK for x-amz-checksum-crc32 with mismatched x-amz-sdk-checksum-algorithm SHA256', + () => { + const result = getChecksumDataFromHeaders({ + 'x-amz-checksum-crc32': validDigests.crc32, + 'x-amz-sdk-checksum-algorithm': 'sha256', + }); + assert.strictEqual(result.error, ChecksumError.AlgoNotSupportedSDK); + assert.strictEqual(result.details.algorithm, 'sha256'); + }); + + it('should return AlgoNotSupportedSDK for x-amz-checksum-crc32 with non-string x-amz-sdk-checksum-algorithm', + () => { + const result = getChecksumDataFromHeaders({ + 'x-amz-checksum-crc32': validDigests.crc32, + 'x-amz-sdk-checksum-algorithm': 1234, + }); + assert.strictEqual(result.error, ChecksumError.AlgoNotSupportedSDK); + assert.strictEqual(result.details.algorithm, 1234); + }); + + it('should return AlgoNotSupportedSDK for x-amz-checksum-crc32 with unknown x-amz-sdk-checksum-algorithm', () => { + const result = getChecksumDataFromHeaders({ + 'x-amz-checksum-crc32': validDigests.crc32, + 'x-amz-sdk-checksum-algorithm': 'md4', + }); + assert.strictEqual(result.error, ChecksumError.AlgoNotSupportedSDK); + assert.strictEqual(result.details.algorithm, 'md4'); + }); + + it('should return isTrailer=true for x-amz-trailer: x-amz-checksum-crc32', () => { + const result = getChecksumDataFromHeaders({ 'x-amz-trailer': 'x-amz-checksum-crc32' }); + assert.deepStrictEqual(result, { algorithm: 'crc32', isTrailer: true, expected: undefined }); + }); + + it('should return isTrailer=true for x-amz-trailer: x-amz-checksum-crc64nvme', () => { + const result = getChecksumDataFromHeaders({ 'x-amz-trailer': 'x-amz-checksum-crc64nvme' }); + assert.deepStrictEqual(result, { algorithm: 'crc64nvme', isTrailer: true, expected: undefined }); + }); + + it('should return TrailerNotSupported for x-amz-trailer with unsupported value (not x-amz-checksum- prefix)', + () => { + const result = getChecksumDataFromHeaders({ 'x-amz-trailer': 'x-custom-header' }); + assert.strictEqual(result.error, ChecksumError.TrailerNotSupported); + assert.strictEqual(result.details.value, 'x-custom-header'); + }); + + it('should return TrailerNotSupported for x-amz-trailer: x-amz-checksum-unknown-algo', () => { + const result = getChecksumDataFromHeaders({ 'x-amz-trailer': 'x-amz-checksum-md4' }); + assert.strictEqual(result.error, ChecksumError.TrailerNotSupported); + assert.strictEqual(result.details.value, 'x-amz-checksum-md4'); + }); + + it('should return TrailerAndChecksum error for x-amz-trailer with also an x-amz-checksum- header', () => { + const result = getChecksumDataFromHeaders({ + 'x-amz-trailer': 'x-amz-checksum-crc32', + 'x-amz-checksum-crc32': validDigests.crc32, + }); + assert.strictEqual(result.error, ChecksumError.TrailerAndChecksum); + }); + + it('should return success for x-amz-trailer with matching x-amz-sdk-checksum-algorithm', () => { + const result = getChecksumDataFromHeaders({ + 'x-amz-trailer': 'x-amz-checksum-crc32', + 'x-amz-sdk-checksum-algorithm': 'crc32', + }); + assert.deepStrictEqual(result, { algorithm: 'crc32', isTrailer: true, expected: undefined }); + }); + + it('should return AlgoNotSupportedSDK for x-amz-trailer with mismatched x-amz-sdk-checksum-algorithm', () => { + const result = getChecksumDataFromHeaders({ + 'x-amz-trailer': 'x-amz-checksum-crc32', + 'x-amz-sdk-checksum-algorithm': 'sha256', + }); + assert.strictEqual(result.error, ChecksumError.AlgoNotSupportedSDK); + assert.strictEqual(result.details.algorithm, 'sha256'); + }); +}); + +describe('arsenalErrorFromChecksumError', () => { + it('should return null for MissingChecksum', () => { + const result = arsenalErrorFromChecksumError({ error: ChecksumError.MissingChecksum, details: null }); + assert.strictEqual(result, null); + }); + + it('should return BadDigest mentioning CRC32 for XAmzMismatch with crc32', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.XAmzMismatch, + details: { algorithm: 'crc32', calculated: 'a', expected: 'b' }, + }); + assert.strictEqual(result.message, 'BadDigest'); + assert.strictEqual(result.description, 'The CRC32 you specified did not match the calculated checksum.'); + }); + + it('should return BadDigest mentioning SHA256 for XAmzMismatch with sha256', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.XAmzMismatch, + details: { algorithm: 'sha256', calculated: 'a', expected: 'b' }, + }); + assert.strictEqual(result.message, 'BadDigest'); + assert.strictEqual(result.description, 'The SHA256 you specified did not match the calculated checksum.'); + }); + + it('should return InvalidRequest for AlgoNotSupported', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.AlgoNotSupported, + details: { algorithm: 'md4' }, + }); + assert.strictEqual(result.message, 'InvalidRequest'); + }); + + it('should return InvalidRequest for AlgoNotSupportedSDK', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.AlgoNotSupportedSDK, + details: { algorithm: 'md4' }, + }); + assert.strictEqual(result.message, 'InvalidRequest'); + }); + + it('should return InvalidRequest for MissingCorresponding', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.MissingCorresponding, + details: { expected: 'crc32' }, + }); + assert.strictEqual(result.message, 'InvalidRequest'); + }); + + it('should return InvalidRequest for MultipleChecksumTypes', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.MultipleChecksumTypes, + details: { algorithms: ['x-amz-checksum-crc32', 'x-amz-checksum-sha256'] }, + }); + assert.strictEqual(result.message, 'InvalidRequest'); + }); + + it('should return InvalidRequest mentioning crc32 for MalformedChecksum', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.MalformedChecksum, + details: { algorithm: 'crc32', expected: 'bad' }, + }); + assert.strictEqual(result.message, 'InvalidRequest'); + assert.strictEqual(result.description, 'Value for x-amz-checksum-crc32 header is invalid.'); + }); + + it('should return InvalidDigest for MD5Invalid', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.MD5Invalid, + details: { expected: 'bad' }, + }); + assert.deepStrictEqual(result, ArsenalErrors.InvalidDigest); + }); + + it('should return MalformedTrailerError for TrailerAlgoMismatch', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.TrailerAlgoMismatch, + details: { algorithm: 'crc32' }, + }); + assert.deepStrictEqual(result, ArsenalErrors.MalformedTrailerError); + }); + + it('should return MalformedTrailerError for TrailerMissing', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.TrailerMissing, + details: { expectedTrailer: 'x-amz-checksum-crc32' }, + }); + assert.deepStrictEqual(result, ArsenalErrors.MalformedTrailerError); + }); + + it('should return MalformedTrailerError for TrailerUnexpected', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.TrailerUnexpected, + details: { name: 'x-amz-checksum-crc32', val: 'AAAAAA==' }, + }); + assert.deepStrictEqual(result, ArsenalErrors.MalformedTrailerError); + }); + + it('should return InvalidRequest mentioning the algorithm for TrailerChecksumMalformed', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.TrailerChecksumMalformed, + details: { algorithm: 'sha256', expected: 'bad' }, + }); + assert.strictEqual(result.message, 'InvalidRequest'); + assert.strictEqual(result.description, 'Value for x-amz-checksum-sha256 trailing header is invalid.'); + }); + + it('should return InvalidRequest for TrailerAndChecksum', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.TrailerAndChecksum, + details: { trailer: 'x-amz-checksum-crc32', checksum: ['x-amz-checksum-crc32'] }, + }); + assert.strictEqual(result.message, 'InvalidRequest'); + }); + + it('should return InvalidRequest for TrailerNotSupported', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.TrailerNotSupported, + details: { value: 'x-custom-header' }, + }); + assert.strictEqual(result.message, 'InvalidRequest'); + }); + + it('should return BadDigest for unknown error type (default)', () => { + const result = arsenalErrorFromChecksumError({ error: 'SomeUnknownError', details: null }); + assert.deepStrictEqual(result, ArsenalErrors.BadDigest); + }); +}); diff --git a/tests/unit/api/apiUtils/object/prepareStream.js b/tests/unit/api/apiUtils/object/prepareStream.js new file mode 100644 index 0000000000..ff54da33b9 --- /dev/null +++ b/tests/unit/api/apiUtils/object/prepareStream.js @@ -0,0 +1,202 @@ +const assert = require('assert'); +const { errors } = require('arsenal'); + +const { prepareStream } = require('../../../../../lib/api/apiUtils/object/prepareStream'); +const ChecksumTransform = require('../../../../../lib/auth/streamingV4/ChecksumTransform'); +const { DummyRequestLogger } = require('../../../helpers'); +const DummyRequest = require('../../../DummyRequest'); + +const log = new DummyRequestLogger(); + +function makeRequest(headers, body) { + return new DummyRequest({ headers }, body != null ? Buffer.from(body) : undefined); +} + +const mockV4Params = { + accessKey: 'AKIAIOSFODNN7EXAMPLE', + signatureFromRequest: 'abc123', + region: 'us-east-1', + scopeDate: '20210101', + timestamp: '20210101T000000Z', + credentialScope: '20210101/us-east-1/s3/aws4_request', +}; + +describe('prepareStream', () => { + describe('return value shape', () => { + it('should return { error: null, stream: ChecksumTransform } for UNSIGNED-PAYLOAD', () => { + const request = makeRequest({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + const result = prepareStream(request, null, log, () => {}); + assert.strictEqual(result.error, null); + assert(result.stream instanceof ChecksumTransform); + }); + + it('should return { error: InvalidRequest, stream: null } on invalid checksum headers', () => { + const request = makeRequest({ + 'x-amz-checksum-crc32': 'AAAAAA==', + 'x-amz-checksum-sha256': 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=', + }); + const result = prepareStream(request, null, log, () => {}); + assert.strictEqual(result.error.message, 'InvalidRequest'); + assert.strictEqual(result.stream, null); + }); + + it('should return { error: BadRequest, stream: null } for unsupported x-amz-content-sha256', () => { + const request = makeRequest({ + 'x-amz-content-sha256': 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER', + }); + const result = prepareStream(request, null, log, () => {}); + assert.strictEqual(result.error.message, 'BadRequest'); + assert.strictEqual(result.stream, null); + }); + }); + + describe('STREAMING-AWS4-HMAC-SHA256-PAYLOAD', () => { + it('should return ChecksumTransform as final stream with valid streamingV4Params', () => { + const request = makeRequest({ 'x-amz-content-sha256': 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD' }); + const result = prepareStream(request, mockV4Params, log, () => {}); + assert.strictEqual(result.error, null); + assert(result.stream instanceof ChecksumTransform); + }); + + it('should use crc64nvme by default with valid streamingV4Params', () => { + const request = makeRequest({ 'x-amz-content-sha256': 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD' }); + const result = prepareStream(request, mockV4Params, log, () => {}); + assert.strictEqual(result.stream.algoName, 'crc64nvme'); + }); + + it('should use crc32c algorithm when x-amz-checksum-crc32c header is present', () => { + const request = makeRequest({ + 'x-amz-content-sha256': 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD', + 'x-amz-checksum-crc32c': 'AAAAAA==', + }); + const result = prepareStream(request, mockV4Params, log, () => {}); + assert.strictEqual(result.error, null); + assert.strictEqual(result.stream.algoName, 'crc32c'); + assert.strictEqual(result.stream.expectedDigest, 'AAAAAA=='); + }); + + it('should return InvalidArgument error with null streamingV4Params', () => { + const request = makeRequest({ 'x-amz-content-sha256': 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD' }); + const result = prepareStream(request, null, log, () => {}); + assert.deepStrictEqual(result.error, errors.InvalidArgument); + assert.strictEqual(result.stream, null); + }); + + it('should return InvalidArgument error with non-object streamingV4Params', () => { + const request = makeRequest({ 'x-amz-content-sha256': 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD' }); + const result = prepareStream(request, 'not-an-object', log, () => {}); + assert.deepStrictEqual(result.error, errors.InvalidArgument); + assert.strictEqual(result.stream, null); + }); + }); + + describe('STREAMING-UNSIGNED-PAYLOAD-TRAILER', () => { + it('should return ChecksumTransform with isTrailer=true', () => { + const request = makeRequest({ + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-crc32', + }); + const result = prepareStream(request, null, log, () => {}); + assert.strictEqual(result.error, null); + assert(result.stream instanceof ChecksumTransform); + assert.strictEqual(result.stream.isTrailer, true); + assert.strictEqual(result.stream.algoName, 'crc32'); + }); + + it('should call setExpectedChecksum on ChecksumTransform when trailer event fires', done => { + const body = '0\r\nx-amz-checksum-crc32:AAAAAA==\r\n'; + const request = makeRequest({ + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-crc32', + }, body); + const result = prepareStream(request, null, log, done); + result.stream.resume(); + result.stream.on('finish', () => { + assert.strictEqual(result.stream.trailerChecksumName, 'x-amz-checksum-crc32'); + assert.strictEqual(result.stream.trailerChecksumValue, 'AAAAAA=='); + done(); + }); + result.stream.on('error', done); + }); + + it('should call errCb when TrailingChecksumTransform emits an error', done => { + // malformed chunked data triggers an error in TrailingChecksumTransform + const request = makeRequest({ + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-crc32', + }, 'zz\r\n'); // invalid hex chunk size + prepareStream(request, null, log, err => { + assert.strictEqual(err.message, 'InvalidArgument'); + done(); + }); + }); + + it('should call errCb when ChecksumTransform emits an error', done => { + const request = makeRequest({ + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-crc32', + }); + const result = prepareStream(request, null, log, err => { + assert.deepStrictEqual(err, errors.InternalError); + done(); + }); + result.stream.emit('error', errors.InternalError); + }); + }); + + describe('UNSIGNED-PAYLOAD', () => { + it('should return ChecksumTransform as final stream', () => { + const request = makeRequest({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + const result = prepareStream(request, null, log, () => {}); + assert.strictEqual(result.error, null); + assert(result.stream instanceof ChecksumTransform); + }); + + it('should set algorithm and expected digest from headers on ChecksumTransform', () => { + const request = makeRequest({ + 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', + 'x-amz-checksum-crc32': 'AAAAAA==', + }); + const result = prepareStream(request, null, log, () => {}); + assert.strictEqual(result.stream.algoName, 'crc32'); + assert.strictEqual(result.stream.expectedDigest, 'AAAAAA=='); + }); + + it('should call errCb when ChecksumTransform emits an error', done => { + const request = makeRequest({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + const result = prepareStream(request, null, log, err => { + assert.deepStrictEqual(err, errors.InternalError); + done(); + }); + result.stream.emit('error', errors.InternalError); + }); + }); + + describe('default (no x-amz-content-sha256)', () => { + it('should return ChecksumTransform with crc64nvme algorithm when no x-amz-content-sha256 header', () => { + const request = makeRequest({}); + const result = prepareStream(request, null, log, () => {}); + assert.strictEqual(result.error, null); + assert(result.stream instanceof ChecksumTransform); + assert.strictEqual(result.stream.algoName, 'crc64nvme'); + }); + + it('should return BadRequest for unsupported x-amz-content-sha256 value', () => { + const request = makeRequest({ + 'x-amz-content-sha256': 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER', + }); + const result = prepareStream(request, null, log, () => {}); + assert.strictEqual(result.error.message, 'BadRequest'); + assert.strictEqual(result.stream, null); + }); + + it('should call errCb when ChecksumTransform emits an error', done => { + const request = makeRequest({}); + const result = prepareStream(request, null, log, err => { + assert.deepStrictEqual(err, errors.InternalError); + done(); + }); + result.stream.emit('error', errors.InternalError); + }); + }); +}); diff --git a/tests/unit/api/apiUtils/object/storeObject.js b/tests/unit/api/apiUtils/object/storeObject.js new file mode 100644 index 0000000000..fab625f4b5 --- /dev/null +++ b/tests/unit/api/apiUtils/object/storeObject.js @@ -0,0 +1,302 @@ +const assert = require('assert'); +const sinon = require('sinon'); +const { errors } = require('arsenal'); + +const { dataStore } = require('../../../../../lib/api/apiUtils/object/storeObject'); +const dataWrapper = require('../../../../../lib/data/wrapper'); +const { DummyRequestLogger } = require('../../../helpers'); +const DummyRequest = require('../../../DummyRequest'); + +const log = new DummyRequestLogger(); + +const fakeDataRetrievalInfo = { key: 'test-key', dataStoreName: 'mem' }; + +function makeStream(headers = {}, body = '') { + return new DummyRequest({ headers }, body ? Buffer.from(body) : undefined); +} + +describe('dataStore', () => { + let putStub, batchDeleteStub; + + beforeEach(() => { + putStub = sinon.stub(dataWrapper.data, 'put'); + batchDeleteStub = sinon.stub(dataWrapper.data, 'batchDelete'); + }); + + afterEach(() => { + sinon.restore(); + }); + + // Stub data.put to drain the readable side and succeed once the stream finishes. + function putSucceeds(completedHash = null) { + putStub.callsFake((cipher, stream, size, ctx, backend, log2, cb) => { + stream.resume(); + stream.once('finish', () => cb(null, fakeDataRetrievalInfo, { completedHash })); + }); + } + + // Stub data.batchDelete to succeed. + function batchDeleteSucceeds() { + batchDeleteStub.callsFake((keys, a, b, log2, cb) => cb(null)); + } + + describe('normal behaviour', () => { + it('should call data.put with the stream returned by prepareStream', done => { + putSucceeds(); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + dataStore({}, null, request, 0, null, {}, log, err => { + assert.strictEqual(err, null); + assert(putStub.calledOnce); + done(); + }); + }); + + it('should call cb with (null, dataRetrievalInfo, completedHash) on success', done => { + putSucceeds('abc123'); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + dataStore({}, null, request, 0, null, {}, log, (err, dataInfo, completedHash) => { + assert.strictEqual(err, null); + assert.strictEqual(dataInfo, fakeDataRetrievalInfo); + assert.strictEqual(completedHash, 'abc123'); + done(); + }); + }); + + it('should call cb with the error from data.put when data.put fails', done => { + putStub.callsFake((cipher, stream, size, ctx, backend, log2, cb) => { + stream.resume(); + cb(errors.InternalError); + }); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + dataStore({}, null, request, 0, null, {}, log, err => { + assert.deepStrictEqual(err, errors.InternalError); + done(); + }); + }); + + it('should not delete stored data when data.put fails', done => { + putStub.callsFake((cipher, stream, size, ctx, backend, log2, cb) => { + stream.resume(); + cb(errors.InternalError); + }); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + dataStore({}, null, request, 0, null, {}, log, () => { + assert(batchDeleteStub.notCalled); + done(); + }); + }); + + it('should call cb with InternalError when data.put returns neither error nor dataRetrievalInfo', done => { + putStub.callsFake((cipher, stream, size, ctx, backend, log2, cb) => { + stream.resume(); + cb(null, null, null); + }); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + dataStore({}, null, request, 0, null, {}, log, err => { + assert.deepStrictEqual(err, errors.InternalError); + done(); + }); + }); + + it('should call cb with BadDigest and delete stored data when content-md5 does not match', done => { + batchDeleteSucceeds(); + putSucceeds('correct-md5'); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + request.contentMD5 = 'wrong-md5'; + dataStore({}, null, request, 0, null, {}, log, err => { + assert.deepStrictEqual(err, errors.BadDigest); + assert(batchDeleteStub.calledOnce); + done(); + }); + }); + + it('should call cb with (null, ...) and not delete stored data when content-md5 matches', done => { + putSucceeds('abc123'); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + request.contentMD5 = 'abc123'; + dataStore({}, null, request, 0, null, {}, log, err => { + assert.strictEqual(err, null); + assert(batchDeleteStub.notCalled); + done(); + }); + }); + + it('should call cb exactly once on success', done => { + putSucceeds(); + let cbCount = 0; + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + dataStore({}, null, request, 0, null, {}, log, () => { + cbCount++; + setImmediate(() => { + assert.strictEqual(cbCount, 1); + done(); + }); + }); + }); + }); + + describe('checksum behaviour', () => { + it('should call cb with error from prepareStream when stream headers are invalid', done => { + const request = makeStream({ + 'x-amz-checksum-crc32': 'AAAAAA==', + 'x-amz-checksum-sha256': 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=', + }); + dataStore({}, null, request, 0, null, {}, log, err => { + assert.strictEqual(err.message, 'InvalidRequest'); + done(); + }); + }); + + it('should not call data.put when prepareStream returns an error', done => { + const request = makeStream({ + 'x-amz-checksum-crc32': 'AAAAAA==', + 'x-amz-checksum-sha256': 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=', + }); + dataStore({}, null, request, 0, null, {}, log, () => { + assert(putStub.notCalled); + done(); + }); + }); + + it('should call cb with BadDigest and delete stored data when checksum validation fails', done => { + batchDeleteSucceeds(); + putSucceeds(); + // CRC32 of 'hello world' is not 0x00000000 (AAAAAA==) + const request = makeStream({ + 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', + 'x-amz-checksum-crc32': 'AAAAAA==', + }, 'hello world'); + dataStore({}, null, request, 0, null, {}, log, err => { + assert.strictEqual(err.message, 'BadDigest'); + assert(batchDeleteStub.calledOnce); + done(); + }); + }); + + it('should not delete stored data when checksum validation passes', done => { + putSucceeds(); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + dataStore({}, null, request, 0, null, {}, log, err => { + assert.strictEqual(err, null); + assert(batchDeleteStub.notCalled); + done(); + }); + }); + + it('should wait for finish before validating when checksumedStream is not yet writableFinished after data.put', + done => { + let capturedStream; + putStub.callsFake((cipher, stream, size, ctx, backend, log2, cb) => { + capturedStream = stream; + stream.resume(); + // Call cb synchronously — _flush uses Promise.resolve().then() so + // writableFinished is false here, exercising the finish-wait path. + cb(null, fakeDataRetrievalInfo, { completedHash: null }); + }); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + dataStore({}, null, request, 0, null, {}, log, err => { + assert.strictEqual(err, null); + assert(capturedStream.writableFinished); + done(); + }); + }); + + it('should delete stored data and call cb with the error when checksumedStream emits error after data.put', + done => { + batchDeleteSucceeds(); + let capturedStream; + putStub.callsFake((cipher, stream, size, ctx, backend, log2, cb) => { + capturedStream = stream; + // Do not resume — keeps writableFinished false, so onError listener is registered. + cb(null, fakeDataRetrievalInfo, { completedHash: null }); + }); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + dataStore({}, null, request, 0, null, {}, log, err => { + assert.deepStrictEqual(err, errors.InternalError); + assert(batchDeleteStub.calledOnce); + done(); + }); + // process.nextTick fires before Promise microtasks, so the error arrives + // before _flush resolves, ensuring onError fires rather than onFinish. + process.nextTick(() => capturedStream.emit('error', errors.InternalError)); + }); + + it('should call cb exactly once when finish fires (no double callback)', done => { + let cbCount = 0; + putStub.callsFake((cipher, stream, size, ctx, backend, log2, cb) => { + stream.resume(); + // Synchronous cb → writableFinished is false → finish-wait path. + cb(null, fakeDataRetrievalInfo, { completedHash: null }); + }); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + dataStore({}, null, request, 0, null, {}, log, () => { + cbCount++; + setImmediate(() => { + assert.strictEqual(cbCount, 1); + done(); + }); + }); + }); + + it('should call cb exactly once when stream errors after data.put (no double callback)', done => { + batchDeleteSucceeds(); + let cbCount = 0; + let capturedStream; + putStub.callsFake((cipher, stream, size, ctx, backend, log2, cb) => { + capturedStream = stream; + cb(null, fakeDataRetrievalInfo, { completedHash: null }); + }); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + dataStore({}, null, request, 0, null, {}, log, () => { + cbCount++; + setImmediate(() => { + assert.strictEqual(cbCount, 1); + done(); + }); + }); + process.nextTick(() => capturedStream.emit('error', errors.InternalError)); + }); + }); + + describe('batchDelete failure paths', () => { + it('should call cb with checksum error when validateChecksum fails and batchDelete also fails', done => { + batchDeleteStub.callsFake((keys, a, b, log2, cb) => cb(errors.InternalError)); + putSucceeds(); + const request = makeStream({ + 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', + 'x-amz-checksum-crc32': 'AAAAAA==', + }, 'hello world'); + dataStore({}, null, request, 0, null, {}, log, err => { + assert.strictEqual(err.message, 'BadDigest'); + done(); + }); + }); + + it('should call cb with BadDigest when content-md5 mismatches and batchDelete also fails', done => { + batchDeleteStub.callsFake((keys, a, b, log2, cb) => cb(errors.InternalError)); + putSucceeds('correct-md5'); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + request.contentMD5 = 'wrong-md5'; + dataStore({}, null, request, 0, null, {}, log, err => { + assert.deepStrictEqual(err, errors.BadDigest); + done(); + }); + }); + + it('should call cb with stream error when checksumedStream errors after data.put and batchDelete also fails', + done => { + batchDeleteStub.callsFake((keys, a, b, log2, cb) => cb(errors.BadRequest)); + let capturedStream; + putStub.callsFake((cipher, stream, size, ctx, backend, log2, cb) => { + capturedStream = stream; + cb(null, fakeDataRetrievalInfo, { completedHash: null }); + }); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + dataStore({}, null, request, 0, null, {}, log, err => { + assert.deepStrictEqual(err, errors.InternalError); + done(); + }); + process.nextTick(() => capturedStream.emit('error', errors.InternalError)); + }); + }); +}); diff --git a/tests/unit/auth/ChecksumTransform.js b/tests/unit/auth/ChecksumTransform.js new file mode 100644 index 0000000000..a4265d2b56 --- /dev/null +++ b/tests/unit/auth/ChecksumTransform.js @@ -0,0 +1,185 @@ +const assert = require('assert'); +const { errors } = require('arsenal'); + +const { algorithms, ChecksumError } = require('../../../lib/api/apiUtils/integrity/validateChecksums'); +const ChecksumTransform = require('../../../lib/auth/streamingV4/ChecksumTransform'); +const { DummyRequestLogger } = require('../helpers'); + +const log = new DummyRequestLogger(); +const testData = Buffer.from('hello world'); +const algos = ['crc32', 'crc32c', 'sha1', 'sha256', 'crc64nvme']; + +// Helper: pipe chunks into a ChecksumTransform, collect output, resolve on finish +function runTransform(stream, chunks) { + return new Promise((resolve, reject) => { + const output = []; + stream.on('data', chunk => output.push(chunk)); + stream.on('end', () => resolve(Buffer.concat(output))); + stream.on('error', reject); + for (const chunk of chunks) { + stream.write(chunk); + } + stream.end(); + }); +} + +// Helper: pipe data through and wait for finish without collecting output +function drainTransform(stream, chunks) { + return new Promise((resolve, reject) => { + stream.resume(); + stream.on('finish', resolve); + stream.on('error', reject); + for (const chunk of chunks) { + stream.write(chunk); + } + stream.end(); + }); +} + +describe('ChecksumTransform', () => { + describe('basic behaviour', () => { + let expectedDigests; + + before(async () => { + expectedDigests = {}; + for (const algo of algos) { + expectedDigests[algo] = await Promise.resolve(algorithms[algo].digest(testData)); + } + }); + + for (const algo of algos) { + it(`should pass data through unchanged [${algo}]`, async () => { + const stream = new ChecksumTransform(algo, undefined, false, log); + const output = await runTransform(stream, [testData]); + assert.deepStrictEqual(output, testData); + }); + + it(`should compute digest correctly after stream ends [${algo}]`, async () => { + const stream = new ChecksumTransform(algo, undefined, false, log); + await drainTransform(stream, [testData]); + assert.strictEqual(stream.digest, expectedDigests[algo]); + }); + + it(`should handle multi-chunk input: digest matches single-chunk equivalent [${algo}]`, async () => { + const half = Math.floor(testData.length / 2); + const stream = new ChecksumTransform(algo, undefined, false, log); + await drainTransform(stream, [testData.subarray(0, half), testData.subarray(half)]); + assert.strictEqual(stream.digest, expectedDigests[algo]); + }); + + it(`should handle Buffer and string chunks equally [${algo}]`, async () => { + const streamBuf = new ChecksumTransform(algo, undefined, false, log); + const streamStr = new ChecksumTransform(algo, undefined, false, log); + await drainTransform(streamBuf, [testData]); + await drainTransform(streamStr, [testData.toString()]); + assert.strictEqual(streamBuf.digest, streamStr.digest); + }); + } + + it('should emit error via stream error event if digestFromHash fails', done => { + const stream = new ChecksumTransform('crc32', undefined, false, log); + // Replace digestFromHash to return a rejected Promise + stream.algo = Object.assign({}, stream.algo, { + digestFromHash: () => Promise.reject(new Error('simulated digest failure')), + }); + stream.on('error', err => { + assert.deepStrictEqual(err, errors.InternalError); + done(); + }); + stream.write(testData); + stream.end(); + stream.resume(); + }); + }); + + describe('validateChecksum — non-trailer mode (isTrailer=false)', () => { + let crc32Digest; + + before(async () => { + crc32Digest = await Promise.resolve(algorithms.crc32.digest(testData)); + }); + + it('should return null when no expectedDigest and no trailer received', async () => { + const stream = new ChecksumTransform('crc32', undefined, false, log); + await drainTransform(stream, [testData]); + assert.strictEqual(stream.validateChecksum(), null); + }); + + it('should return null when expectedDigest matches computed digest', async () => { + const stream = new ChecksumTransform('crc32', crc32Digest, false, log); + await drainTransform(stream, [testData]); + assert.strictEqual(stream.validateChecksum(), null); + }); + + it('should return XAmzMismatch when expectedDigest does not match computed digest', async () => { + const stream = new ChecksumTransform('crc32', 'AAAAAA==', false, log); + await drainTransform(stream, [testData]); + const result = stream.validateChecksum(); + assert.strictEqual(result.error, ChecksumError.XAmzMismatch); + assert.strictEqual(result.details.algorithm, 'crc32'); + assert.strictEqual(result.details.calculated, crc32Digest); + assert.strictEqual(result.details.expected, 'AAAAAA=='); + }); + + it('should return TrailerUnexpected when setExpectedChecksum was called but isTrailer=false', async () => { + const stream = new ChecksumTransform('crc32', undefined, false, log); + stream.setExpectedChecksum('x-amz-checksum-crc32', crc32Digest); + await drainTransform(stream, [testData]); + const result = stream.validateChecksum(); + assert.strictEqual(result.error, ChecksumError.TrailerUnexpected); + }); + }); + + describe('validateChecksum — trailer mode (isTrailer=true)', () => { + let crc32Digest; + + before(async () => { + crc32Digest = await Promise.resolve(algorithms.crc32.digest(testData)); + }); + + it('should return TrailerMissing when setExpectedChecksum was never called', async () => { + const stream = new ChecksumTransform('crc32', undefined, true, log); + await drainTransform(stream, [testData]); + const result = stream.validateChecksum(); + assert.strictEqual(result.error, ChecksumError.TrailerMissing); + assert.strictEqual(result.details.expectedTrailer, 'x-amz-checksum-crc32'); + }); + + it('should return TrailerAlgoMismatch when trailer name does not match algo', async () => { + const stream = new ChecksumTransform('crc32', undefined, true, log); + stream.setExpectedChecksum('x-amz-checksum-sha256', 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA='); + await drainTransform(stream, [testData]); + const result = stream.validateChecksum(); + assert.strictEqual(result.error, ChecksumError.TrailerAlgoMismatch); + assert.strictEqual(result.details.algorithm, 'crc32'); + }); + + it('should return TrailerChecksumMalformed when trailer value is not a valid digest for the algo', + async () => { + const stream = new ChecksumTransform('crc32', undefined, true, log); + stream.setExpectedChecksum('x-amz-checksum-crc32', 'not-valid!'); + await drainTransform(stream, [testData]); + const result = stream.validateChecksum(); + assert.strictEqual(result.error, ChecksumError.TrailerChecksumMalformed); + assert.strictEqual(result.details.algorithm, 'crc32'); + }); + + it('should return XAmzMismatch when trailer value is valid but does not match computed digest', async () => { + const stream = new ChecksumTransform('crc32', undefined, true, log); + stream.setExpectedChecksum('x-amz-checksum-crc32', 'AAAAAA=='); + await drainTransform(stream, [testData]); + const result = stream.validateChecksum(); + assert.strictEqual(result.error, ChecksumError.XAmzMismatch); + assert.strictEqual(result.details.algorithm, 'crc32'); + assert.strictEqual(result.details.calculated, crc32Digest); + assert.strictEqual(result.details.expected, 'AAAAAA=='); + }); + + it('should return null when trailer name and value match computed digest', async () => { + const stream = new ChecksumTransform('crc32', undefined, true, log); + stream.setExpectedChecksum('x-amz-checksum-crc32', crc32Digest); + await drainTransform(stream, [testData]); + assert.strictEqual(stream.validateChecksum(), null); + }); + }); +}); diff --git a/tests/unit/auth/TrailingChecksumTransform.js b/tests/unit/auth/TrailingChecksumTransform.js index dd88b3719b..ae37ef179f 100644 --- a/tests/unit/auth/TrailingChecksumTransform.js +++ b/tests/unit/auth/TrailingChecksumTransform.js @@ -4,9 +4,39 @@ const async = require('async'); const { Readable } = require('stream'); const TrailingChecksumTransform = require('../../../lib/auth/streamingV4/trailingChecksumTransform'); -const { stripTrailingChecksumStream } = require('../../../lib/api/apiUtils/object/prepareStream'); const { DummyRequestLogger } = require('../helpers'); +// Helper: pipe input chunks through TrailingChecksumTransform, collect output and trailer events +function runTransform(inputChunks) { + const stream = new TrailingChecksumTransform(new DummyRequestLogger()); + return new Promise((resolve, reject) => { + const output = []; + const trailerEvents = []; + stream.on('data', chunk => output.push(Buffer.from(chunk))); + stream.on('trailer', (name, value) => trailerEvents.push({ name, value })); + stream.on('finish', () => resolve({ data: Buffer.concat(output), trailers: trailerEvents, stream })); + stream.on('error', reject); + for (const chunk of inputChunks) { + stream.write(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + stream.end(); + }); +} + +// Helper: expect a stream error from the given input chunks +function expectError(inputChunks) { + const stream = new TrailingChecksumTransform(new DummyRequestLogger()); + return new Promise((resolve, reject) => { + stream.on('error', resolve); + stream.on('finish', () => reject(new Error('expected error but stream finished cleanly'))); + for (const chunk of inputChunks) { + stream.write(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + stream.end(); + stream.resume(); + }); +} + const log = new DummyRequestLogger(); // note this is not the correct checksum in objDataWithTrailingChecksum @@ -174,30 +204,33 @@ describe('TrailingChecksumTransform class', () => { it('should propagate _flush error via errCb when stream closes without chunked encoding', done => { const incompleteData = '10\r\n01234\r6789abcd\r\n\r\n'; const source = new ChunkedReader([Buffer.from(incompleteData)]); - source.headers = { 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER' }; - const stream = stripTrailingChecksumStream(source, log, err => { - assert.deepStrictEqual(err, errors.InvalidArgument); + const stream = new TrailingChecksumTransform(log); + stream.on('error', err => { + assert.deepStrictEqual(err, errors.IncompleteBody); done(); }); + source.pipe(stream); stream.resume(); }); it('should propagate _transform error via errCb for invalid chunk size', done => { const badData = '500000000000\r\n'; const source = new ChunkedReader([Buffer.from(badData)]); - source.headers = { 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER' }; - const stream = stripTrailingChecksumStream(source, log, err => { + const stream = new TrailingChecksumTransform(log); + stream.on('error', err => { assert.deepStrictEqual(err, errors.InvalidArgument); done(); }); + source.pipe(stream); stream.resume(); }); - it('should return early if supplied with an out of specification chunk size', done => { + it('should return early if supplied with an out-of-specification chunk size', done => { const trailingChecksumTransform = new TrailingChecksumTransform(log); const chunks = [ Buffer.from('500000'), Buffer.from('000000\r\n'), + Buffer.alloc(1000000), Buffer.alloc(1000000), Buffer.alloc(1000000), @@ -227,3 +260,189 @@ describe('TrailingChecksumTransform class', () => { chunkedReader.pipe(trailingChecksumTransform); }); }); + +describe('TrailingChecksumTransform trailer parsing and emitting', () => { + describe('happy path', () => { + it('should forward data and emit trailer name and value for single chunk with data and trailer', async () => { + const input = '5\r\nhello\r\n0\r\nx-amz-checksum-crc32:AAAAAA==\r\n'; + const { data, trailers } = await runTransform([input]); + assert.strictEqual(data.toString(), 'hello'); + assert.strictEqual(trailers.length, 1); + assert.strictEqual(trailers[0].name, 'x-amz-checksum-crc32'); + assert.strictEqual(trailers[0].value, 'AAAAAA=='); + }); + + it('should forward all data and emit trailer once for multiple data chunks followed by trailer', async () => { + const input = '5\r\nhello\r\n5\r\nworld\r\n0\r\nx-amz-checksum-sha256:AAAAAA==\r\n'; + const { data, trailers } = await runTransform([input]); + assert.strictEqual(data.toString(), 'helloworld'); + assert.strictEqual(trailers.length, 1); + assert.strictEqual(trailers[0].name, 'x-amz-checksum-sha256'); + }); + + it('should forward data chunk containing \\r\\n in payload correctly', async () => { + // 7 bytes: h e l \r \n l o + const input = '7\r\nhel\r\nlo\r\n0\r\nx-amz-checksum-crc32:AAAAAA==\r\n'; + const { data } = await runTransform([input]); + assert.strictEqual(data.toString(), 'hel\r\nlo'); + }); + + it('should trim name and value for trailer with whitespace around them', async () => { + const input = '0\r\n x-amz-checksum-crc32 : AAAAAA== \r\n'; + const { trailers } = await runTransform([input]); + assert.strictEqual(trailers.length, 1); + assert.strictEqual(trailers[0].name, 'x-amz-checksum-crc32'); + assert.strictEqual(trailers[0].value, 'AAAAAA=='); + }); + + it('should strip trailing \\n from parsed line for trailer terminated with \\n\\r\\n', async () => { + const input = '0\r\nx-amz-checksum-crc32:AAAAAA==\n\r\n'; + const { trailers } = await runTransform([input]); + assert.strictEqual(trailers.length, 1); + assert.strictEqual(trailers[0].name, 'x-amz-checksum-crc32'); + assert.strictEqual(trailers[0].value, 'AAAAAA=='); + }); + + it('should use only first colon as separator for trailer value containing a colon', async () => { + const input = '0\r\nx-amz-checksum-crc32:AA:BB==\r\n'; + const { trailers } = await runTransform([input]); + assert.strictEqual(trailers.length, 1); + assert.strictEqual(trailers[0].name, 'x-amz-checksum-crc32'); + assert.strictEqual(trailers[0].value, 'AA:BB=='); + }); + + it('should silently discard bytes after trailer \\r\\n', async () => { + const input = '5\r\nhello\r\n0\r\nx-amz-checksum-crc32:AAAAAA==\r\nextra bytes ignored'; + const { data, trailers } = await runTransform([input]); + assert.strictEqual(data.toString(), 'hello'); + assert.strictEqual(trailers.length, 1); + }); + }); + + describe('chunk boundary edge cases', () => { + it('should parse chunk size field correctly when split across two input chunks', async () => { + // size 'a' (hex) = 10 bytes; split the size field across two chunks + const c1 = 'a'; + const c2 = '\r\nAAAAAAAAAA\r\n0\r\nx-amz-checksum-crc32:AAAAAA==\r\n'; + const { data, trailers } = await runTransform([c1, c2]); + assert.strictEqual(data.toString(), 'AAAAAAAAAA'); + assert.strictEqual(trailers.length, 1); + assert.strictEqual(trailers[0].name, 'x-amz-checksum-crc32'); + }); + + it('should forward all data bytes when split across two input chunks', async () => { + // 5 bytes 'hello', split after 'hel' + const c1 = '5\r\nhel'; + const c2 = 'lo\r\n0\r\nx-amz-checksum-crc32:AAAAAA==\r\n'; + const { data, trailers } = await runTransform([c1, c2]); + assert.strictEqual(data.toString(), 'hello'); + assert.strictEqual(trailers.length, 1); + }); + + it('should parse correctly when \\r\\n delimiter after chunk size is split across two chunks', async () => { + // '5\r' in chunk1, '\nhello\r\n0\r\n...' in chunk2 + const c1 = '5\r'; + const c2 = '\nhello\r\n0\r\nx-amz-checksum-crc32:AAAAAA==\r\n'; + const { data, trailers } = await runTransform([c1, c2]); + assert.strictEqual(data.toString(), 'hello'); + assert.strictEqual(trailers.length, 1); + }); + + it('should emit trailer correctly when trailer line is split across two input chunks', async () => { + const c1 = '5\r\nhello\r\n0\r\nx-amz-checksum-'; + const c2 = 'crc32:AAAAAA==\r\n'; + const { data, trailers } = await runTransform([c1, c2]); + assert.strictEqual(data.toString(), 'hello'); + assert.strictEqual(trailers.length, 1); + assert.strictEqual(trailers[0].name, 'x-amz-checksum-crc32'); + assert.strictEqual(trailers[0].value, 'AAAAAA=='); + }); + + it('should emit trailer correctly when trailer \\r\\n is split across two input chunks', async () => { + const c1 = '5\r\nhello\r\n0\r\nx-amz-checksum-crc32:AAAAAA==\r'; + const c2 = '\n'; + const { data, trailers } = await runTransform([c1, c2]); + assert.strictEqual(data.toString(), 'hello'); + assert.strictEqual(trailers.length, 1); + assert.strictEqual(trailers[0].name, 'x-amz-checksum-crc32'); + assert.strictEqual(trailers[0].value, 'AAAAAA=='); + }); + }); + + describe('zero-size terminator and trailer', () => { + it('should emit no trailer event and close cleanly for empty trailer line (0\\r\\n\\r\\n)', async () => { + const input = '5\r\nhello\r\n0\r\n\r\n'; + const { data, trailers } = await runTransform([input]); + assert.strictEqual(data.toString(), 'hello'); + assert.strictEqual(trailers.length, 0); + }); + + it('should forward no data and emit trailer for zero data chunks (only terminator + trailer)', async () => { + const input = '0\r\nx-amz-checksum-crc32:AAAAAA==\r\n'; + const { data, trailers } = await runTransform([input]); + assert.strictEqual(data.length, 0); + assert.strictEqual(trailers.length, 1); + assert.strictEqual(trailers[0].name, 'x-amz-checksum-crc32'); + assert.strictEqual(trailers[0].value, 'AAAAAA=='); + }); + }); + + describe('_flush error cases', () => { + it('should return IncompleteBody error when stream ends mid-data (no zero-chunk)', async () => { + // 5 bytes declared but stream ends after only 3 + const err = await expectError(['5\r\nhel']); + assert.deepStrictEqual(err, errors.IncompleteBody); + }); + + it('should return IncompleteBody error when stream ends after zero-chunk with partial trailer content', + async () => { + // zero-chunk received, trailer starts but no \r\n terminator + const err = await expectError(['0\r\nx-amz-checksum-crc32:AAAAAA==']); + assert.deepStrictEqual(err, errors.IncompleteBody); + }); + + it('should return no error when stream ends after zero-chunk with no trailer content', async () => { + // only '0\r\n' — readingTrailer=true, trailerBuffer empty → no error + const { data, trailers } = await runTransform(['0\r\n']); + assert.strictEqual(data.length, 0); + assert.strictEqual(trailers.length, 0); + }); + }); + + describe('_transform error cases', () => { + it('should return InvalidArgument error for chunk size field larger than 10 bytes', async () => { + // 11 hex digits — exceeds the 10-byte field size limit + const err = await expectError(['12345678901\r\n']); + assert.deepStrictEqual(err, errors.InvalidArgument); + }); + + it('should return InvalidArgument error when chunk size is not valid hex', async () => { + // 2 chars, short enough to pass size check, but not valid hex + const err = await expectError(['zz\r\n']); + assert.deepStrictEqual(err, errors.InvalidArgument); + }); + + it('should return EntityTooLarge error when chunk size exceeds maximumAllowedPartSize', async () => { + // 0x200000000 = 8589934592 > maximumAllowedPartSize (5GB = 0x140000000) + const err = await expectError(['200000000\r\n']); + assert.deepStrictEqual(err, errors.EntityTooLarge); + }); + + it('should return MalformedTrailerError for trailer line longer than 1024 bytes', async () => { + // send zero-chunk then a trailer line > 1024 bytes with no \r\n + const longTrailer = 'x'.repeat(1025); + const err = await expectError([`0\r\n${longTrailer}`]); + assert.deepStrictEqual(err, errors.MalformedTrailerError); + }); + + it('should return IncompleteBody error for trailer line missing colon', async () => { + const err = await expectError(['0\r\nnocolon\r\n']); + assert.deepStrictEqual(err, errors.IncompleteBody); + }); + + it('should return IncompleteBody error for trailer line with colon at position 0 (empty name)', async () => { + const err = await expectError(['0\r\n:value\r\n']); + assert.deepStrictEqual(err, errors.IncompleteBody); + }); + }); +}); diff --git a/yarn.lock b/yarn.lock index f8ee0a1ff1..82bf9887f0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5766,9 +5766,9 @@ arraybuffer.prototype.slice@^1.0.4: optionalDependencies: ioctl "^2.0.2" -"arsenal@git+https://github.com/scality/Arsenal#8.3.5": - version "8.3.5" - resolved "git+https://github.com/scality/Arsenal#720a2a8e30998a8b08839bcbb73f059968e303b2" +"arsenal@git+https://github.com/scality/Arsenal#8.3.6": + version "8.3.6" + resolved "git+https://github.com/scality/Arsenal#21f6a54ac70f2f6c94b592760780b3ca3bab7a95" dependencies: "@aws-sdk/client-kms" "^3.975.0" "@aws-sdk/client-s3" "^3.975.0"