CLDSRV-863: Checksums for PutObject and UploadPart

lib/api/apiUtils/integrity/validateChecksums.js

@@ -37,6 +37,12 @@ const ChecksumError = Object.freeze({
     MultipleChecksumTypes: 'MultipleChecksumTypes',
     MissingCorresponding: 'MissingCorresponding',
     MalformedChecksum: 'MalformedChecksum',
+    TrailerAlgoMismatch: 'TrailerAlgoMismatch',
+    TrailerChecksumMalformed: 'TrailerChecksumMalformed',
+    TrailerMissing: 'TrailerMissing',
+    TrailerUnexpected: 'TrailerUnexpected',
+    TrailerAndChecksum: 'TrailerAndChecksum',
+    TrailerNotSupported: 'TrailerNotSupported',
 });
 
 const base64Regex = /^[A-Za-z0-9+/]*={0,2}$/;
@@ -56,35 +62,51 @@ const algorithms = Object.freeze({
             const result = await crc.digest();
             return Buffer.from(result).toString('base64');
         },
+        digestFromHash: async hash => {
+            const result = await hash.digest();
+            return Buffer.from(result).toString('base64');
+        },
         isValidDigest: expected => typeof expected === 'string' && expected.length === 12 && base64Regex.test(expected),
+        createHash: () => new CrtCrc64Nvme()
     },
     crc32: {
         digest: data => {
             const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
             return uint32ToBase64(new Crc32().update(input).digest() >>> 0); // >>> 0 coerce number to uint32
         },
+        digestFromHash: hash => {
+            const result = hash.digest();
+            return uint32ToBase64(result >>> 0);
+        },
         isValidDigest: expected => typeof expected === 'string' && expected.length === 8 && base64Regex.test(expected),
+        createHash: () => new Crc32()
     },
     crc32c: {
         digest: data => {
             const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
             return uint32ToBase64(new Crc32c().update(input).digest() >>> 0); // >>> 0 coerce number to uint32
         },
+        digestFromHash: hash => uint32ToBase64(hash.digest() >>> 0),
         isValidDigest: expected => typeof expected === 'string' && expected.length === 8 && base64Regex.test(expected),
+        createHash: () => new Crc32c()
     },
     sha1: {
         digest: data => {
             const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
             return crypto.createHash('sha1').update(input).digest('base64');
         },
+        digestFromHash: hash => hash.digest('base64'),
         isValidDigest: expected => typeof expected === 'string' && expected.length === 28 && base64Regex.test(expected),
+        createHash: () => crypto.createHash('sha1')
     },
     sha256: {
         digest: data => {
             const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
             return crypto.createHash('sha256').update(input).digest('base64');
         },
+        digestFromHash: hash => hash.digest('base64'),
         isValidDigest: expected => typeof expected === 'string' && expected.length === 44 && base64Regex.test(expected),
+        createHash: () => crypto.createHash('sha256')
     }
 });
 
@@ -132,7 +154,7 @@ async function validateXAmzChecksums(headers, body) {
             return { error: ChecksumError.AlgoNotSupportedSDK, details: { algorithm: sdkAlgo } };
         }
 
-        // If AWS there is a mismatch, AWS returns the same error as if the algo was invalid.
+        // If there is a mismatch, AWS returns the same error as if the algo was invalid.
         if (sdkLowerAlgo !== algo) {
             return { error: ChecksumError.AlgoNotSupportedSDK, details: { algorithm: sdkAlgo } };
         }
@@ -141,6 +163,94 @@ async function validateXAmzChecksums(headers, body) {
     return null;
 }
 
+function getChecksumDataFromHeaders(headers) {
+    const checkSdk = algo => {
+        if (!('x-amz-sdk-checksum-algorithm' in headers)) {
+            return null;
+        }
+
+        const sdkAlgo = headers['x-amz-sdk-checksum-algorithm'];
+        if (typeof sdkAlgo !== 'string') {
+            return { error: ChecksumError.AlgoNotSupportedSDK, details: { algorithm: sdkAlgo } };
+        }
+
+        const sdkLowerAlgo = sdkAlgo.toLowerCase();
+        if (!(sdkLowerAlgo in algorithms)) {
+            return { error: ChecksumError.AlgoNotSupportedSDK, details: { algorithm: sdkAlgo } };
+        }
+
+        // If there is a mismatch, AWS returns the same error as if the algo was invalid.
+        if (sdkLowerAlgo !== algo) {
+            return { error: ChecksumError.AlgoNotSupportedSDK, details: { algorithm: sdkAlgo } };
+        }
+
+        return null;
+    };
+
+    const checksumHeaders = Object.keys(headers).filter(header => header.startsWith('x-amz-checksum-'));
+    const xAmzChecksumCnt = checksumHeaders.length;
+    if (xAmzChecksumCnt > 1) {
+        return { error: ChecksumError.MultipleChecksumTypes, details: { algorithms: checksumHeaders } };
+    }
+
+    if (xAmzChecksumCnt === 0 && !('x-amz-trailer' in headers) && 'x-amz-sdk-checksum-algorithm' in headers) {
+        return {
+            error: ChecksumError.MissingCorresponding,
+            details: { expected: headers['x-amz-sdk-checksum-algorithm'] }
+        };
+    }
+
+    if ('x-amz-trailer' in headers) {
+        if (xAmzChecksumCnt !== 0) {
+            return {
+                error: ChecksumError.TrailerAndChecksum,
+                details: { trailer: headers['x-amz-trailer'], checksum: checksumHeaders },
+            };
+        }
+
+        const trailer = headers['x-amz-trailer'];
+        if (!trailer.startsWith('x-amz-checksum-')) {
+            return { error: ChecksumError.TrailerNotSupported, details: { value: trailer } };
+        }
+
+        const trailerAlgo = trailer.slice('x-amz-checksum-'.length);
+        if (!(trailerAlgo in algorithms)) {
+            return { error: ChecksumError.TrailerNotSupported, details: { value: trailer } };
+        }
+
+        const err = checkSdk(trailerAlgo);
+        if (err) {
+            return err;
+        }
+
+        return { algorithm: trailerAlgo, isTrailer: true, expected: undefined };
+    }
+
+    if (xAmzChecksumCnt === 0) {
+        // There was no x-amz-checksum- or x-amz-trailer return crc64nvme.
+        // The calculated crc64nvme will be stored in the object metadata.
+        return { algorithm: 'crc64nvme', isTrailer: false, expected: undefined };
+    }
+
+    // No x-amz-sdk-checksum-algorithm we expect one x-amz-checksum-[crc64nvme, crc32, crc32C, sha1, sha256].
+    const algo = checksumHeaders[0].slice('x-amz-checksum-'.length);
+    if (!(algo in algorithms)) {
+        return { error: ChecksumError.AlgoNotSupported, details: { algorithm: algo } };
+    }
+
+    const expected = headers[`x-amz-checksum-${algo}`];
+    if (!algorithms[algo].isValidDigest(expected)) {
+        return { error: ChecksumError.MalformedChecksum, details: { algorithm: algo, expected } };
+    }
+
+    const err = checkSdk(algo);
+    if (err) {
+        return err;
+    }
+
+    return { algorithm: algo, isTrailer: false, expected };
+}
+
 /**
  * validateChecksumsNoChunking - Validate the checksums of a request.
  * @param {object} headers - http headers
@@ -183,16 +293,7 @@ async function validateChecksumsNoChunking(headers, body) {
     return err;
 }
 
-async function defaultValidationFunc(request, body, log) {
-    const err = await validateChecksumsNoChunking(request.headers, body);
-    if (!err) {
-        return null;
-    }
-
-    if (err.error !== ChecksumError.MissingChecksum) {
-        log.debug('failed checksum validation', { method: request.apiMethod }, err);
-    }
-
+function arsenalErrorFromChecksumError(err) {
     switch (err.error) {
         case ChecksumError.MissingChecksum:
             return null;
@@ -225,11 +326,40 @@ async function defaultValidationFunc(request, body, log) {
             );
         case ChecksumError.MD5Invalid:
             return ArsenalErrors.InvalidDigest;
+        case ChecksumError.TrailerAlgoMismatch:
+            return ArsenalErrors.MalformedTrailerError;
+        case ChecksumError.TrailerMissing:
+            return ArsenalErrors.MalformedTrailerError;
+        case ChecksumError.TrailerUnexpected:
+            return ArsenalErrors.MalformedTrailerError;
+        case ChecksumError.TrailerChecksumMalformed:
+            return ArsenalErrors.InvalidRequest.customizeDescription(
+                `Value for x-amz-checksum-${err.details.algorithm} trailing header is invalid.`
+            );
+        case ChecksumError.TrailerAndChecksum:
+            return ArsenalErrors.InvalidRequest.customizeDescription('Expecting a single x-amz-checksum- header');
+        case ChecksumError.TrailerNotSupported:
+            return ArsenalErrors.InvalidRequest.customizeDescription(
+                'The value specified in the x-amz-trailer header is not supported'
+            );
         default:
             return ArsenalErrors.BadDigest;
     }
 }
 
+async function defaultValidationFunc(request, body, log) {
+    const err = await validateChecksumsNoChunking(request.headers, body);
+    if (!err) {
+        return null;
+    }
+
+    if (err.error !== ChecksumError.MissingChecksum) {
+        log.debug('failed checksum validation', { method: request.apiMethod }, err);
+    }
+
+    return arsenalErrorFromChecksumError(err);
+}
+
 /**
  * validateMethodChecksumsNoChunking - Validate the checksums of a request.
  * @param {object} request - http request
@@ -253,5 +383,8 @@ module.exports = {
     ChecksumError,
     validateChecksumsNoChunking,
     validateMethodChecksumNoChunking,
+    getChecksumDataFromHeaders,
+    arsenalErrorFromChecksumError,
+    algorithms,
     checksumedMethods,
 };

lib/api/apiUtils/object/prepareStream.js

@@ -1,50 +1,114 @@
 const V4Transform = require('../../../auth/streamingV4/V4Transform');
 const TrailingChecksumTransform = require('../../../auth/streamingV4/trailingChecksumTransform');
+const ChecksumTransform = require('../../../auth/streamingV4/ChecksumTransform');
+const {
+    getChecksumDataFromHeaders,
+    arsenalErrorFromChecksumError,
+} = require('../../apiUtils/integrity/validateChecksums');
+const { errors, jsutil } = require('arsenal');
+const { unsupportedSignatureChecksums } = require('../../../../constants');
 
 /**
- * Prepares the stream if the chunks are sent in a v4 Auth request
- * @param {object} stream - stream containing the data
- * @param {object | null } streamingV4Params - if v4 auth, object containing
- * accessKey, signatureFromRequest, region, scopeDate, timestamp, and
- * credentialScope (to be used for streaming v4 auth if applicable)
- * @param {RequestLogger} log - the current request logger
- * @param {function} errCb - callback called if an error occurs
- * @return {object|null} - V4Transform object if v4 Auth request, or
- * the original stream, or null if the request has no V4 params but
- * the type of request requires them
+ * Prepares the request stream for data storage by wrapping it in the
+ * appropriate transform pipeline based on the x-amz-content-sha256 header.
+ * Always returns a ChecksumTransform as the final stream.
+ * If no checksum was sent by the client a CRC64NVME ChecksumTransform is returned.
+ *
+ * @param {object} request - incoming HTTP request with headers and body stream
+ * @param {object|null} streamingV4Params - v4 streaming auth params (accessKey,
+ * signatureFromRequest, region, scopeDate, timestamp, credentialScope), or
+ * null/undefined for non-v4 requests
+ * @param {RequestLogger} log - request logger
+ * @param {function} errCb - error callback invoked if a stream error occurs
+ * @return {{ error: Arsenal.Error|null, stream: ChecksumTransform|null }}
+ * error is set and stream is null if the request headers are invalid;
+ * otherwise error is null and stream is the ChecksumTransform to read from
  */
-function prepareStream(stream, streamingV4Params, log, errCb) {
-    if (stream.headers['x-amz-content-sha256'] ===
-        'STREAMING-AWS4-HMAC-SHA256-PAYLOAD') {
-        if (typeof streamingV4Params !== 'object') {
-            // this might happen if the user provided a valid V2
-            // Authentication header, while the chunked upload method
-            // requires V4: in such case we don't get any V4 params
-            // and we should return an error to the client.
-            return null;
-        }
-        const v4Transform = new V4Transform(streamingV4Params, log, errCb);
-        stream.pipe(v4Transform);
-        v4Transform.headers = stream.headers;
-        return v4Transform;
-    }
-    return stream;
-}
+function prepareStream(request, streamingV4Params, log, errCb) {
+    const xAmzContentSHA256 = request.headers['x-amz-content-sha256'];
 
-function stripTrailingChecksumStream(stream, log, errCb) {
-    // don't do anything if we are not in the correct integrity check mode
-    if (stream.headers['x-amz-content-sha256'] !== 'STREAMING-UNSIGNED-PAYLOAD-TRAILER') {
-        return stream;
+    const checksumAlgo = getChecksumDataFromHeaders(request.headers);
+    if (checksumAlgo.error) {
+        log.debug('prepareStream invalid checksum headers', checksumAlgo);
+        return { error: arsenalErrorFromChecksumError(checksumAlgo), stream: null };
     }
 
-    const trailingChecksumTransform = new TrailingChecksumTransform(log);
-    trailingChecksumTransform.on('error', errCb);
-    stream.pipe(trailingChecksumTransform);
-    trailingChecksumTransform.headers = stream.headers;
-    return trailingChecksumTransform;
+    let stream = request;
+    switch (xAmzContentSHA256) {
+        case 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD': {
+            if (streamingV4Params === null || typeof streamingV4Params !== 'object') {
+                // this might happen if the user provided a valid V2
+                // Authentication header, while the chunked upload method
+                // requires V4: in such case we don't get any V4 params
+                // and we should return an error to the client.
+                log.error('missing v4 streaming params for chunked upload', {
+                    method: 'prepareStream',
+                    streamingV4ParamsType: typeof streamingV4Params,
+                    streamingV4Params,
+                });
+                return { error: errors.InvalidArgument, stream: null };
+            }
+            // Use a once-guard so that if both V4Transform and ChecksumTransform
+            // independently error, errCb is only called once.
+            const onStreamError = jsutil.once(errCb);
+            const v4Transform = new V4Transform(streamingV4Params, log, onStreamError);
+            request.pipe(v4Transform);
+            v4Transform.headers = request.headers;
+            stream = v4Transform;
+
+            const checksumedStream = new ChecksumTransform(
+                checksumAlgo.algorithm,
+                checksumAlgo.expected,
+                checksumAlgo.isTrailer,
+                log,
+            );
+            checksumedStream.on('error', onStreamError);
+            stream.pipe(checksumedStream);
+            return { error: null, stream: checksumedStream };
+        }
+        case 'STREAMING-UNSIGNED-PAYLOAD-TRAILER': {
+            // Use a once-guard so that auto-destroying both piped streams
+            // when one errors does not result in errCb being called twice.
+            const onStreamError = jsutil.once(errCb);
+            const trailingChecksumTransform = new TrailingChecksumTransform(log);
+            trailingChecksumTransform.on('error', onStreamError);
+            request.pipe(trailingChecksumTransform);
+            trailingChecksumTransform.headers = request.headers;
+            stream = trailingChecksumTransform;
+
+            const checksumedStream = new ChecksumTransform(
+                checksumAlgo.algorithm,
+                checksumAlgo.expected,
+                checksumAlgo.isTrailer,
+                log,
+            );
+            checksumedStream.on('error', onStreamError);
+            trailingChecksumTransform.on('trailer', (name, value) => {
+                checksumedStream.setExpectedChecksum(name, value);
+            });
+            stream.pipe(checksumedStream);
+            return { error: null, stream: checksumedStream };
+        }
+        case 'UNSIGNED-PAYLOAD': // Fallthrough
+        default: {
+            if (unsupportedSignatureChecksums.has(xAmzContentSHA256)) {
+                return {
+                    error: errors.BadRequest.customizeDescription(`${xAmzContentSHA256} is not supported`),
+                    stream: null,
+                };
+            }
+
+            const checksumedStream = new ChecksumTransform(
+                checksumAlgo.algorithm,
+                checksumAlgo.expected,
+                checksumAlgo.isTrailer,
+                log,
+            );
+            checksumedStream.on('error', errCb);
+            stream.pipe(checksumedStream);
+            return { error: null, stream: checksumedStream };
+        }
+    }
 }
 
-module.exports = {
-    prepareStream,
-    stripTrailingChecksumStream,
-};
+module.exports = { prepareStream };