Skip to content

CLDSRV-872: store checksum in object metadata #6113

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
leif-scality wants to merge 8 commits into from
+2,418 −101
Closed

CLDSRV-872: store checksum in object metadata #6113

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
4838bb0
CLDSRV-863: bump arsenal to 8.3.6
leif-scality Mar 12, 2026
464f12d
CLDSRV-863: handle NoSuchKey error in isCompleteMPUInProgress
leif-scality Mar 16, 2026
09435e2
CLDSRV-863: emit trailer in TrailingChecksumTransform
leif-scality Mar 16, 2026
7c7367d
CLDSRV-863: handle trailer headers and add new trailer errors
leif-scality Mar 16, 2026
588f37e
CLDSRV-863: add ChecksumTransform to calculate and verify stream chec…
leif-scality Mar 16, 2026
176ee41
CLDSRV-863: pipe requests to ChecksumTransform and validate checksum …
leif-scality Mar 16, 2026
888ea43
CLDSRV-863: PutObject/UploadPart checksum service tests
leif-scality Mar 16, 2026
85e28c8
CLDSRV-872: store checksum in metadata for objects uploaded with PutO…
leif-scality Mar 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
155 changes: 144 additions & 11 deletions lib/api/apiUtils/integrity/validateChecksums.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,12 @@ const ChecksumError = Object.freeze({
MultipleChecksumTypes: 'MultipleChecksumTypes',
MissingCorresponding: 'MissingCorresponding',
MalformedChecksum: 'MalformedChecksum',
TrailerAlgoMismatch: 'TrailerAlgoMismatch',
TrailerChecksumMalformed: 'TrailerChecksumMalformed',
TrailerMissing: 'TrailerMissing',
TrailerUnexpected: 'TrailerUnexpected',
TrailerAndChecksum: 'TrailerAndChecksum',
TrailerNotSupported: 'TrailerNotSupported',
});

const base64Regex = /^[A-Za-z0-9+/]*={0,2}$/;
Expand All @@ -56,35 +62,51 @@ const algorithms = Object.freeze({
const result = await crc.digest();
return Buffer.from(result).toString('base64');
},
digestFromHash: async hash => {
const result = await hash.digest();
return Buffer.from(result).toString('base64');
},
isValidDigest: expected => typeof expected === 'string' && expected.length === 12 && base64Regex.test(expected),
createHash: () => new CrtCrc64Nvme()
},
crc32: {
digest: data => {
const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
return uint32ToBase64(new Crc32().update(input).digest() >>> 0); // >>> 0 coerce number to uint32
},
digestFromHash: hash => {
const result = hash.digest();
return uint32ToBase64(result >>> 0);
},
isValidDigest: expected => typeof expected === 'string' && expected.length === 8 && base64Regex.test(expected),
createHash: () => new Crc32()
},
crc32c: {
digest: data => {
const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
return uint32ToBase64(new Crc32c().update(input).digest() >>> 0); // >>> 0 coerce number to uint32
},
digestFromHash: hash => uint32ToBase64(hash.digest() >>> 0),
isValidDigest: expected => typeof expected === 'string' && expected.length === 8 && base64Regex.test(expected),
createHash: () => new Crc32c()
},
sha1: {
digest: data => {
const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
return crypto.createHash('sha1').update(input).digest('base64');
},
digestFromHash: hash => hash.digest('base64'),
isValidDigest: expected => typeof expected === 'string' && expected.length === 28 && base64Regex.test(expected),
createHash: () => crypto.createHash('sha1')
},
sha256: {
digest: data => {
const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
return crypto.createHash('sha256').update(input).digest('base64');
},
digestFromHash: hash => hash.digest('base64'),
isValidDigest: expected => typeof expected === 'string' && expected.length === 44 && base64Regex.test(expected),
createHash: () => crypto.createHash('sha256')
}
});

Expand Down Expand Up @@ -132,7 +154,7 @@ async function validateXAmzChecksums(headers, body) {
return { error: ChecksumError.AlgoNotSupportedSDK, details: { algorithm: sdkAlgo } };
}

// If AWS there is a mismatch, AWS returns the same error as if the algo was invalid.
// If there is a mismatch, AWS returns the same error as if the algo was invalid.
if (sdkLowerAlgo !== algo) {
return { error: ChecksumError.AlgoNotSupportedSDK, details: { algorithm: sdkAlgo } };
}
Expand All @@ -141,6 +163,94 @@ async function validateXAmzChecksums(headers, body) {
return null;
}

function getChecksumDataFromHeaders(headers) {
const checkSdk = algo => {
if (!('x-amz-sdk-checksum-algorithm' in headers)) {
return null;
}

const sdkAlgo = headers['x-amz-sdk-checksum-algorithm'];
if (typeof sdkAlgo !== 'string') {
return { error: ChecksumError.AlgoNotSupportedSDK, details: { algorithm: sdkAlgo } };
}

const sdkLowerAlgo = sdkAlgo.toLowerCase();
if (!(sdkLowerAlgo in algorithms)) {
return { error: ChecksumError.AlgoNotSupportedSDK, details: { algorithm: sdkAlgo } };
}

// If there is a mismatch, AWS returns the same error as if the algo was invalid.
if (sdkLowerAlgo !== algo) {
return { error: ChecksumError.AlgoNotSupportedSDK, details: { algorithm: sdkAlgo } };
}

return null;
};

const checksumHeaders = Object.keys(headers).filter(header => header.startsWith('x-amz-checksum-'));
const xAmzChecksumCnt = checksumHeaders.length;
if (xAmzChecksumCnt > 1) {
return { error: ChecksumError.MultipleChecksumTypes, details: { algorithms: checksumHeaders } };
}

if (xAmzChecksumCnt === 0 && !('x-amz-trailer' in headers) && 'x-amz-sdk-checksum-algorithm' in headers) {
return {
error: ChecksumError.MissingCorresponding,
details: { expected: headers['x-amz-sdk-checksum-algorithm'] }
};
}

if ('x-amz-trailer' in headers) {
if (xAmzChecksumCnt !== 0) {
return {
error: ChecksumError.TrailerAndChecksum,
details: { trailer: headers['x-amz-trailer'], checksum: checksumHeaders },
};
}

const trailer = headers['x-amz-trailer'];
Copy link

@claude claude bot Mar 17, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In getChecksumDataFromHeaders, when no checksum headers and no trailer are present, this defaults to crc64nvme. This means every PutObject request (even those not requesting checksum validation) will now compute a CRC64NVME hash of the entire body. This is an intentional design choice (always store a checksum in metadata), but it is a performance change — every put now pays the cost of CRC64NVME computation. Worth documenting this behavioral change.

— Claude Code

if (!trailer.startsWith('x-amz-checksum-')) {
return { error: ChecksumError.TrailerNotSupported, details: { value: trailer } };
}

const trailerAlgo = trailer.slice('x-amz-checksum-'.length);
if (!(trailerAlgo in algorithms)) {
return { error: ChecksumError.TrailerNotSupported, details: { value: trailer } };
}

const err = checkSdk(trailerAlgo);
if (err) {
return err;
}

return { algorithm: trailerAlgo, isTrailer: true, expected: undefined };
}

if (xAmzChecksumCnt === 0) {
// There was no x-amz-checksum- or x-amz-trailer return crc64nvme.
// The calculated crc64nvme will be stored in the object metadata.
return { algorithm: 'crc64nvme', isTrailer: false, expected: undefined };
}

// No x-amz-sdk-checksum-algorithm we expect one x-amz-checksum-[crc64nvme, crc32, crc32C, sha1, sha256].
const algo = checksumHeaders[0].slice('x-amz-checksum-'.length);
if (!(algo in algorithms)) {
return { error: ChecksumError.AlgoNotSupported, details: { algorithm: algo } };
}

const expected = headers[`x-amz-checksum-${algo}`];
if (!algorithms[algo].isValidDigest(expected)) {
return { error: ChecksumError.MalformedChecksum, details: { algorithm: algo, expected } };
}

const err = checkSdk(algo);
if (err) {
return err;
}

return { algorithm: algo, isTrailer: false, expected };
}

/**
* validateChecksumsNoChunking - Validate the checksums of a request.
* @param {object} headers - http headers
Expand Down Expand Up @@ -183,16 +293,7 @@ async function validateChecksumsNoChunking(headers, body) {
return err;
}

async function defaultValidationFunc(request, body, log) {
const err = await validateChecksumsNoChunking(request.headers, body);
if (!err) {
return null;
}

if (err.error !== ChecksumError.MissingChecksum) {
log.debug('failed checksum validation', { method: request.apiMethod }, err);
}

function arsenalErrorFromChecksumError(err) {
switch (err.error) {
case ChecksumError.MissingChecksum:
return null;
Expand Down Expand Up @@ -225,11 +326,40 @@ async function defaultValidationFunc(request, body, log) {
);
case ChecksumError.MD5Invalid:
return ArsenalErrors.InvalidDigest;
case ChecksumError.TrailerAlgoMismatch:
return ArsenalErrors.MalformedTrailerError;
case ChecksumError.TrailerMissing:
return ArsenalErrors.MalformedTrailerError;
case ChecksumError.TrailerUnexpected:
return ArsenalErrors.MalformedTrailerError;
case ChecksumError.TrailerChecksumMalformed:
return ArsenalErrors.InvalidRequest.customizeDescription(
`Value for x-amz-checksum-${err.details.algorithm} trailing header is invalid.`
);
case ChecksumError.TrailerAndChecksum:
return ArsenalErrors.InvalidRequest.customizeDescription('Expecting a single x-amz-checksum- header');
case ChecksumError.TrailerNotSupported:
return ArsenalErrors.InvalidRequest.customizeDescription(
'The value specified in the x-amz-trailer header is not supported'
);
default:
return ArsenalErrors.BadDigest;
}
}

async function defaultValidationFunc(request, body, log) {
const err = await validateChecksumsNoChunking(request.headers, body);
if (!err) {
return null;
}

if (err.error !== ChecksumError.MissingChecksum) {
log.debug('failed checksum validation', { method: request.apiMethod }, err);
}

return arsenalErrorFromChecksumError(err);
}

/**
* validateMethodChecksumsNoChunking - Validate the checksums of a request.
* @param {object} request - http request
Expand All @@ -253,5 +383,8 @@ module.exports = {
ChecksumError,
validateChecksumsNoChunking,
validateMethodChecksumNoChunking,
getChecksumDataFromHeaders,
arsenalErrorFromChecksumError,
algorithms,
checksumedMethods,
};
7 changes: 4 additions & 3 deletions lib/api/apiUtils/object/createAndStoreObject.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
if (size === 0) {
if (!dontSkipBackend[locationType]) {
metadataStoreParams.contentMD5 = constants.emptyFileMd5;
return next(null, null, null);
return next(null, null, null, null);
}

// Handle mdOnlyHeader as a metadata only operation. If
Expand All @@ -243,14 +243,14 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
dataStoreVersionId: versionId,
dataStoreMD5: _md5,
};
return next(null, dataGetInfo, _md5);
return next(null, dataGetInfo, _md5, null);
}
}

return dataStore(objectKeyContext, cipherBundle, request, size,
streamingV4Params, backendInfo, log, next);
},
function processDataResult(dataGetInfo, calculatedHash, next) {
function processDataResult(dataGetInfo, calculatedHash, checksum, next) {
if (dataGetInfo === null || dataGetInfo === undefined) {
return next(null, null);
}
Expand All @@ -275,6 +275,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
dataGetInfoArr[0].size = mdOnlySize;
}
metadataStoreParams.contentMD5 = calculatedHash;
metadataStoreParams.checksum = checksum;
return next(null, dataGetInfoArr);
},
function getVersioningInfo(infoArr, next) {
Expand Down
Loading
Loading