Skip to content

CLDSRV-835: Fix flakiness tcp socket hang up #6115

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
bert-e merged 2 commits into from
Mar 18, 2026
+36 −26
Merged

CLDSRV-835: Fix flakiness tcp socket hang up #6115

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
eedcd6b
CLDSRV-835: Fix flakiness tcp socket hang up
BourgoisMickael Mar 18, 2026
c4c44f0
CLDSRV-835: Add newlines to log lines
BourgoisMickael Mar 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 36 additions & 26 deletions tests/functional/aws-node-sdk/test/object/copyPart.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -307,34 +307,44 @@ describe('Object Part Copy', () => {
})).then(initiateRes => {
sourceMpuId = initiateRes.UploadId;
}).catch(err => {
process.stdout.write(`Error initiating MPU ' +
'in MPU beforeEach: ${err}\n`);
process.stdout.write(`Error initiating MPU in MPU beforeEach: ${err}\n`);
throw err;
}).then(() => {
const partUploads = [];
// Concurrent uploads help trigger flakiness with "TimeoutError:
// socket hang up" due to a keep-alive race: the server closes
// an idle connection just as the client picks it from the pool.
const uploadWithRetry = (params, attempt = 0) =>
s3.send(new UploadPartCommand(params)).catch(err => {
if (attempt < 3) {
Copy link

@claude claude bot Mar 18, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The retry catches all errors, not just the transient socket hang up this PR targets. Consider narrowing the condition to only retry on TimeoutError or connection-related errors, so genuine failures (e.g. NoSuchUpload, auth errors) fail fast instead of retrying 3 times.

— Claude Code

Copy link
Contributor Author

@BourgoisMickael BourgoisMickael Mar 18, 2026 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's ok like that, other errors should not happen, 3 retries for non transient errors is not that bad.

process.stdout.write(`Retrying UploadPart ${params.PartNumber} `
+ `(attempt ${attempt + 1}/3): ${err}\n`);
return uploadWithRetry(params, attempt + 1);
}
throw err;
});
for (let i = 1; i < 10; i++) {
const partBuffHere = i % 2 ? partBuff : otherPartBuff;
const partHashHere = i % 2 ? partHash : otherPartHash;
partUploads.push(s3.send(new UploadPartCommand({
partUploads.push(uploadWithRetry({
Bucket: sourceBucketName,
Key: sourceMpuKey,
PartNumber: i,
UploadId: sourceMpuId,
Body: partBuffHere,
})));
}));
parts.push({
ETag: partHashHere,
PartNumber: i,
});
}
process.stdout.write('about to put parts');
process.stdout.write('about to put parts\n');
return Promise.all(partUploads);
}).catch(err => {
process.stdout.write(`Error putting parts in ' +
'MPU beforeEach: ${err}\n`);
process.stdout.write(`Error putting parts in MPU beforeEach: ${err}\n`);
throw err;
}).then(() => {
process.stdout.write('completing mpu');
process.stdout.write('completing mpu\n');
return s3.send(new CompleteMultipartUploadCommand({
Bucket: sourceBucketName,
Key: sourceMpuKey,
Expand All @@ -344,7 +354,7 @@ describe('Object Part Copy', () => {
},
}));
}).then(() => {
process.stdout.write('finished completing mpu');
process.stdout.write('finished completing mpu\n');
}).catch(err => {
process.stdout.write(`Error in MPU beforeEach: ${err}\n`);
throw err;
Expand All @@ -365,7 +375,7 @@ describe('Object Part Copy', () => {

it('should copy a part from a source bucket to a different ' +
'destination bucket', () => {
process.stdout.write('Entered first mpu test');
process.stdout.write('Entered first mpu test\n');
return s3.send(new UploadPartCopyCommand({
Bucket: destBucketName,
Key: destObjName,
Expand All @@ -381,7 +391,7 @@ describe('Object Part Copy', () => {

it('should copy two parts from a source bucket to a different ' +
'destination bucket and complete the MPU', () => {
process.stdout.write('Putting first part in MPU test');
process.stdout.write('Putting first part in MPU test\n');
return s3.send(new UploadPartCopyCommand({
Bucket: destBucketName,
Key: destObjName,
Expand All @@ -392,7 +402,7 @@ describe('Object Part Copy', () => {
assert.strictEqual(res.CopyPartResult.ETag, totalMpuObjectHash);
assert(res.CopyPartResult.LastModified);
}).then(() => {
process.stdout.write('Putting second part in MPU test');
process.stdout.write('Putting second part in MPU test\n');
return s3.send(new UploadPartCopyCommand({
Bucket: destBucketName,
Key: destObjName,
Expand All @@ -403,7 +413,7 @@ describe('Object Part Copy', () => {
assert.strictEqual(res.CopyPartResult.ETag, totalMpuObjectHash);
assert(res.CopyPartResult.LastModified);
}).then(() => {
process.stdout.write('Completing MPU');
process.stdout.write('Completing MPU\n');
return s3.send(new CompleteMultipartUploadCommand({
Bucket: destBucketName,
Key: destObjName,
Expand Down Expand Up @@ -431,7 +441,7 @@ describe('Object Part Copy', () => {
it('should copy two parts with range headers from a source ' +
'bucket to a different destination bucket and ' +
'complete the MPU', () => {
process.stdout.write('Putting first part in MPU range test');
process.stdout.write('Putting first part in MPU range test\n');
const part1ETag = '"b1e0d096c8f0670c5367d131e392b84a"';
const part2ETag = '"a2468d5c0ec2d4d5fc13b73beb63080a"';
// combined ETag returned by AWS (combination of part ETags
Expand All @@ -449,7 +459,7 @@ describe('Object Part Copy', () => {
assert.strictEqual(res.CopyPartResult.ETag, part1ETag);
assert(res.CopyPartResult.LastModified);
}).then(() => {
process.stdout.write('Putting second part in MPU test');
process.stdout.write('Putting second part in MPU test\n');
return s3.send(new UploadPartCopyCommand({
Bucket: destBucketName,
Key: destObjName,
Expand All @@ -461,7 +471,7 @@ describe('Object Part Copy', () => {
assert.strictEqual(res.CopyPartResult.ETag, part2ETag);
assert(res.CopyPartResult.LastModified);
}).then(() => {
process.stdout.write('Completing MPU');
process.stdout.write('Completing MPU\n');
return s3.send(new CompleteMultipartUploadCommand({
Bucket: destBucketName,
Key: destObjName,
Expand All @@ -477,7 +487,7 @@ describe('Object Part Copy', () => {
assert.strictEqual(res.Key, destObjName);
assert.strictEqual(res.ETag, finalCombinedETag);
}).then(() => {
process.stdout.write('Getting new object');
process.stdout.write('Getting new object\n');
return s3.send(new GetObjectCommand({
Bucket: destBucketName,
Key: destObjName,
Expand All @@ -496,7 +506,7 @@ describe('Object Part Copy', () => {
it('should overwrite an existing part by copying a part', () => {
// AWS response etag for this completed MPU
const finalObjETag = '"db77ebbae9e9f5a244a26b86193ad818-1"';
process.stdout.write('Putting first part in MPU test');
process.stdout.write('Putting first part in MPU test\n');
return s3.send(new UploadPartCopyCommand({
Bucket: destBucketName,
Key: destObjName,
Expand All @@ -507,7 +517,7 @@ describe('Object Part Copy', () => {
assert.strictEqual(res.CopyPartResult.ETag, totalMpuObjectHash);
assert(res.CopyPartResult.LastModified);
}).then(() => {
process.stdout.write('Overwriting first part in MPU test');
process.stdout.write('Overwriting first part in MPU test\n');
return s3.send(new UploadPartCopyCommand({
Bucket: destBucketName,
Key: destObjName,
Expand All @@ -518,7 +528,7 @@ describe('Object Part Copy', () => {
).then(res => {
assert.strictEqual(res.CopyPartResult.ETag, etag);
assert(res.CopyPartResult.LastModified);
process.stdout.write('Completing MPU');
process.stdout.write('Completing MPU\n');
return s3.send(new CompleteMultipartUploadCommand({
Bucket: destBucketName,
Key: destObjName,
Expand All @@ -535,7 +545,7 @@ describe('Object Part Copy', () => {
assert.strictEqual(res.ETag, finalObjETag);
}).then(() => {
process.stdout.write('Getting object put by MPU with ' +
'overwrite part');
'overwrite part\n');
return s3.send(new GetObjectCommand({
Bucket: destBucketName,
Key: destObjName,
Expand All @@ -552,7 +562,7 @@ describe('Object Part Copy', () => {
it('should not corrupt object if overwriting an existing part by copying a part ' +
'while the MPU is being completed', async () => {
const finalObjETag = '"db77ebbae9e9f5a244a26b86193ad818-1"';
process.stdout.write('Putting first part in MPU test"');
process.stdout.write('Putting first part in MPU test\n');
const randomDestObjName = `copycatobject${Math.floor(Math.random() * 100000)}`;

const initiateRes = await s3.send(new CreateMultipartUploadCommand({
Expand All @@ -572,7 +582,7 @@ describe('Object Part Copy', () => {
assert(res.CopyPartResult.LastModified);

process.stdout.write(
'Overwriting first part in MPU test and completing MPU at the same time',
'Overwriting first part in MPU test and completing MPU at the same time\n',
);
const [completeRes, uploadRes] = await Promise.all([
s3.send(new CompleteMultipartUploadCommand({
Expand Down Expand Up @@ -611,7 +621,7 @@ describe('Object Part Copy', () => {
assert.strictEqual(completeRes.Key, randomDestObjName);
assert.strictEqual(completeRes.ETag, finalObjETag);
process.stdout.write(
'Getting object put by MPU with ' + 'overwrite part',
'Getting object put by MPU with overwrite part\n',
);
const resGet = await s3
.send(new GetObjectCommand({
Expand Down Expand Up @@ -749,15 +759,15 @@ describe('Object Part Copy', () => {
let otherAccountUploadId;

beforeEach(() => {
process.stdout.write('In other account before each');
process.stdout.write('In other account before each\n');
return otherAccountS3.send(new CreateBucketCommand({ Bucket:
otherAccountBucket }))
.catch(err => {
process.stdout.write('Error creating other account ' +
`bucket: ${err}\n`);
throw err;
}).then(() => {
process.stdout.write('Initiating other account MPU');
process.stdout.write('Initiating other account MPU\n');
return otherAccountS3.send(new CreateMultipartUploadCommand({
Bucket: otherAccountBucket,
Key: otherAccountKey,
Expand Down
Loading