Skip to content

Commit e5870d0

Browse files
committed
feat(azblob): add support for eager merge
Signed-off-by: Jonas Beck <dev@jonasbeck.dk>
1 parent 4930656 commit e5870d0

1 file changed

Lines changed: 32 additions & 0 deletions

File tree

‎lib/storage.ts‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1443,6 +1443,38 @@ class AzBlobAdapter implements StorageAdapter {
14431443
private usesSharedKey
14441444
private keyPrefix = 'gh-actions-cache'
14451445

1446+
// Put Block From URL, service version 2020-04-08+: 4000 MiB per block, 50000 blocks per blob.
1447+
// https://learn.microsoft.com/en-us/rest/api/storageservices/put-block-from-url
1448+
composeParts = {
1449+
minPartBytes: 0,
1450+
maxPartBytes: 4000 * 1024 ** 2,
1451+
maxParts: 50_000,
1452+
run: async (folderName: string, partCount: number) => {
1453+
const mergedBlobClient = this.containerClient.getBlockBlobClient(
1454+
this.blobKey(`${folderName}/merged`),
1455+
)
1456+
// stageBlockFromURL needs an authenticated URL to each source part.
1457+
const expiresAt = Date.now() + DIRECT_DOWNLOAD_LEASE_DURATION_MS
1458+
const blockIds: string[] = []
1459+
for (let index = 0; index < partCount; index++) {
1460+
const blockId = Buffer.from(String(index).padStart(10, '0')).toString('base64')
1461+
const sourceUrl = await this.createDownloadUrl(`${folderName}/parts/${index}`, expiresAt)
1462+
try {
1463+
await mergedBlobClient.stageBlockFromURL(blockId, sourceUrl)
1464+
} catch (err) {
1465+
logger.error('AzBlob stageBlockFromURL failed', {
1466+
error: err,
1467+
sourceUrl,
1468+
blockId,
1469+
})
1470+
throw err
1471+
}
1472+
blockIds.push(blockId)
1473+
}
1474+
await mergedBlobClient.commitBlockList(blockIds)
1475+
},
1476+
}
1477+
14461478
constructor({
14471479
client,
14481480
account,

0 commit comments

Comments
 (0)