From 6856869750fcd2b48f6f89ba13e84dacd836a04a Mon Sep 17 00:00:00 2001 From: Alan Codega Date: Mon, 15 May 2023 10:55:18 +0200 Subject: [PATCH 1/2] 1st raw possible code for azure blob storage --- lib/plugins/azure/index.js | 89 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 lib/plugins/azure/index.js diff --git a/lib/plugins/azure/index.js b/lib/plugins/azure/index.js new file mode 100644 index 0000000000..4b24387197 --- /dev/null +++ b/lib/plugins/azure/index.js @@ -0,0 +1,89 @@ +import { relative, join, resolve as _resolve, sep } from 'node:path'; +import fs from 'node:fs'; +import { BlobServiceClient } from '@azure/storage-blob'; +import { SitespeedioPlugin } from '@sitespeed.io/plugin'; +import readdir from 'recursive-readdir'; +import pLimit from 'p-limit'; +import mime from 'mime-types'; +import intel from 'intel'; +import { throwIfMissing } from '../../support/util.js'; + +const log = intel.getLogger('sitespeedio.plugin.azure'); + +function ignoreDirectories(file, stats) { + return stats.isDirectory(); +} + +async function upload(dir, azureOptions, prefix) { + const blobServiceClient = BlobServiceClient.fromConnectionString(azureOptions.connectionString); + const containerClient = blobServiceClient.getContainerClient(azureOptions.containerName); + await containerClient.createIfNotExists(); + + const files = await readdir(dir); + const limit = pLimit(azureOptions.maxAsyncAzure || 20); + const promises = []; + + for (let file of files) { + promises.push(limit(() => uploadFile(file, containerClient, azureOptions, prefix, dir))); + } + + return Promise.all(promises); +} + +async function uploadFile(file, containerClient, azureOptions, prefix, baseDir) { + const subPath = relative(baseDir, file); + const blobName = join(azureOptions.path || prefix, subPath); + const blockBlobClient = containerClient.getBlockBlobClient(blobName); + + const data = fs.readFileSync(file); + const contentType = mime.lookup(file) || 'application/octet-stream'; + const options = { blobHTTPHeaders: { blobContentType: contentType } }; + // add custom parameters if any + Object.assign(options, azureOptions.params); + + await blockBlobClient.upload(data, data.length, options); + + log.info(`Uploaded file to Azure Blob Storage: ${blobName}`); +} + +export default class AzurePlugin extends SitespeedioPlugin { + constructor(options, context, queue) { + super({ name: 'azure', options, context, queue }); + } + + open(context, options) { + this.azureOptions = options.azure; + this.options = options; + this.make = context.messageMaker('azure').make; + throwIfMissing(this.azureOptions, ['containerName', 'connectionString'], 'azure'); + this.storageManager = context.storageManager; + } + + async processMessage(message, queue) { + if (message.type === 'sitespeedio.setup') { + queue.postMessage(this.make('azure.setup')); + } else if (message.type === 'html.finished') { + const make = this.make; + const azureOptions = this.azureOptions; + const baseDir = this.storageManager.getBaseDir(); + + log.info(`Uploading ${baseDir} to Azure Blob Storage container ${azureOptions.containerName}, this can take a while ...`); + + try { + await upload(baseDir, azureOptions, this.storageManager.getStoragePrefix()); + log.info('Finished upload to Azure Blob Storage'); + if (azureOptions.removeLocalResult) { + fs.rmSync(baseDir, { recursive: true }); + log.debug(`Removed local files and directory ${baseDir}`); + } else { + log.debug(`Local result files and directories are stored in ${baseDir}`); + } + } catch (error) { + queue.postMessage(make('error', error)); + log.error('Could not upload to Azure Blob Storage', error); + } + queue.postMessage(make('azure.finished')); + } + } +} + From d347df529a351489fdce6ba1822da85d7b2c8f57 Mon Sep 17 00:00:00 2001 From: Alan Codega Date: Mon, 15 May 2023 11:24:52 +0200 Subject: [PATCH 2/2] lint fix --- lib/plugins/azure/index.js | 47 ++++++++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/lib/plugins/azure/index.js b/lib/plugins/azure/index.js index 4b24387197..db1dfd7b71 100644 --- a/lib/plugins/azure/index.js +++ b/lib/plugins/azure/index.js @@ -1,4 +1,4 @@ -import { relative, join, resolve as _resolve, sep } from 'node:path'; +import { relative, join } from 'node:path'; import fs from 'node:fs'; import { BlobServiceClient } from '@azure/storage-blob'; import { SitespeedioPlugin } from '@sitespeed.io/plugin'; @@ -10,13 +10,13 @@ import { throwIfMissing } from '../../support/util.js'; const log = intel.getLogger('sitespeedio.plugin.azure'); -function ignoreDirectories(file, stats) { - return stats.isDirectory(); -} - async function upload(dir, azureOptions, prefix) { - const blobServiceClient = BlobServiceClient.fromConnectionString(azureOptions.connectionString); - const containerClient = blobServiceClient.getContainerClient(azureOptions.containerName); + const blobServiceClient = BlobServiceClient.fromConnectionString( + azureOptions.connectionString + ); + const containerClient = blobServiceClient.getContainerClient( + azureOptions.containerName + ); await containerClient.createIfNotExists(); const files = await readdir(dir); @@ -24,13 +24,21 @@ async function upload(dir, azureOptions, prefix) { const promises = []; for (let file of files) { - promises.push(limit(() => uploadFile(file, containerClient, azureOptions, prefix, dir))); + promises.push( + limit(() => uploadFile(file, containerClient, azureOptions, prefix, dir)) + ); } return Promise.all(promises); } -async function uploadFile(file, containerClient, azureOptions, prefix, baseDir) { +async function uploadFile( + file, + containerClient, + azureOptions, + prefix, + baseDir +) { const subPath = relative(baseDir, file); const blobName = join(azureOptions.path || prefix, subPath); const blockBlobClient = containerClient.getBlockBlobClient(blobName); @@ -55,7 +63,11 @@ export default class AzurePlugin extends SitespeedioPlugin { this.azureOptions = options.azure; this.options = options; this.make = context.messageMaker('azure').make; - throwIfMissing(this.azureOptions, ['containerName', 'connectionString'], 'azure'); + throwIfMissing( + this.azureOptions, + ['containerName', 'connectionString'], + 'azure' + ); this.storageManager = context.storageManager; } @@ -67,16 +79,24 @@ export default class AzurePlugin extends SitespeedioPlugin { const azureOptions = this.azureOptions; const baseDir = this.storageManager.getBaseDir(); - log.info(`Uploading ${baseDir} to Azure Blob Storage container ${azureOptions.containerName}, this can take a while ...`); + log.info( + `Uploading ${baseDir} to Azure Blob Storage container ${azureOptions.containerName}, this can take a while ...` + ); try { - await upload(baseDir, azureOptions, this.storageManager.getStoragePrefix()); + await upload( + baseDir, + azureOptions, + this.storageManager.getStoragePrefix() + ); log.info('Finished upload to Azure Blob Storage'); if (azureOptions.removeLocalResult) { fs.rmSync(baseDir, { recursive: true }); log.debug(`Removed local files and directory ${baseDir}`); } else { - log.debug(`Local result files and directories are stored in ${baseDir}`); + log.debug( + `Local result files and directories are stored in ${baseDir}` + ); } } catch (error) { queue.postMessage(make('error', error)); @@ -86,4 +106,3 @@ export default class AzurePlugin extends SitespeedioPlugin { } } } -