diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index 3b4c304..9b2e3d1 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -23,4 +23,4 @@ jobs: uses: actions/setup-node@v4 with: node-version: ${{ matrix.node }} - - run: npm run test \ No newline at end of file + - run: npm run test diff --git a/src/stream.js b/src/stream.js new file mode 100644 index 0000000..3fa511d --- /dev/null +++ b/src/stream.js @@ -0,0 +1,139 @@ +import { createReadStream, createWriteStream, statSync, existsSync } from 'node:fs' +import { readdir } from 'node:fs/promises' +import { ReadableStream } from 'node:stream/web' +import { resolve, join, dirname } from 'node:path' +import { Entry, endOfCentralDirectoryRecord } from './zip.js' + +export default class ZipStream { + constructor (file) { + this.zip = createWriteStream(file) + this.entries = [] + } + + async addFolder (dir) { + if (existsSync(dir)) { + const stats = statSync(dir) + if (stats.isDirectory()) { + const folder = new Entry() + folder.timeStamp = stats.mtime ?? new Date() + folder.uncompressedByteSize = 0 + folder.compressedByteSize = 0 + folder.crc32 = 0 + folder.compressionMethod = 0 + folder.localFileHeaderOffset = this.zip.bytesWritten + folder.externalFileAttributes = 0x0000ED41 + const utf8Encode = new TextEncoder() + folder.encodedName = utf8Encode.encode(`${dir}/`) + // write folder + const localFileHeader = folder.localFileHeader() + const aa = await localFileHeader.arrayBuffer() + const ab = new Uint8Array(aa) + await new Promise((resolve, reject) => { + this.zip.write(ab, resolve) + }) + this.entries.push(folder) + const files = await readdir(dir, { withFileTypes: true }) + for (const file of files) { + const path = join(file.path, file.name) + if (file.isDirectory()) { + await this.addFolder(path) + } else { + await this.addFile(path) + } + } + } + } + } + + async addFile (path) { + const file = new Entry() + const stats = statSync(path) + file.timeStamp = stats.mtime ?? new Date() + file.compressionMethod = 0x0800 + file.externalFileAttributes = 0x0000A481 + const utf8Encode = new TextEncoder() + file.encodedName = utf8Encode.encode(path) + // unknown data + file.crc32 = 0 + file.uncompressedByteSize = 0 + file.compressedByteSize = 0 + file.localFileHeaderOffset = this.zip.bytesWritten + + const localFileHeader = file.localFileHeader({ stream: true }) + const aa = await localFileHeader.arrayBuffer() + const ab = new Uint8Array(aa) + await new Promise((resolve, reject) => { + this.zip.write(ab, resolve) + }) + + const stream = createReadStream(path) + const gzip = new CompressionStream('gzip') + const compressedStream = ReadableStream.from(stream).pipeThrough(gzip) + + let header + let previousChunk + + const headerBytes = 10 + const trailingBytes = 8 + + // get chunks from gzip + const start = this.zip.bytesWritten + for await (const chunk of compressedStream) { + if (previousChunk) { + await new Promise((resolve, reject) => { + this.zip.write(previousChunk, resolve) + }) + } + if (!header) { + header = chunk.slice(0, headerBytes) + if (chunk.length > headerBytes) { + previousChunk = chunk.subarray(headerBytes) + } + } else { + previousChunk = chunk + } + } + file.uncompressedByteSize = stats.size + const footer = previousChunk.slice(-trailingBytes) + const dataView = new DataView(footer.buffer) + // extract crc32 checksum + file.crc32 = dataView.getUint32(0) + // write last chunk of compressed file + await new Promise((resolve, reject) => { + this.zip.write(previousChunk.subarray(0, previousChunk.length - trailingBytes), resolve) + }) + file.compressedByteSize = this.zip.bytesWritten - start + + // Data descriptor + const buffer = new ArrayBuffer(16) + const dv = new DataView(buffer) + dv.setUint32(0, 0x08074b50, true) // Local file header signature + dv.setUint32(4, file.crc32) // Version needed to extract (minimum) + dv.setUint32(8, file.compressedByteSize, true) // Compressed size + dv.setUint32(12, file.uncompressedByteSize, true) // Uncompressed size + // write the data desscriptor + await new Promise((resolve, reject) => { + const cc = new Uint8Array(buffer) + this.zip.write(cc, resolve) + }) + this.entries.push(file) + } + + async close () { + // write central directories + const centralDirectoryOffset = this.zip.bytesWritten + for (const entry of this.entries) { + const aa = await entry.centralDirectoryFileHeader().arrayBuffer() + const ab = new Uint8Array(aa) + await new Promise((resolve, reject) => { + this.zip.write(ab, resolve) + }) + } + const centralDirectorySize = this.zip.bytesWritten - centralDirectoryOffset + await new Promise((resolve, reject) => { + const bb = new Uint8Array(endOfCentralDirectoryRecord(this.entries.length, centralDirectorySize, centralDirectoryOffset)) + this.zip.write(bb, resolve) + }) + this.zip.close() + } +} diff --git a/src/zip.js b/src/zip.js index e3f5663..94f3d16 100644 --- a/src/zip.js +++ b/src/zip.js @@ -1,4 +1,4 @@ -class Entry { +export class Entry { static #localFileHeaderLength = 30 static #centralDirectoryFileHeaderLength = 46 @@ -44,12 +44,16 @@ class Entry { * Generate localFileHeader * @return {Blob} */ - localFileHeader () { + localFileHeader ({ stream = false } = {}) { const buffer = new ArrayBuffer(this.constructor.#localFileHeaderLength) const dv = new DataView(buffer) dv.setUint32(0, 0x04034b50, true) // Local file header signature dv.setUint16(4, 0x1400) // Version needed to extract (minimum) - dv.setUint16(6, 0b100000000000, true) // General purpose bit flag + let genFlag = 0b100000000000 + if (stream) { + genFlag |= 0b1000 + } + dv.setUint16(6, genFlag, true) // General purpose bit flag this.commonHeaders(dv, 8) dv.setUint16(28, 0, true) // Extra field length return new Blob([buffer, this.encodedName]) @@ -190,3 +194,21 @@ export default class { return new Blob(blobs, { type: 'application/zip' }) } } + +/** +* Generate endOfCentralDirectoryRecord +* @return {ArrayBuffer} +*/ +export const endOfCentralDirectoryRecord = (entriesCount, size, offset) => { + const buffer = new ArrayBuffer(22) + const dv = new DataView(buffer) + dv.setUint32(0, 0x06054b50, true) // End of central directory signature + dv.setUint16(4, 0) // Number of this disk + dv.setUint16(6, 0) // Disk where central directory starts + dv.setUint16(8, entriesCount, true) // Number of central directory records on this disk + dv.setUint16(10, entriesCount, true) // Total number of central directory records + dv.setUint32(12, size, true) // Size of central directory + dv.setUint32(16, offset, true) // Offset of start of central directory + dv.setUint16(20, 0) // Comment length + return buffer +} diff --git a/test/node/zip.test.js b/test/node/zip.test.js index afba775..2039c69 100644 --- a/test/node/zip.test.js +++ b/test/node/zip.test.js @@ -35,5 +35,6 @@ test('zip with folder', async () => { const zipBlob = await zip.write() const buffer = await zipBlob.arrayBuffer() const dv = new DataView(buffer) - writeFileSync('/tmp/tast.zip', dv) + const zipFile = join(tmpdir(), 'tast.zip') + writeFileSync(zipFile, dv) })