From 2bbcbd9807f68d5fd7628e289329ed8354ae1b30 Mon Sep 17 00:00:00 2001 From: Patrick Griffin <58729+firien@users.noreply.github.com> Date: Thu, 1 Aug 2024 21:34:08 -0400 Subject: [PATCH 1/7] init --- src/zip.js | 127 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 125 insertions(+), 2 deletions(-) diff --git a/src/zip.js b/src/zip.js index e3f5663..a0e475b 100644 --- a/src/zip.js +++ b/src/zip.js @@ -1,3 +1,5 @@ +import { createReadStream, createWriteStream, statSync } from 'node:fs' +import { ReadableStream } from 'node:stream/web' class Entry { static #localFileHeaderLength = 30 static #centralDirectoryFileHeaderLength = 46 @@ -44,12 +46,16 @@ class Entry { * Generate localFileHeader * @return {Blob} */ - localFileHeader () { + localFileHeader ({ stream = false }) { const buffer = new ArrayBuffer(this.constructor.#localFileHeaderLength) const dv = new DataView(buffer) dv.setUint32(0, 0x04034b50, true) // Local file header signature dv.setUint16(4, 0x1400) // Version needed to extract (minimum) - dv.setUint16(6, 0b100000000000, true) // General purpose bit flag + let genFlag = 0b100000000000 + if (stream) { + genFlag |= 0b1000 + } + dv.setUint16(6, genFlag, true) // General purpose bit flag this.commonHeaders(dv, 8) dv.setUint16(28, 0, true) // Extra field length return new Blob([buffer, this.encodedName]) @@ -190,3 +196,120 @@ export default class { return new Blob(blobs, { type: 'application/zip' }) } } + +export class ZipStream { + constructor (file) { + this.zip = createWriteStream(file) + this.entries = [] + } + + async addFile (path) { + const file = new Entry() + file.timeStamp = new Date() + const stats = statSync(path) + file.compressionMethod = 0x0800 + file.externalFileAttributes = 0x0000A481 + const utf8Encode = new TextEncoder() + file.encodedName = utf8Encode.encode(path) + // unknown data + file.crc32 = 0 + file.uncompressedByteSize = 0 + file.compressedByteSize = 0 + file.localFileHeaderOffset = this.zip.bytesWritten + + const localFileHeader = file.localFileHeader({ stream: true }) + const aa = await localFileHeader.arrayBuffer() + const ab = new Uint8Array(aa) + await new Promise((resolve, reject) => { + this.zip.write(ab, resolve) + }) + + const stream = createReadStream(path) + const gzip = new CompressionStream('gzip') + const compressedStream = ReadableStream.from(stream).pipeThrough(gzip) + + let header + let previousChunk + + const headerBytes = 10 + const trailingBytes = 8 + + // get chunks from gzip + const start = this.zip.bytesWritten + for await (const chunk of compressedStream) { + if (previousChunk) { + await new Promise((resolve, reject) => { + this.zip.write(previousChunk, resolve) + }) + } + if (!header) { + header = chunk.slice(0, headerBytes) + if (chunk.length > headerBytes) { + previousChunk = chunk.subarray(headerBytes) + } + } else { + previousChunk = chunk + } + } + file.uncompressedByteSize = stats.size + const footer = previousChunk.slice(-trailingBytes) + const dataView = new DataView(footer.buffer) + // extract crc32 checksum + file.crc32 = dataView.getUint32(0) + // write last chunk of compressed file + await new Promise((resolve, reject) => { + this.zip.write(previousChunk.subarray(0, previousChunk.length - trailingBytes), resolve) + }) + file.compressedByteSize = this.zip.bytesWritten - start + + // Data descriptor + const buffer = new ArrayBuffer(16) + const dv = new DataView(buffer) + dv.setUint32(0, 0x08074b50, true) // Local file header signature + dv.setUint32(4, file.crc32) // Version needed to extract (minimum) + dv.setUint32(8, file.compressedByteSize, true) // Compressed size + dv.setUint32(12, file.uncompressedByteSize, true) // Uncompressed size + // write the data desscriptor + await new Promise((resolve, reject) => { + const cc = new Uint8Array(buffer) + this.zip.write(cc, resolve) + }) + this.entries.push(file) + } + + async close () { + // write central directories + for (const entry of this.entries) { + this.centralDirectoryOffset = this.zip.bytesWritten + const aa = await entry.centralDirectoryFileHeader().arrayBuffer() + const ab = new Uint8Array(aa) + await new Promise((resolve, reject) => { + this.zip.write(ab, resolve) + }) + } + this.centralDirectorySize = this.zip.bytesWritten - this.centralDirectoryOffset + await new Promise((resolve, reject) => { + const bb = new Uint8Array(this.endOfCentralDirectoryRecord()) + this.zip.write(bb, resolve) + }) + this.zip.close() + } + + /** + * Generate endOfCentralDirectoryRecord + * @return {ArrayBuffer} + */ + endOfCentralDirectoryRecord () { + const buffer = new ArrayBuffer(22) + const dv = new DataView(buffer) + dv.setUint32(0, 0x06054b50, true) // End of central directory signature + dv.setUint16(4, 0) // Number of this disk + dv.setUint16(6, 0) // Disk where central directory starts + dv.setUint16(8, this.entries.length, true) // Number of central directory records on this disk + dv.setUint16(10, this.entries.length, true) // Total number of central directory records + dv.setUint32(12, this.centralDirectorySize, true) // Size of central directory + dv.setUint32(16, this.centralDirectoryOffset, true) // Offset of start of central directory + dv.setUint16(20, 0) // Comment length + return buffer + } +} From 3ee5ef19fecef25dcafd342550799b592b20a7fb Mon Sep 17 00:00:00 2001 From: Patrick Griffin <58729+firien@users.noreply.github.com> Date: Thu, 1 Aug 2024 21:46:32 -0400 Subject: [PATCH 2/7] refactor --- src/zip.js | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/src/zip.js b/src/zip.js index a0e475b..153c170 100644 --- a/src/zip.js +++ b/src/zip.js @@ -197,6 +197,24 @@ export default class { } } +/** +* Generate endOfCentralDirectoryRecord +* @return {ArrayBuffer} +*/ +const endOfCentralDirectoryRecord = (entriesCount, size, offset) => { + const buffer = new ArrayBuffer(22) + const dv = new DataView(buffer) + dv.setUint32(0, 0x06054b50, true) // End of central directory signature + dv.setUint16(4, 0) // Number of this disk + dv.setUint16(6, 0) // Disk where central directory starts + dv.setUint16(8, entriesCount, true) // Number of central directory records on this disk + dv.setUint16(10, entriesCount, true) // Total number of central directory records + dv.setUint32(12, size, true) // Size of central directory + dv.setUint32(16, offset, true) // Offset of start of central directory + dv.setUint16(20, 0) // Comment length + return buffer +} + export class ZipStream { constructor (file) { this.zip = createWriteStream(file) @@ -279,37 +297,19 @@ export class ZipStream { async close () { // write central directories + const centralDirectoryOffset = this.zip.bytesWritten for (const entry of this.entries) { - this.centralDirectoryOffset = this.zip.bytesWritten const aa = await entry.centralDirectoryFileHeader().arrayBuffer() const ab = new Uint8Array(aa) await new Promise((resolve, reject) => { this.zip.write(ab, resolve) }) } - this.centralDirectorySize = this.zip.bytesWritten - this.centralDirectoryOffset + const centralDirectorySize = this.zip.bytesWritten - centralDirectoryOffset await new Promise((resolve, reject) => { - const bb = new Uint8Array(this.endOfCentralDirectoryRecord()) + const bb = new Uint8Array(endOfCentralDirectoryRecord(this.entries.length, centralDirectorySize, centralDirectoryOffset)) this.zip.write(bb, resolve) }) this.zip.close() } - - /** - * Generate endOfCentralDirectoryRecord - * @return {ArrayBuffer} - */ - endOfCentralDirectoryRecord () { - const buffer = new ArrayBuffer(22) - const dv = new DataView(buffer) - dv.setUint32(0, 0x06054b50, true) // End of central directory signature - dv.setUint16(4, 0) // Number of this disk - dv.setUint16(6, 0) // Disk where central directory starts - dv.setUint16(8, this.entries.length, true) // Number of central directory records on this disk - dv.setUint16(10, this.entries.length, true) // Total number of central directory records - dv.setUint32(12, this.centralDirectorySize, true) // Size of central directory - dv.setUint32(16, this.centralDirectoryOffset, true) // Offset of start of central directory - dv.setUint16(20, 0) // Comment length - return buffer - } } From 97fb5c34c969688f5feebe5923f9fe465f3e91fc Mon Sep 17 00:00:00 2001 From: Patrick Griffin <58729+firien@users.noreply.github.com> Date: Fri, 2 Aug 2024 21:02:55 -0400 Subject: [PATCH 3/7] refactor --- src/stream.js | 102 ++++++++++++++++++++++++++++++++++++++++++++++++ src/zip.js | 105 +------------------------------------------------- 2 files changed, 104 insertions(+), 103 deletions(-) create mode 100644 src/stream.js diff --git a/src/stream.js b/src/stream.js new file mode 100644 index 0000000..6429336 --- /dev/null +++ b/src/stream.js @@ -0,0 +1,102 @@ +import { createReadStream, createWriteStream, statSync } from 'node:fs' +import { ReadableStream } from 'node:stream/web' +import { Entry, endOfCentralDirectoryRecord } from './zip.js' + +export default class ZipStream { + constructor (file) { + this.zip = createWriteStream(file) + this.entries = [] + } + + async addFile (path) { + const file = new Entry() + const stats = statSync(path) + file.timeStamp = stats.mtime ?? new Date() + file.compressionMethod = 0x0800 + file.externalFileAttributes = 0x0000A481 + const utf8Encode = new TextEncoder() + file.encodedName = utf8Encode.encode(path) + // unknown data + file.crc32 = 0 + file.uncompressedByteSize = 0 + file.compressedByteSize = 0 + file.localFileHeaderOffset = this.zip.bytesWritten + + const localFileHeader = file.localFileHeader({ stream: true }) + const aa = await localFileHeader.arrayBuffer() + const ab = new Uint8Array(aa) + await new Promise((resolve, reject) => { + this.zip.write(ab, resolve) + }) + + const stream = createReadStream(path) + const gzip = new CompressionStream('gzip') + const compressedStream = ReadableStream.from(stream).pipeThrough(gzip) + + let header + let previousChunk + + const headerBytes = 10 + const trailingBytes = 8 + + // get chunks from gzip + const start = this.zip.bytesWritten + for await (const chunk of compressedStream) { + if (previousChunk) { + await new Promise((resolve, reject) => { + this.zip.write(previousChunk, resolve) + }) + } + if (!header) { + header = chunk.slice(0, headerBytes) + if (chunk.length > headerBytes) { + previousChunk = chunk.subarray(headerBytes) + } + } else { + previousChunk = chunk + } + } + file.uncompressedByteSize = stats.size + const footer = previousChunk.slice(-trailingBytes) + const dataView = new DataView(footer.buffer) + // extract crc32 checksum + file.crc32 = dataView.getUint32(0) + // write last chunk of compressed file + await new Promise((resolve, reject) => { + this.zip.write(previousChunk.subarray(0, previousChunk.length - trailingBytes), resolve) + }) + file.compressedByteSize = this.zip.bytesWritten - start + + // Data descriptor + const buffer = new ArrayBuffer(16) + const dv = new DataView(buffer) + dv.setUint32(0, 0x08074b50, true) // Local file header signature + dv.setUint32(4, file.crc32) // Version needed to extract (minimum) + dv.setUint32(8, file.compressedByteSize, true) // Compressed size + dv.setUint32(12, file.uncompressedByteSize, true) // Uncompressed size + // write the data desscriptor + await new Promise((resolve, reject) => { + const cc = new Uint8Array(buffer) + this.zip.write(cc, resolve) + }) + this.entries.push(file) + } + + async close () { + // write central directories + const centralDirectoryOffset = this.zip.bytesWritten + for (const entry of this.entries) { + const aa = await entry.centralDirectoryFileHeader().arrayBuffer() + const ab = new Uint8Array(aa) + await new Promise((resolve, reject) => { + this.zip.write(ab, resolve) + }) + } + const centralDirectorySize = this.zip.bytesWritten - centralDirectoryOffset + await new Promise((resolve, reject) => { + const bb = new Uint8Array(endOfCentralDirectoryRecord(this.entries.length, centralDirectorySize, centralDirectoryOffset)) + this.zip.write(bb, resolve) + }) + this.zip.close() + } +} diff --git a/src/zip.js b/src/zip.js index 153c170..ce8d81c 100644 --- a/src/zip.js +++ b/src/zip.js @@ -1,6 +1,4 @@ -import { createReadStream, createWriteStream, statSync } from 'node:fs' -import { ReadableStream } from 'node:stream/web' -class Entry { +export class Entry { static #localFileHeaderLength = 30 static #centralDirectoryFileHeaderLength = 46 @@ -201,7 +199,7 @@ export default class { * Generate endOfCentralDirectoryRecord * @return {ArrayBuffer} */ -const endOfCentralDirectoryRecord = (entriesCount, size, offset) => { +export const endOfCentralDirectoryRecord = (entriesCount, size, offset) => { const buffer = new ArrayBuffer(22) const dv = new DataView(buffer) dv.setUint32(0, 0x06054b50, true) // End of central directory signature @@ -214,102 +212,3 @@ const endOfCentralDirectoryRecord = (entriesCount, size, offset) => { dv.setUint16(20, 0) // Comment length return buffer } - -export class ZipStream { - constructor (file) { - this.zip = createWriteStream(file) - this.entries = [] - } - - async addFile (path) { - const file = new Entry() - file.timeStamp = new Date() - const stats = statSync(path) - file.compressionMethod = 0x0800 - file.externalFileAttributes = 0x0000A481 - const utf8Encode = new TextEncoder() - file.encodedName = utf8Encode.encode(path) - // unknown data - file.crc32 = 0 - file.uncompressedByteSize = 0 - file.compressedByteSize = 0 - file.localFileHeaderOffset = this.zip.bytesWritten - - const localFileHeader = file.localFileHeader({ stream: true }) - const aa = await localFileHeader.arrayBuffer() - const ab = new Uint8Array(aa) - await new Promise((resolve, reject) => { - this.zip.write(ab, resolve) - }) - - const stream = createReadStream(path) - const gzip = new CompressionStream('gzip') - const compressedStream = ReadableStream.from(stream).pipeThrough(gzip) - - let header - let previousChunk - - const headerBytes = 10 - const trailingBytes = 8 - - // get chunks from gzip - const start = this.zip.bytesWritten - for await (const chunk of compressedStream) { - if (previousChunk) { - await new Promise((resolve, reject) => { - this.zip.write(previousChunk, resolve) - }) - } - if (!header) { - header = chunk.slice(0, headerBytes) - if (chunk.length > headerBytes) { - previousChunk = chunk.subarray(headerBytes) - } - } else { - previousChunk = chunk - } - } - file.uncompressedByteSize = stats.size - const footer = previousChunk.slice(-trailingBytes) - const dataView = new DataView(footer.buffer) - // extract crc32 checksum - file.crc32 = dataView.getUint32(0) - // write last chunk of compressed file - await new Promise((resolve, reject) => { - this.zip.write(previousChunk.subarray(0, previousChunk.length - trailingBytes), resolve) - }) - file.compressedByteSize = this.zip.bytesWritten - start - - // Data descriptor - const buffer = new ArrayBuffer(16) - const dv = new DataView(buffer) - dv.setUint32(0, 0x08074b50, true) // Local file header signature - dv.setUint32(4, file.crc32) // Version needed to extract (minimum) - dv.setUint32(8, file.compressedByteSize, true) // Compressed size - dv.setUint32(12, file.uncompressedByteSize, true) // Uncompressed size - // write the data desscriptor - await new Promise((resolve, reject) => { - const cc = new Uint8Array(buffer) - this.zip.write(cc, resolve) - }) - this.entries.push(file) - } - - async close () { - // write central directories - const centralDirectoryOffset = this.zip.bytesWritten - for (const entry of this.entries) { - const aa = await entry.centralDirectoryFileHeader().arrayBuffer() - const ab = new Uint8Array(aa) - await new Promise((resolve, reject) => { - this.zip.write(ab, resolve) - }) - } - const centralDirectorySize = this.zip.bytesWritten - centralDirectoryOffset - await new Promise((resolve, reject) => { - const bb = new Uint8Array(endOfCentralDirectoryRecord(this.entries.length, centralDirectorySize, centralDirectoryOffset)) - this.zip.write(bb, resolve) - }) - this.zip.close() - } -} From 47af0c1ac096974464a899a8cec55095af28dc47 Mon Sep 17 00:00:00 2001 From: Patrick Griffin <58729+firien@users.noreply.github.com> Date: Fri, 2 Aug 2024 22:01:24 -0400 Subject: [PATCH 4/7] init recursive folder --- src/stream.js | 39 ++++++++++++++++++++++++++++++++++++++- src/zip.js | 2 +- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/src/stream.js b/src/stream.js index 6429336..3fa511d 100644 --- a/src/stream.js +++ b/src/stream.js @@ -1,5 +1,7 @@ -import { createReadStream, createWriteStream, statSync } from 'node:fs' +import { createReadStream, createWriteStream, statSync, existsSync } from 'node:fs' +import { readdir } from 'node:fs/promises' import { ReadableStream } from 'node:stream/web' +import { resolve, join, dirname } from 'node:path' import { Entry, endOfCentralDirectoryRecord } from './zip.js' export default class ZipStream { @@ -8,6 +10,41 @@ export default class ZipStream { this.entries = [] } + async addFolder (dir) { + if (existsSync(dir)) { + const stats = statSync(dir) + if (stats.isDirectory()) { + const folder = new Entry() + folder.timeStamp = stats.mtime ?? new Date() + folder.uncompressedByteSize = 0 + folder.compressedByteSize = 0 + folder.crc32 = 0 + folder.compressionMethod = 0 + folder.localFileHeaderOffset = this.zip.bytesWritten + folder.externalFileAttributes = 0x0000ED41 + const utf8Encode = new TextEncoder() + folder.encodedName = utf8Encode.encode(`${dir}/`) + // write folder + const localFileHeader = folder.localFileHeader() + const aa = await localFileHeader.arrayBuffer() + const ab = new Uint8Array(aa) + await new Promise((resolve, reject) => { + this.zip.write(ab, resolve) + }) + this.entries.push(folder) + const files = await readdir(dir, { withFileTypes: true }) + for (const file of files) { + const path = join(file.path, file.name) + if (file.isDirectory()) { + await this.addFolder(path) + } else { + await this.addFile(path) + } + } + } + } + } + async addFile (path) { const file = new Entry() const stats = statSync(path) diff --git a/src/zip.js b/src/zip.js index ce8d81c..94f3d16 100644 --- a/src/zip.js +++ b/src/zip.js @@ -44,7 +44,7 @@ export class Entry { * Generate localFileHeader * @return {Blob} */ - localFileHeader ({ stream = false }) { + localFileHeader ({ stream = false } = {}) { const buffer = new ArrayBuffer(this.constructor.#localFileHeaderLength) const dv = new DataView(buffer) dv.setUint32(0, 0x04034b50, true) // Local file header signature From fe5a26d7d91df779ee90970c770db14d36600787 Mon Sep 17 00:00:00 2001 From: Patrick Griffin <58729+firien@users.noreply.github.com> Date: Wed, 30 Jul 2025 09:51:25 -0400 Subject: [PATCH 5/7] update github actions --- test/node/zip.test.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/node/zip.test.js b/test/node/zip.test.js index afba775..2039c69 100644 --- a/test/node/zip.test.js +++ b/test/node/zip.test.js @@ -35,5 +35,6 @@ test('zip with folder', async () => { const zipBlob = await zip.write() const buffer = await zipBlob.arrayBuffer() const dv = new DataView(buffer) - writeFileSync('/tmp/tast.zip', dv) + const zipFile = join(tmpdir(), 'tast.zip') + writeFileSync(zipFile, dv) }) From dccb69d7ef9fa9c6ae6f63c705f309d17b701bbe Mon Sep 17 00:00:00 2001 From: Patrick Griffin <58729+firien@users.noreply.github.com> Date: Wed, 30 Jul 2025 09:59:29 -0400 Subject: [PATCH 6/7] show unzip version --- .github/workflows/node.js.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index 3b4c304..8e9bdd0 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -23,4 +23,4 @@ jobs: uses: actions/setup-node@v4 with: node-version: ${{ matrix.node }} - - run: npm run test \ No newline at end of file + - run: unzip --version \ No newline at end of file From 376e4fd68b41c0e53e5896254cefb10877c9079e Mon Sep 17 00:00:00 2001 From: Patrick Griffin <58729+firien@users.noreply.github.com> Date: Thu, 7 Aug 2025 05:14:20 -0400 Subject: [PATCH 7/7] revert to test --- .github/workflows/node.js.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index 8e9bdd0..9b2e3d1 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -23,4 +23,4 @@ jobs: uses: actions/setup-node@v4 with: node-version: ${{ matrix.node }} - - run: unzip --version \ No newline at end of file + - run: npm run test