streams

.github/workflows/node.js.yml

@@ -23,4 +23,4 @@ jobs:
       uses: actions/setup-node@v4
       with:
         node-version: ${{ matrix.node }}
-    - run: npm run test
+    - run: npm run test

src/stream.js

@@ -0,0 +1,139 @@
+import { createReadStream, createWriteStream, statSync, existsSync } from 'node:fs'
+import { readdir } from 'node:fs/promises'
+import { ReadableStream } from 'node:stream/web'
+import { resolve, join, dirname } from 'node:path'
+import { Entry, endOfCentralDirectoryRecord } from './zip.js'
+
+export default class ZipStream {
+  constructor (file) {
+    this.zip = createWriteStream(file)
+    this.entries = []
+  }
+
+  async addFolder (dir) {
+    if (existsSync(dir)) {
+      const stats = statSync(dir)
+      if (stats.isDirectory()) {
+        const folder = new Entry()
+        folder.timeStamp = stats.mtime ?? new Date()
+        folder.uncompressedByteSize = 0
+        folder.compressedByteSize = 0
+        folder.crc32 = 0
+        folder.compressionMethod = 0
+        folder.localFileHeaderOffset = this.zip.bytesWritten
+        folder.externalFileAttributes = 0x0000ED41
+        const utf8Encode = new TextEncoder()
+        folder.encodedName = utf8Encode.encode(`${dir}/`)
+        // write folder
+        const localFileHeader = folder.localFileHeader()
+        const aa = await localFileHeader.arrayBuffer()
+        const ab = new Uint8Array(aa)
+        await new Promise((resolve, reject) => {
+          this.zip.write(ab, resolve)
+        })
+        this.entries.push(folder)
+        const files = await readdir(dir, { withFileTypes: true })
+        for (const file of files) {
+          const path = join(file.path, file.name)
+          if (file.isDirectory()) {
+            await this.addFolder(path)
+          } else {
+            await this.addFile(path)
+          }
+        }
+      }
+    }
+  }
+
+  async addFile (path) {
+    const file = new Entry()
+    const stats = statSync(path)
+    file.timeStamp = stats.mtime ?? new Date()
+    file.compressionMethod = 0x0800
+    file.externalFileAttributes = 0x0000A481
+    const utf8Encode = new TextEncoder()
+    file.encodedName = utf8Encode.encode(path)
+    // unknown data
+    file.crc32 = 0
+    file.uncompressedByteSize = 0
+    file.compressedByteSize = 0
+    file.localFileHeaderOffset = this.zip.bytesWritten
+
+    const localFileHeader = file.localFileHeader({ stream: true })
+    const aa = await localFileHeader.arrayBuffer()
+    const ab = new Uint8Array(aa)
+    await new Promise((resolve, reject) => {
+      this.zip.write(ab, resolve)
+    })
+
+    const stream = createReadStream(path)
+    const gzip = new CompressionStream('gzip')
+    const compressedStream = ReadableStream.from(stream).pipeThrough(gzip)
+
+    let header
+    let previousChunk
+
+    const headerBytes = 10
+    const trailingBytes = 8
+
+    // get chunks from gzip
+    const start = this.zip.bytesWritten
+    for await (const chunk of compressedStream) {
+      if (previousChunk) {
+        await new Promise((resolve, reject) => {
+          this.zip.write(previousChunk, resolve)
+        })
+      }
+      if (!header) {
+        header = chunk.slice(0, headerBytes)
+        if (chunk.length > headerBytes) {
+          previousChunk = chunk.subarray(headerBytes)
+        }
+      } else {
+        previousChunk = chunk
+      }
+    }
+    file.uncompressedByteSize = stats.size
+    const footer = previousChunk.slice(-trailingBytes)
+    const dataView = new DataView(footer.buffer)
+    // extract crc32 checksum
+    file.crc32 = dataView.getUint32(0)
+    // write last chunk of compressed file
+    await new Promise((resolve, reject) => {
+      this.zip.write(previousChunk.subarray(0, previousChunk.length - trailingBytes), resolve)
+    })
+    file.compressedByteSize = this.zip.bytesWritten - start
+
+    // Data descriptor
+    const buffer = new ArrayBuffer(16)
+    const dv = new DataView(buffer)
+    dv.setUint32(0, 0x08074b50, true) // Local file header signature
+    dv.setUint32(4, file.crc32) // Version needed to extract (minimum)
+    dv.setUint32(8, file.compressedByteSize, true) // Compressed size
+    dv.setUint32(12, file.uncompressedByteSize, true) // Uncompressed size
+    // write the data desscriptor
+    await new Promise((resolve, reject) => {
+      const cc = new Uint8Array(buffer)
+      this.zip.write(cc, resolve)
+    })
+    this.entries.push(file)
+  }
+
+  async close () {
+    // write central directories
+    const centralDirectoryOffset = this.zip.bytesWritten
+    for (const entry of this.entries) {
+      const aa = await entry.centralDirectoryFileHeader().arrayBuffer()
+      const ab = new Uint8Array(aa)
+      await new Promise((resolve, reject) => {
+        this.zip.write(ab, resolve)
+      })
+    }
+    const centralDirectorySize = this.zip.bytesWritten - centralDirectoryOffset
+    await new Promise((resolve, reject) => {
+      const bb = new Uint8Array(endOfCentralDirectoryRecord(this.entries.length, centralDirectorySize, centralDirectoryOffset))
+      this.zip.write(bb, resolve)
+    })
+    this.zip.close()
+  }
+}

src/zip.js

@@ -1,4 +1,4 @@
-class Entry {
+export class Entry {
   static #localFileHeaderLength = 30
   static #centralDirectoryFileHeaderLength = 46
 
@@ -44,12 +44,16 @@ class Entry {
    * Generate localFileHeader
    * @return {Blob}
    */
-  localFileHeader () {
+  localFileHeader ({ stream = false } = {}) {
     const buffer = new ArrayBuffer(this.constructor.#localFileHeaderLength)
     const dv = new DataView(buffer)
     dv.setUint32(0, 0x04034b50, true) // Local file header signature
     dv.setUint16(4, 0x1400) // Version needed to extract (minimum)
-    dv.setUint16(6, 0b100000000000, true) // General purpose bit flag
+    let genFlag = 0b100000000000
+    if (stream) {
+      genFlag |= 0b1000
+    }
+    dv.setUint16(6, genFlag, true) // General purpose bit flag
     this.commonHeaders(dv, 8)
     dv.setUint16(28, 0, true) // Extra field length
     return new Blob([buffer, this.encodedName])
@@ -190,3 +194,21 @@ export default class {
     return new Blob(blobs, { type: 'application/zip' })
   }
 }
+
+/**
+* Generate endOfCentralDirectoryRecord
+* @return {ArrayBuffer}
+*/
+export const endOfCentralDirectoryRecord = (entriesCount, size, offset) => {
+  const buffer = new ArrayBuffer(22)
+  const dv = new DataView(buffer)
+  dv.setUint32(0, 0x06054b50, true) // End of central directory signature
+  dv.setUint16(4, 0) // Number of this disk
+  dv.setUint16(6, 0) // Disk where central directory starts
+  dv.setUint16(8, entriesCount, true) // Number of central directory records on this disk
+  dv.setUint16(10, entriesCount, true) // Total number of central directory records
+  dv.setUint32(12, size, true) // Size of central directory
+  dv.setUint32(16, offset, true) // Offset of start of central directory
+  dv.setUint16(20, 0) // Comment length
+  return buffer
+}

test/node/zip.test.js

@@ -35,5 +35,6 @@ test('zip with folder', async () => {
   const zipBlob = await zip.write()
   const buffer = await zipBlob.arrayBuffer()
   const dv = new DataView(buffer)
-  writeFileSync('/tmp/tast.zip', dv)
+  const zipFile = join(tmpdir(), 'tast.zip')
+  writeFileSync(zipFile, dv)
 })