Support loading a dictionary for decompression

src/index.ts

@@ -82,7 +82,8 @@ export const ZstdErrorCode = {
   InvalidBlockType: 2,
   FSEAccuracyTooHigh: 3,
   DistanceTooFarBack: 4,
-  UnexpectedEOF: 5
+  UnexpectedEOF: 5,
+  WrongDictionary: 6
 } as const;
 
 type ZEC = (typeof ZstdErrorCode)[keyof typeof ZstdErrorCode];
@@ -94,7 +95,8 @@ const ec: Record<ZEC, string | undefined> = [
   'invalid block type',
   'FSE accuracy too high',
   'match distance too far back',
-  'unexpected EOF'
+  'unexpected EOF',
+  'wrong dictionary'
 ];
 
 /**
@@ -123,6 +125,24 @@ const rb = (d: Uint8Array, b: number, n: number) => {
 
 const b4 = (d: Uint8Array, b: number) => (d[b] | (d[b + 1] << 8) | (d[b + 2] << 16) | (d[b + 3] << 24)) >>> 0;
 
+// apply a dictionary to decompressor state
+const rdic = (dic: Uint8Array, st: DZstdState) => {
+  if (b4(dic, 0) == 0xec30a437) {
+    if (st.d && st.d !== b4(dic, 4)) err(6);
+    let bt = 8;
+    [bt, st.h] = rhu(dic, bt);
+    let mlt: FSEDT, olt: FSEDT, llt: FSEDT;
+    [bt, olt] = rfse(dic, bt, 8);
+    [bt, mlt] = rfse(dic, bt, 9);
+    [bt, llt] = rfse(dic, bt, 9);
+    st.t = [mlt, olt, llt];
+    st.o = new i32([b4(dic, bt), b4(dic, bt + 4), b4(dic, bt + 8)]);
+    dic = dic.subarray(bt + 12);
+  }
+  st.w = dic.slice();
+  st.e = dic.length;
+};
+
 // read Zstandard frame header
 const rzfh = (dat: Uint8Array, w?: Uint8Array | 1): number | DZstdState => {
   const n3 = dat[0] | (dat[1] << 8) | (dat[2] << 16);
@@ -628,14 +648,18 @@ const cct = (bufs: Uint8Array[], ol: number) => {
  *            exactly enough memory to fit the decompressed data. If your
  *            data has multiple frames and you know the output size, specifying
  *            it will yield better performance.
+ * @param dic The dictionary buffer. May be a Zstandard-formatted dictionary
+ *            (as generated by zstd --train) or raw content.
  * @returns The decompressed data
  */
-export function decompress(dat: Uint8Array, buf?: Uint8Array) {
+export function decompress(dat: Uint8Array, buf?: Uint8Array, dic?: Uint8Array) {
   const bufs: Uint8Array[] = [], nb = +!buf as 0 | 1;
   let bt = 0, ol = 0;
   for (; dat.length;) {
     const st = rzfh(dat, nb || buf);
     if (typeof st == 'object') {
+      if (st.d && !dic) err(6);
+      if (dic) rdic(dic, st);
       if (nb) {
         buf = null;
         if (st.w.length == st.u) {
@@ -679,6 +703,7 @@ export class Decompress {
   private c: Uint8Array[];
   private l: number;
   private z: number;
+  private d: Uint8Array;
   /**
    * Creates a Zstandard decompressor
    * @param ondata The handler for stream data
@@ -690,6 +715,15 @@ export class Decompress {
     this.z = 0;
   }
 
+  /**
+   * Loads a dictionary
+   * @param dic The dictionary buffer. May be a Zstandard-formatted dictionary
+   *            (as generated by zstd --train) or raw content.
+   */
+  loadDictionary(dic: Uint8Array) {
+    this.d = dic;
+  }
+
   /**
    * Pushes data to be decompressed
    * @param chunk The chunk of data to push
@@ -723,6 +757,8 @@ export class Decompress {
         this.l = 0;
       }
       if (typeof (this.s = rzfh(chunk)) == 'number') return this.push(chunk, final);
+      if (this.s.d && !this.d) err(6);
+      if (this.d) rdic(this.d, this.s);
     }
     if (typeof this.s != 'number') {
       if (ncs < (this.z || 3)) {
@@ -770,4 +806,4 @@ export class Decompress {
    * Handler called whenever data is decompressed
    */
   ondata: ZstdStreamHandler;
-}
+}

tests/lorem-ipsum-samples/1

@@ -0,0 +1 @@
+Est qui non sit laborum aliquid. Ea id perspiciatis doloremque. Deleniti et quo velit distinctio neque. Qui nihil ipsum nam assumenda officia. Nam ea unde laudantium consectetur corrupti ullam fugiat in.

tests/lorem-ipsum-samples/10

@@ -0,0 +1 @@
+Donec nec accumsan felis, vel lobortis ipsum. Suspendisse porttitor, ligula a aliquam lobortis, metus sapien elementum nibh, ac dapibus ex risus ac dui. In nec ex ut est molestie efficitur. Pellentesque aliquet dui et libero ullamcorper, et sodales risus laoreet. Ut et pharetra nulla. Pellentesque quis sapien tempus, lacinia tellus non, malesuada lorem. Curabitur et tortor in felis malesuada imperdiet. Pellentesque aliquam ligula a urna dignissim accumsan. Proin sit amet malesuada lorem. Aenean vitae est vel ante tincidunt sodales sit amet at leo. Donec maximus dapibus ultrices. Donec quis odio in urna dapibus vulputate eget eget lorem. Nulla tempus lacus a lorem varius dapibus. Etiam nulla turpis, tincidunt sit amet luctus in, pellentesque eget erat. Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Aliquam facilisis lorem ut elementum ornare.

tests/lorem-ipsum-samples/11

@@ -0,0 +1 @@
+Quisque porttitor, lacus eu porta rutrum, eros est sollicitudin purus, et pellentesque augue libero fringilla quam. Nulla mattis est quis purus tristique, vitae dictum metus ullamcorper. Donec sed congue ligula, eget blandit metus. Vestibulum sagittis facilisis velit id tempus. Sed varius ut turpis sed fermentum. Mauris gravida sapien volutpat aliquet dignissim. Integer placerat tellus a nibh convallis varius.

tests/lorem-ipsum-samples/12

@@ -0,0 +1 @@
+Duis nec mollis ipsum. Integer quis pulvinar felis, sit amet euismod augue. Morbi eget consectetur tortor. Quisque vel magna sed libero tincidunt commodo sed nec elit. Nullam libero quam, feugiat sed urna quis, accumsan elementum libero. Aenean eget mi auctor, eleifend dolor id, semper mi. Integer efficitur magna urna, in aliquet libero ultricies ut.

tests/lorem-ipsum-samples/13

@@ -0,0 +1 @@
+Aenean at elit quam. In dapibus iaculis cursus. Aenean nisi est, auctor ut eros sit amet, ornare commodo justo. Praesent feugiat leo metus, ac fringilla felis rutrum vitae. Fusce placerat turpis eu nunc consequat egestas. Nulla pharetra sapien convallis risus molestie, pretium vestibulum tellus maximus. Donec porta libero sapien, quis dictum felis molestie nec. Vestibulum suscipit posuere magna, in suscipit odio porttitor finibus. Duis justo elit, lacinia non turpis in, euismod iaculis nibh. Ut sit amet velit dui. Morbi nec tristique sapien.

tests/lorem-ipsum-samples/14

@@ -0,0 +1 @@
+Morbi pretium eu magna sed placerat. Duis auctor eros at mi eleifend dignissim. Curabitur sed lorem nec nisi suscipit posuere. Ut ornare imperdiet quam, ut elementum massa feugiat interdum. Maecenas non augue in mauris tincidunt imperdiet. Morbi a cursus odio. Donec magna lacus, efficitur nec mattis at, blandit vitae ipsum. Aenean dictum rhoncus semper. Aliquam erat volutpat. Cras eget eros vel lorem scelerisque ultricies. Quisque id dolor dolor. Sed tristique consequat odio, eu dictum erat tincidunt ac.

tests/lorem-ipsum-samples/15

@@ -0,0 +1 @@
+Curabitur nec ante id elit ultrices elementum ut eget leo. Aliquam in purus mollis, molestie odio at, luctus arcu. Donec at justo scelerisque, pellentesque felis vel, convallis felis. Etiam interdum arcu lectus, non venenatis tellus fringilla eu. Fusce sit amet mattis neque. Proin lobortis aliquet dolor, et efficitur dui elementum at. Phasellus aliquet eget nisi vitae tempor. Pellentesque tempor commodo risus at volutpat.

tests/lorem-ipsum-samples/16

@@ -0,0 +1 @@
+Morbi et mauris bibendum, placerat metus vitae, mollis tellus. Pellentesque tempus tincidunt neque, a ullamcorper velit vehicula ac. Morbi varius tortor in ante ultricies, non facilisis urna egestas. Curabitur et tortor odio. Suspendisse pellentesque sapien et felis iaculis, a accumsan velit suscipit. Sed at dapibus urna. Donec sed nisi eu dui volutpat sodales vitae vel nisl.

tests/lorem-ipsum-samples/17

@@ -0,0 +1 @@
+Quisque eget dui ac dui vehicula efficitur. Donec pharetra arcu ante, nec porttitor eros interdum sit amet. Aenean interdum urna nulla, a pellentesque est fringilla eget. Maecenas luctus nisl at lacus efficitur ultrices. Suspendisse rutrum cursus tellus, eu sagittis tortor euismod a. Quisque sit amet leo lacus. In auctor fringilla iaculis. Nullam at erat orci. Cras nisl ex, egestas et ex id, condimentum tincidunt justo. Vivamus vestibulum malesuada vehicula. Maecenas pharetra dui diam, et interdum ligula molestie in. Phasellus metus metus, mattis at maximus et, sagittis nec erat. Mauris quis nisl ex.

tests/lorem-ipsum-samples/18

@@ -0,0 +1 @@
+Sed aliquam massa in volutpat mollis. Pellentesque imperdiet libero et blandit rutrum. Cras aliquet orci vitae convallis aliquet. Pellentesque viverra lobortis elit a pulvinar. Ut accumsan enim nibh, at sodales arcu sagittis at. Class aptent taciti sociosqu ad litora torquent per conubia nostra, per inceptos himenaeos. Curabitur aliquet tempus condimentum. Morbi ac volutpat mi. Suspendisse potenti. Etiam ut congue ipsum. Mauris convallis vehicula felis, at condimentum neque suscipit vitae. Phasellus bibendum faucibus sollicitudin. Interdum et malesuada fames ac ante ipsum primis in faucibus. Praesent faucibus elit nisi, non condimentum enim volutpat at.

tests/lorem-ipsum-samples/2

@@ -0,0 +1 @@
+Ea perspiciatis tenetur ea. Ea vel sequi id autem dignissimos. Ipsam natus magnam omnis non excepturi doloremque consequuntur.

tests/lorem-ipsum-samples/3

@@ -0,0 +1 @@
+Quisquam voluptatem sit doloribus. Tempora libero aut et et omnis iure blanditiis labore. Omnis et ullam et quas officiis et voluptatem.

tests/lorem-ipsum-samples/4

@@ -0,0 +1 @@
+Qui dolores dolorem autem officia. Molestiae iusto qui culpa saepe sequi corrupti. Et debitis aliquid expedita necessitatibus reprehenderit id cupiditate. Sequi cum aperiam nostrum aliquam. Sit voluptas facere est quis nostrum omnis.

tests/lorem-ipsum-samples/5

@@ -0,0 +1 @@
+Ipsum culpa quas quibusdam dolores autem. Ipsam enim porro natus illum molestiae assumenda. Quae qui labore non dolores et. Voluptate voluptas expedita repellendus.

tests/lorem-ipsum-samples/6

@@ -0,0 +1 @@
+Curabitur orci ligula, maximus et viverra eu, gravida eu purus. Nulla viverra posuere dictum. Cras posuere augue sit amet lorem volutpat tempor. Vivamus sit amet mattis magna. Cras mollis nisi ac venenatis volutpat. Pellentesque in semper velit. Interdum et malesuada fames ac ante ipsum primis in faucibus. Sed pulvinar in dui id condimentum. Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Sed tortor arcu, feugiat egestas risus vitae, consectetur accumsan lacus. Integer id augue libero. Maecenas volutpat rhoncus mauris, ac tempor magna faucibus sit amet. Aliquam eget justo vel nunc finibus posuere at eget augue. Fusce sollicitudin blandit eros, in tempus est lobortis a. Mauris sodales fermentum maximus.

tests/lorem-ipsum-samples/7

@@ -0,0 +1 @@
+Nullam vitae metus massa. Morbi blandit massa eu orci placerat, quis tempus massa imperdiet. Etiam ut posuere sapien, et blandit lectus. Phasellus at pharetra velit. Aenean non ipsum in lorem egestas auctor vel a justo. Donec interdum aliquam scelerisque. Suspendisse arcu eros, fermentum sit amet arcu nec, placerat hendrerit orci. Vestibulum faucibus et odio quis pharetra.

tests/lorem-ipsum-samples/8

@@ -0,0 +1 @@
+Maecenas tincidunt auctor vehicula. Nulla ut nunc tincidunt, iaculis lacus ut, blandit purus. Maecenas sit amet blandit leo, non tempor est. Phasellus molestie urna ac quam cursus, eget lobortis risus finibus. Nulla at porttitor erat. Curabitur nec tortor dui. Cras facilisis est id nisi rhoncus, eget dictum lectus feugiat. Aenean in urna congue, rhoncus felis id, fermentum ex. Aliquam lacinia tellus et justo consectetur maximus. Ut augue justo, cursus dictum consequat suscipit, sollicitudin eu tortor. Curabitur justo dui, condimentum non nulla ut, blandit consectetur mauris. Sed feugiat cursus lacus, in sodales enim placerat in. Donec sit amet nunc eget arcu varius lobortis a ut felis. Mauris vel sem sit amet dolor ultrices iaculis.

tests/lorem-ipsum-samples/9

@@ -0,0 +1 @@
+In quis lorem in dui convallis fermentum in a quam. Class aptent taciti sociosqu ad litora torquent per conubia nostra, per inceptos himenaeos. Aliquam et tristique tellus. Vestibulum non nulla fermentum, euismod tellus a, faucibus tortor. Aliquam sed arcu nec nisl fringilla fermentum vitae sagittis sem. Suspendisse ac varius massa. Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Nunc porttitor metus venenatis ante venenatis, vitae venenatis nunc pellentesque. Cras consectetur molestie erat a tincidunt. Integer mattis nulla sem. Etiam consectetur odio vitae neque aliquet, in imperdiet orci feugiat. Ut nec faucibus elit, et feugiat nunc.

tests/simple_cases_test.ts

@@ -43,4 +43,70 @@ Deno.test("Decompression of 1 000 000 nuls", () => {
   const arraySum = decompressed.reduce((accumulator, currentValue) => accumulator + currentValue);
   assertEquals(1_000_000, decompressed.length);
   assertEquals(0, arraySum);
+});
+
+Deno.test("Decompression of Lorem ipsum text using a dictionary", () => {
+  const dictionaryBytes = [
+    0x37, 0xa4, 0x30, 0xec, 0x33, 0x4b, 0xd1, 0x42, 0x2a, 0x10, 0xc8, 0x92,
+    0x24, 0x1d, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xbf, 0x6c, 0x26, 0x52,
+    0x6e, 0xd9, 0xc8, 0x96, 0xb7, 0x41, 0xd2, 0x36, 0x2b, 0xf7, 0x86, 0xf9,
+    0x7f, 0x1a, 0xa9, 0x95, 0x45, 0xa1, 0xcf, 0xc6, 0xf3, 0xed, 0xe5, 0x7a,
+    0x55, 0xc6, 0x01, 0x03, 0x00, 0x00, 0x60, 0x90, 0xe0, 0x40, 0x32, 0xcb,
+    0xde, 0x02, 0x00, 0x00, 0x24, 0xa0, 0xd0, 0x8e, 0x0c, 0x8b, 0x09, 0x08,
+    0x87, 0x69, 0x10, 0x03, 0x19, 0x42, 0x0c, 0x21, 0x84, 0x10, 0x43, 0x08,
+    0x21, 0x84, 0x10, 0x42, 0x08, 0x21, 0x64, 0xd2, 0x88, 0x88, 0x24, 0x49,
+    0x92, 0x0e, 0x04, 0x69, 0x64, 0xc2, 0xc5, 0xc3, 0xa2, 0x22, 0xd1, 0x54,
+    0x10, 0x10, 0xc7, 0x82, 0x2c, 0x12, 0x52, 0x50, 0x4a, 0xa7, 0x6c, 0x24,
+    0x49, 0x92, 0x0e, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x08,
+    0x00, 0x00, 0x00, 0x73, 0x20, 0x61, 0x75, 0x63, 0x74, 0x6f, 0x72, 0x20,
+    0x65, 0x20, 0x65, 0x74, 0x20, 0x65, 0x66, 0x66, 0x69, 0x63, 0x69, 0x74,
+    0x75, 0x72, 0x20, 0x64, 0x75, 0x69, 0x20, 0x65, 0x6c, 0x65, 0x6d, 0x65,
+    0x6e, 0x74, 0x75, 0x6d, 0x20, 0x61, 0x74, 0x2e, 0x20, 0x50, 0x68, 0x61,
+    0x73, 0x65, 0x6c, 0x6c, 0x75, 0x73, 0x20, 0x61, 0x6c, 0x69, 0x71, 0x75,
+    0x65, 0x74, 0x20, 0x65, 0x73, 0x75, 0x61, 0x64, 0x61, 0x20, 0x66, 0x61,
+    0x6d, 0x65, 0x73, 0x20, 0x61, 0x63, 0x20, 0x74, 0x75, 0x72, 0x70, 0x69,
+    0x73, 0x20, 0x65, 0x67, 0x65, 0x73, 0x74, 0x61, 0x73, 0x2e, 0x20, 0x41,
+    0x6c, 0x69, 0x71, 0x75, 0x61, 0x6d, 0x20, 0x66, 0x61, 0x63, 0x69, 0x6c,
+    0x69, 0x73, 0x69, 0x73, 0x20, 0x6c, 0x20, 0x6c, 0x6f, 0x62, 0x6f, 0x72,
+    0x74, 0x69, 0x73, 0x20, 0x69, 0x70, 0x73, 0x75, 0x6d, 0x2e, 0x20, 0x53,
+    0x75, 0x73, 0x70, 0x65, 0x6e, 0x64, 0x69, 0x73, 0x73, 0x65, 0x20, 0x70,
+    0x6f, 0x72, 0x74, 0x74, 0x69, 0x74, 0x6f, 0x72, 0x2c, 0x20, 0x6c, 0x69,
+    0x67, 0x75, 0x6c, 0x61, 0x20, 0x61, 0x20, 0x61, 0x63, 0x75, 0x6c, 0x69,
+    0x73, 0x20, 0x6c, 0x61, 0x63, 0x75, 0x73, 0x20, 0x75, 0x74, 0x2c, 0x20,
+    0x62, 0x6c, 0x61, 0x6e, 0x64, 0x69, 0x74, 0x20, 0x70, 0x75, 0x72, 0x75,
+    0x73, 0x2e, 0x20, 0x4d, 0x61, 0x65, 0x63, 0x65, 0x6e, 0x61, 0x73, 0x20,
+    0x73, 0x69, 0x74, 0x20, 0x61, 0x6d, 0x65, 0x74, 0x20, 0x6e, 0x74, 0x20,
+    0x6d, 0x6f, 0x72, 0x62, 0x69, 0x20, 0x74, 0x72, 0x69, 0x73, 0x74, 0x69,
+  ];
+  const data = [
+    0x28, 0xb5, 0x2f, 0xfd, 0x03, 0x68, 0x33, 0x4b, 0xd1, 0x42, 0x65, 0x02,
+    0x00, 0xa3, 0x05, 0x0d, 0x07, 0xbb, 0x82, 0xfa, 0x96, 0xdb, 0x4a, 0x5d,
+    0x73, 0xbf, 0xd6, 0x6e, 0x7d, 0x69, 0xa9, 0x14, 0x5b, 0x3b, 0x7e, 0x4e,
+    0xb7, 0xbf, 0xb0, 0x1d, 0xa1, 0x7e, 0x9d, 0x52, 0xcd, 0xe3, 0x87, 0x53,
+    0xe1, 0x29, 0x9a, 0xa8, 0x25, 0x93, 0xe9, 0x90, 0x11, 0x36, 0x65, 0xe8,
+    0xcb, 0x32, 0xc5, 0xd2, 0x17, 0x2e, 0x13, 0x42, 0x08, 0xfc, 0x1e, 0x33,
+    0xa8, 0xac, 0xa2, 0x97, 0x18, 0xfe, 0x6e, 0xde, 0xf4, 0x34, 0xd8, 0xbc,
+    0x89, 0x4f, 0x51, 0xe7, 0x0b,
+  ];
+  const compressed = new Uint8Array(data);
+  const dictionary = new Uint8Array(dictionaryBytes);
+  const expected = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nullam nec sem urna. Morbi mollis, massa a convallis iaculis, mauris neque.";
+  {
+    const decompressed = fzstd.decompress(compressed, undefined, dictionary);
+    const shouldBeLoremIpsum = new TextDecoder().decode(decompressed);
+    assertEquals(expected, shouldBeLoremIpsum);
+  }
+  {
+    let outputChunks: Uint8Array[] = [];
+    const decompressor = new fzstd.Decompress((data) => outputChunks.push(data));
+    decompressor.loadDictionary(dictionary);
+    decompressor.push(compressed, true);
+    let output = new Uint8Array(outputChunks.reduce((acc, cur) => acc + cur.byteLength, 0));
+    for (let i = 0, offset = 0; i < outputChunks.length; i++) {
+      output.set(outputChunks[i], offset);
+      offset += outputChunks[i].length;
+    }
+    let shouldBeLoremIpsum = new TextDecoder().decode(output);
+    assertEquals(expected, shouldBeLoremIpsum);
+  }
 });