diff --git a/.changeset/base64-streams.md b/.changeset/base64-streams.md new file mode 100644 index 0000000..cf3491d --- /dev/null +++ b/.changeset/base64-streams.md @@ -0,0 +1,5 @@ +--- +"@hsblabs/web-stream-extras": minor +--- + +Add a `base64` subpath with streaming base64 and base64url encode/decode helpers. diff --git a/README.md b/README.md index 6ca285b..8cdf5b5 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ Use this package when you need to: - build or consume `ReadableStream` pipelines - collect byte streams into a single `Uint8Array` - convert between strings and `Uint8Array` +- encode and decode base64 or base64url streams - decode, encode, split, and join text streams - encrypt or decrypt byte streams with the Web Streams API @@ -155,6 +156,27 @@ const lines = await readAllChunks( ); ``` +### `@hsblabs/web-stream-extras/base64` + +The `base64` subpath provides streaming base64 and base64url codecs: + +- `Base64EncodeStream` +- `Base64DecodeStream` +- `encodeBase64Stream` +- `decodeBase64Stream` +- `encodeBase64UrlStream` +- `decodeBase64UrlStream` + +The encoder preserves chunk boundaries where it can while carrying incomplete 3-byte groups to the next chunk. `encodeBase64UrlStream()` omits padding by default. + +```ts +import { readAllChunks } from "@hsblabs/web-stream-extras"; +import { encodeBase64UrlStream } from "@hsblabs/web-stream-extras/base64"; + +const encoded = await readAllChunks(encodeBase64UrlStream(file.stream())); +const token = encoded.join(""); +``` + ### `webCryptoStream` example ```ts diff --git a/package.json b/package.json index 9105356..06b9473 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ ], "exports": { ".": "./dist/index.js", + "./base64": "./dist/base64.js", "./encryption": "./dist/encryption.js", "./jsonl": "./dist/jsonl.js", "./text": "./dist/text.js", diff --git a/src/base64.test.ts b/src/base64.test.ts new file mode 100644 index 0000000..531c879 --- /dev/null +++ b/src/base64.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, it } from "vitest"; +import { + Base64DecodeStream, + Base64EncodeStream, + decodeBase64Stream, + decodeBase64UrlStream, + encodeBase64Stream, + encodeBase64UrlStream, +} from "./base64"; +import { + binaryToString, + readAllBytes, + readAllChunks, + readableFromChunks, + stringToBinary, +} from "./index"; + +describe("base64", () => { + it("encodes byte chunks across 3-byte boundaries", async () => { + const encoded = await readAllChunks( + readableFromChunks([ + stringToBinary("he"), + stringToBinary("llo "), + stringToBinary("world"), + ]).pipeThrough(new Base64EncodeStream()), + ); + + expect(encoded.join("")).toBe("aGVsbG8gd29ybGQ="); + }); + + it("encodes base64url without padding by default", async () => { + const encoded = await readAllChunks( + encodeBase64UrlStream( + readableFromChunks(new Uint8Array([251, 255, 255])), + ), + ); + + expect(encoded.join("")).toBe("-___"); + }); + + it("can keep base64url padding when requested", async () => { + const encoded = await readAllChunks( + readableFromChunks(stringToBinary("hi")).pipeThrough( + new Base64EncodeStream({ variant: "base64url", padding: true }), + ), + ); + + expect(encoded.join("")).toBe("aGk="); + }); + + it("decodes base64 chunks across 4-character boundaries", async () => { + const decoded = await readAllBytes( + readableFromChunks(["aG", "Vsb", "G8gd29ybGQ="]).pipeThrough( + new Base64DecodeStream(), + ), + ); + + expect(binaryToString(decoded)).toBe("hello world"); + }); + + it("decodes base64url without padding", async () => { + const decoded = await readAllBytes( + decodeBase64UrlStream(readableFromChunks(["-_", "__"])), + ); + + expect(decoded).toEqual(new Uint8Array([251, 255, 255])); + }); + + it("ignores ASCII whitespace while decoding", async () => { + const decoded = await readAllBytes( + decodeBase64Stream(readableFromChunks(["aGVs\n", "bG8gd29y\r\nbGQ="])), + ); + + expect(binaryToString(decoded)).toBe("hello world"); + }); + + it("rejects malformed base64 input", async () => { + await expect( + readAllBytes(decodeBase64Stream(readableFromChunks("%%%"))), + ).rejects.toThrow(SyntaxError); + }); + + it("creates encoded streams from iterables", async () => { + const encoded = await readAllChunks( + encodeBase64Stream([stringToBinary("hello"), stringToBinary(" world")]), + ); + + expect(encoded.join("")).toBe("aGVsbG8gd29ybGQ="); + }); +}); diff --git a/src/base64.ts b/src/base64.ts new file mode 100644 index 0000000..9d11a95 --- /dev/null +++ b/src/base64.ts @@ -0,0 +1,15 @@ +export type { + Base64DecodeStreamOptions, + Base64EncodeStreamOptions, + Base64InputChunk, + Base64Variant, +} from "./base64/public"; + +export { + Base64DecodeStream, + Base64EncodeStream, + decodeBase64Stream, + decodeBase64UrlStream, + encodeBase64Stream, + encodeBase64UrlStream, +} from "./base64/public"; diff --git a/src/base64/public.ts b/src/base64/public.ts new file mode 100644 index 0000000..ad5b2f9 --- /dev/null +++ b/src/base64/public.ts @@ -0,0 +1,14 @@ +export type { + Base64DecodeStreamOptions, + Base64EncodeStreamOptions, + Base64InputChunk, + Base64Variant, +} from "./stream"; +export { + Base64DecodeStream, + Base64EncodeStream, + decodeBase64Stream, + decodeBase64UrlStream, + encodeBase64Stream, + encodeBase64UrlStream, +} from "./stream"; diff --git a/src/base64/stream.ts b/src/base64/stream.ts new file mode 100644 index 0000000..60df76b --- /dev/null +++ b/src/base64/stream.ts @@ -0,0 +1,279 @@ +import { toReadableStream } from "../shared/readable"; + +export type Base64Variant = "base64" | "base64url"; +export type Base64InputChunk = string | Uint8Array; + +export interface Base64EncodeStreamOptions { + variant?: Base64Variant; + padding?: boolean; +} + +export interface Base64DecodeStreamOptions { + variant?: Base64Variant; +} + +export class Base64EncodeStream extends TransformStream { + constructor(options: Base64EncodeStreamOptions = {}) { + const variant = options.variant ?? "base64"; + const padding = options.padding ?? variant === "base64"; + + let pending = new Uint8Array(); + + super({ + transform(chunk, controller) { + const input = + pending.length === 0 ? chunk : concatBytes(pending, chunk); + const encodeLength = input.length - (input.length % 3); + + if (encodeLength > 0) { + controller.enqueue( + formatBase64(encodeBase64Bytes(input.subarray(0, encodeLength)), { + variant, + padding, + }), + ); + } + + pending = input.slice(encodeLength); + }, + + flush(controller) { + if (pending.length > 0) { + controller.enqueue( + formatBase64(encodeBase64Bytes(pending), { variant, padding }), + ); + } + }, + }); + } +} + +export class Base64DecodeStream extends TransformStream< + Base64InputChunk, + Uint8Array +> { + constructor(options: Base64DecodeStreamOptions = {}) { + const variant = options.variant ?? "base64"; + const decoder = new TextDecoder(); + + let buffer = ""; + let finished = false; + + const decodeTextChunk = (chunk: Base64InputChunk): string => { + if (typeof chunk === "string") { + return chunk; + } + + return decoder.decode(chunk, { stream: true }); + }; + + super({ + transform(chunk, controller) { + const value = stripWhitespace(decodeTextChunk(chunk)); + + if (value === "") { + return; + } + if (finished) { + throw new SyntaxError("Base64 input has data after padding"); + } + + assertBase64Chars(value, variant); + buffer += value; + assertPaddingPlacement(buffer); + + while (buffer.length >= 4) { + const group = buffer.slice(0, 4); + + if (group.includes("=")) { + controller.enqueue(decodeBase64Group(group, variant)); + buffer = buffer.slice(4); + if (buffer !== "") { + throw new SyntaxError("Base64 input has data after padding"); + } + finished = true; + return; + } + + controller.enqueue(decodeBase64Group(group, variant)); + buffer = buffer.slice(4); + } + }, + + flush(controller) { + const value = stripWhitespace(decoder.decode()); + if (value !== "") { + if (finished) { + throw new SyntaxError("Base64 input has data after padding"); + } + assertBase64Chars(value, variant); + buffer += value; + assertPaddingPlacement(buffer); + } + + if (buffer === "") { + return; + } + if (buffer.length === 1) { + throw new SyntaxError("Base64 input has an incomplete final group"); + } + + controller.enqueue(decodeBase64Group(padBase64(buffer), variant)); + }, + }); + } +} + +export function encodeBase64Stream( + input: ReadableStream, + options?: Omit, +): ReadableStream; +export function encodeBase64Stream( + input: Iterable | AsyncIterable, + options?: Omit, +): ReadableStream; +export function encodeBase64Stream( + input: + | ReadableStream + | Iterable + | AsyncIterable, + options: Omit = {}, +): ReadableStream { + return toReadableStream(input).pipeThrough( + new Base64EncodeStream({ ...options, variant: "base64" }), + ); +} + +export function encodeBase64UrlStream( + input: ReadableStream, + options?: Omit, +): ReadableStream; +export function encodeBase64UrlStream( + input: Iterable | AsyncIterable, + options?: Omit, +): ReadableStream; +export function encodeBase64UrlStream( + input: + | ReadableStream + | Iterable + | AsyncIterable, + options: Omit = {}, +): ReadableStream { + return toReadableStream(input).pipeThrough( + new Base64EncodeStream({ + padding: false, + ...options, + variant: "base64url", + }), + ); +} + +export function decodeBase64Stream( + input: ReadableStream, + options?: Omit, +): ReadableStream { + return input.pipeThrough( + new Base64DecodeStream({ ...options, variant: "base64" }), + ); +} + +export function decodeBase64UrlStream( + input: ReadableStream, + options?: Omit, +): ReadableStream { + return input.pipeThrough( + new Base64DecodeStream({ ...options, variant: "base64url" }), + ); +} + +function encodeBase64Bytes(bytes: Uint8Array): string { + let binary = ""; + + for (const value of bytes) { + binary += String.fromCharCode(value); + } + + return btoa(binary); +} + +function decodeBase64Group(value: string, variant: Base64Variant): Uint8Array { + const base64 = variant === "base64url" ? toBase64(value) : value; + let binary: string; + + try { + binary = atob(base64); + } catch (cause) { + throw new SyntaxError("Base64 input is not valid", { cause }); + } + + const bytes = new Uint8Array(binary.length); + + for (let i = 0; i < binary.length; i++) { + bytes[i] = binary.charCodeAt(i); + } + + return bytes; +} + +function formatBase64( + value: string, + options: { variant: Base64Variant; padding: boolean }, +): string { + const encoded = + options.variant === "base64url" + ? value.replaceAll("+", "-").replaceAll("/", "_") + : value; + + return options.padding ? encoded : encoded.replace(/=+$/u, ""); +} + +function padBase64(value: string): string { + if (value.includes("=")) { + if (value.length % 4 !== 0) { + throw new SyntaxError("Base64 padding is not aligned to a 4-char group"); + } + return value; + } + + return value.padEnd(Math.ceil(value.length / 4) * 4, "="); +} + +function toBase64(value: string): string { + return value.replaceAll("-", "+").replaceAll("_", "/"); +} + +function stripWhitespace(value: string): string { + return value.replace(/[\t\n\f\r ]/gu, ""); +} + +function assertBase64Chars(value: string, variant: Base64Variant): void { + const valid = + variant === "base64" + ? /^[A-Za-z0-9+/=]*$/u.test(value) + : /^[A-Za-z0-9_\-=]*$/u.test(value); + + if (!valid) { + throw new SyntaxError("Base64 input contains invalid characters"); + } +} + +function assertPaddingPlacement(value: string): void { + const firstPadding = value.indexOf("="); + + if (firstPadding === -1) { + return; + } + if (/[^=]/u.test(value.slice(firstPadding))) { + throw new SyntaxError("Base64 input has data after padding"); + } + if (value.length - firstPadding > 2) { + throw new SyntaxError("Base64 input has too much padding"); + } +} + +function concatBytes(left: Uint8Array, right: Uint8Array): Uint8Array { + const output = new Uint8Array(left.length + right.length); + output.set(left, 0); + output.set(right, left.length); + return output; +} diff --git a/src/public-api.test.ts b/src/public-api.test.ts index 80e1fe9..cc8f800 100644 --- a/src/public-api.test.ts +++ b/src/public-api.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from "vitest"; +import * as base64Api from "./base64"; import * as rootApi from "./index"; import * as jsonlApi from "./jsonl"; import * as textApi from "./text"; @@ -44,3 +45,16 @@ describe("text public API", () => { ]); }); }); + +describe("base64 public API", () => { + it("only exports the supported base64 stream helpers", () => { + expect(Object.keys(base64Api).sort()).toEqual([ + "Base64DecodeStream", + "Base64EncodeStream", + "decodeBase64Stream", + "decodeBase64UrlStream", + "encodeBase64Stream", + "encodeBase64UrlStream", + ]); + }); +}); diff --git a/src/text/iterable.ts b/src/shared/readable.ts similarity index 100% rename from src/text/iterable.ts rename to src/shared/readable.ts diff --git a/src/text/codec.ts b/src/text/codec.ts index 8398e35..429061b 100644 --- a/src/text/codec.ts +++ b/src/text/codec.ts @@ -1,4 +1,4 @@ -import { toReadableStream } from "./iterable"; +import { toReadableStream } from "../shared/readable"; export type TextInputChunk = string | Uint8Array; diff --git a/src/text/lines.ts b/src/text/lines.ts index eed1672..ee72d78 100644 --- a/src/text/lines.ts +++ b/src/text/lines.ts @@ -1,5 +1,5 @@ +import { toReadableStream } from "../shared/readable"; import type { TextDecodeStreamOptions, TextInputChunk } from "./codec"; -import { toReadableStream } from "./iterable"; export interface LineSplitStreamOptions extends TextDecodeStreamOptions { keepLineBreaks?: boolean; diff --git a/tsdown.config.ts b/tsdown.config.ts index f34ead4..5f08113 100644 --- a/tsdown.config.ts +++ b/tsdown.config.ts @@ -1,7 +1,13 @@ import { defineConfig } from "tsdown"; export default defineConfig({ - entry: ["src/index.ts", "src/encryption.ts", "src/jsonl.ts", "src/text.ts"], + entry: [ + "src/index.ts", + "src/base64.ts", + "src/encryption.ts", + "src/jsonl.ts", + "src/text.ts", + ], outDir: "dist", target: "es2022", format: "esm",