Add WOFF2

Co-authored-by: Lucian Popescu <lucian.popescu@tecnico.ulisboa.pt>

Author: nunoplopes

.gitignore

@@ -0,0 +1,18 @@
+# Cargo
+target/
+Cargo.lock
+
+# Python
+*.pyc
+__pycache__
+
+# VSCode
+.vscode
+
+# Clangd
+.cache/
+
+tmp
+*.swp
+
+build

CMakeLists.txt

@@ -0,0 +1,71 @@
+cmake_minimum_required(VERSION 3.20)
+project(Cpp2Rust-TestSuite)
+
+set(CPP2RUST_PROGRAMS woff2)
+set(CPP2RUST_MODELS   unsafe refcount)
+set(CPP2RUST_SRCDIR   ${CMAKE_CURRENT_SOURCE_DIR}/../cpp2rust)
+set(CPP2RUST_BINARY   ${CPP2RUST_SRCDIR}/build/cpp2rust/cpp2rust)
+
+# -----------------------------------------------------------------------------
+
+include(${CPP2RUST_SRCDIR}/cmake/rust-toolchain.cmake)
+
+set(FILTER_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/scripts/filter_compile_commands.py)
+
+foreach(_prog IN LISTS CPP2RUST_PROGRAMS)
+  include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/${_prog}.cmake)
+
+  set(_model_regen_targets "")
+  set(_model_build_targets "")
+  foreach(_model IN LISTS CPP2RUST_MODELS)
+    write_rust_toolchain(${CMAKE_CURRENT_SOURCE_DIR}/${_prog}/out/${_model})
+
+    add_custom_target("build-${_prog}-${_model}"
+      COMMAND ${CMAKE_COMMAND} -E env RUSTFLAGS=-Awarnings cargo build --release
+      WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/${_prog}/out/${_model}"
+      USES_TERMINAL
+    )
+
+    list(APPEND _model_regen_targets "regen-${_prog}-${_model}")
+    list(APPEND _model_build_targets "build-${_prog}-${_model}")
+  endforeach()
+
+  add_custom_target("build-${_prog}"
+    DEPENDS "build-${_prog}-original" ${_model_build_targets}
+  )
+
+  add_custom_target("regen-${_prog}"
+    DEPENDS "build-${_prog}-original" ${_model_regen_targets}
+  )
+
+  add_custom_target("check-${_prog}"
+    COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/scripts/${_prog}-test.sh"
+            ${CMAKE_BINARY_DIR}
+            ${CMAKE_CURRENT_SOURCE_DIR}
+            ${CPP2RUST_MODELS}
+    DEPENDS "build-${_prog}"
+    USES_TERMINAL
+  )
+
+  add_custom_target("benchmark-${_prog}"
+    COMMAND python3
+            "${CPP2RUST_TESTS_DIR}/scripts/${_prog}_bench.py"
+            "${CMAKE_BINARY_DIR}"
+            --models ${CPP2RUST_MODELS}
+            --csv "${CMAKE_BINARY_DIR}/bench_${_prog}.csv"
+            --warmup 2
+            --runs 3
+    DEPENDS "build-${_prog}"
+    USES_TERMINAL
+  )
+endforeach()
+
+list(TRANSFORM CPP2RUST_PROGRAMS PREPEND "build-"     OUTPUT_VARIABLE _build_targets)
+list(TRANSFORM CPP2RUST_PROGRAMS PREPEND "check-"     OUTPUT_VARIABLE _check_targets)
+list(TRANSFORM CPP2RUST_PROGRAMS PREPEND "regen-"     OUTPUT_VARIABLE _regen_targets)
+list(TRANSFORM CPP2RUST_PROGRAMS PREPEND "benchmark-" OUTPUT_VARIABLE _bench_targets)
+
+add_custom_target("build"     DEPENDS ${_check_targets})
+add_custom_target("check"     DEPENDS ${_check_targets})
+add_custom_target("regen"     DEPENDS ${_regen_targets})
+add_custom_target("benchmark" DEPENDS ${_bench_targets})

README.md

@@ -0,0 +1,28 @@
+# Cpp2Rust Test Suite
+
+This repository contains open-source C/C++ programs that are known to be
+automatically translatable by Cpp2Rust to Rust.
+
+The CI bots of Cpp2Rust run these tests and update the expected output if
+needed.
+
+## Current set of included programs
+ - [WOFF2](https://github.com/google/woff2): TTF font compression/decompression
+
+
+## To build and run the benchmarks
+```
+mkdir build
+cd build
+cmake -GNinja ..
+```
+
+Available commands:
+ - `ninja build` - Compile the original and Rust programs.
+ - `ninja check` - Run the original and Rust binaries and compare the output.
+ - `ninja benchmark` - Compare the performance of the original and the Rust
+   binaries.
+ - `ninja regen` - Regenerate the Rust code using Cpp2Rust.
+
+Note: All commands can be appended with `-<prog>` to run for a specific program
+only (e.g., `ninja build-woff2`).

cmake/woff2.cmake

@@ -0,0 +1,58 @@
+set(WOFF2_SRC_DIR "${CMAKE_CURRENT_SOURCE_DIR}/woff2/src")
+set(WOFF2_OUT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/woff2/out")
+
+set(WOFF2_INFO_FILES       font.cc table_tags.cc variable_length.cc
+                           woff2_common.cc woff2_info.cc)
+set(WOFF2_COMPRESS_FILES   table_tags.cc variable_length.cc woff2_common.cc
+                           font.cc glyph.cc normalize.cc transform.cc
+                           woff2_enc.cc woff2_compress.cc)
+set(WOFF2_DECOMPRESS_FILES table_tags.cc variable_length.cc woff2_common.cc
+                           woff2_dec.cc woff2_out.cc woff2_decompress.cc)
+
+add_custom_target("build-woff2-original"
+  COMMAND ${CMAKE_COMMAND} -S ${WOFF2_SRC_DIR}
+                           -B ${WOFF2_SRC_DIR}/build
+                           -G Ninja
+                           -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
+  # Only the Makefile build links against the bundled Brotli
+  # Important so the compressed files match perfectly
+  COMMAND make
+  WORKING_DIRECTORY "${WOFF2_SRC_DIR}"
+  USES_TERMINAL
+)
+
+add_custom_target("config-woff2"
+  # Filter: split compile_commands.json per binary
+  COMMAND ${CMAKE_COMMAND} -E make_directory
+          ${WOFF2_SRC_DIR}/build/cmd_woff2_info
+          ${WOFF2_SRC_DIR}/build/cmd_woff2_compress
+          ${WOFF2_SRC_DIR}/build/cmd_woff2_decompress
+  COMMAND python3 ${FILTER_SCRIPT}
+          ${WOFF2_SRC_DIR}/build/compile_commands.json
+          ${WOFF2_SRC_DIR}/build/cmd_woff2_info/compile_commands.json
+          ${WOFF2_INFO_FILES}
+  COMMAND python3 ${FILTER_SCRIPT}
+          ${WOFF2_SRC_DIR}/build/compile_commands.json
+          ${WOFF2_SRC_DIR}/build/cmd_woff2_compress/compile_commands.json
+          ${WOFF2_COMPRESS_FILES}
+  COMMAND python3 ${FILTER_SCRIPT}
+          ${WOFF2_SRC_DIR}/build/compile_commands.json
+          ${WOFF2_SRC_DIR}/build/cmd_woff2_decompress/compile_commands.json
+          ${WOFF2_DECOMPRESS_FILES}
+)
+
+foreach(_model IN LISTS CPP2RUST_MODELS)
+  add_custom_target("regen-woff2-${_model}"
+    COMMAND ${CPP2RUST_BINARY} -dir ${WOFF2_SRC_DIR}/build/cmd_woff2_info
+            -model ${_model}
+            -o ${WOFF2_OUT_DIR}/${_model}/src/bin/woff2_info.rs
+    COMMAND ${CPP2RUST_BINARY} -dir ${WOFF2_SRC_DIR}/build/cmd_woff2_compress
+            -model ${_model}
+            -o ${WOFF2_OUT_DIR}/${_model}/src/bin/woff2_compress.rs
+    COMMAND ${CPP2RUST_BINARY} -dir ${WOFF2_SRC_DIR}/build/cmd_woff2_decompress
+            -model ${_model}
+            -o ${WOFF2_OUT_DIR}/${_model}/src/bin/woff2_decompress.rs
+    DEPENDS "config-woff2"
+    USES_TERMINAL
+  )
+endforeach()

scripts/filter_compile_commands.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+"""
+Filter compile_commands.json based on target patterns.
+
+Usage:
+    filter_compile_commands.py <input.json> <output.json> <pattern1> [pattern2] ...
+
+Patterns are matched against the 'file' field in compile_commands.json.
+A file is included if it matches ANY of the include patterns (substring match).
+Patterns starting with '!' are exclusion patterns - files matching them are excluded.
+Patterns starting with '~!' exclude entries where the command field contains the pattern.
+Patterns starting with '~' (without !) remove matching text from the command field.
+"""
+
+import json
+import sys
+
+
+def main():
+    if len(sys.argv) < 4:
+        print(f"Usage: {sys.argv[0]} <input.json> <output.json> <pattern1> [pattern2] ...", file=sys.stderr)
+        sys.exit(1)
+
+    input_file = sys.argv[1]
+    output_file = sys.argv[2]
+    all_patterns = sys.argv[3:]
+
+    include_patterns = [p for p in all_patterns if not p.startswith('!') and not p.startswith('~')]
+    exclude_patterns = [p[1:] for p in all_patterns if p.startswith('!') and not p.startswith('~')]
+    command_excludes = [p[2:] for p in all_patterns if p.startswith('~!')]
+    command_filters = [p[1:] for p in all_patterns if p.startswith('~') and not p.startswith('~!')]
+
+    with open(input_file, 'r') as f:
+        commands = json.load(f)
+
+    filtered = []
+    for cmd in commands:
+        file_path = cmd.get('file', '')
+        command = cmd.get('command', '')
+        if any(pattern in file_path for pattern in exclude_patterns):
+            continue
+        if any(pattern in command for pattern in command_excludes):
+            continue
+        if any(pattern in file_path for pattern in include_patterns):
+            if command_filters:
+                cmd = cmd.copy()
+                for cf in command_filters:
+                    command = command.replace(cf, '')
+                cmd['command'] = command
+            filtered.append(cmd)
+
+    with open(output_file, 'w') as f:
+        json.dump(filtered, f, indent=2)
+
+    print(f"Filtered {len(commands)} -> {len(filtered)} entries")
+
+
+if __name__ == '__main__':
+    main()

scripts/woff2-test.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+
+set -e
+
+# Usage: woff2-test.sh <build_dir> <source_dir> <model...>
+BUILD_DIR="$1"
+SRC_DIR="$2/woff2"
+shift 2
+MODELS=("$@")
+
+FONTS_DIR="$SRC_DIR/tests"
+TMP_DIR="$BUILD_DIR/tmp-woff2"
+
+rm -fr "$TMP_DIR"
+mkdir -p "$TMP_DIR/original"
+
+for f in "$FONTS_DIR"/**/*.ttf; do
+  cp "$f" "$TMP_DIR/original"
+done
+
+# Run original binaries
+for f in "$TMP_DIR"/original/*.ttf; do
+  "$SRC_DIR/src/woff2_compress" "$f" &
+done
+wait
+
+for f in "$TMP_DIR"/original/*.woff2; do
+  "$SRC_DIR/src/woff2_info" "$f" | tail -n2 > "$TMP_DIR/original/$(basename "$f" .woff2).info"
+done
+
+# Run each model and compare against original
+
+for model in "${MODELS[@]}"; do
+  RUST_BIN="$SRC_DIR/out/$model/target/release"
+  MODEL_DIR="$TMP_DIR/$model"
+
+  rm -fr "$MODEL_DIR"
+  mkdir -p "$MODEL_DIR"
+
+  cp "$TMP_DIR"/original/*.ttf "$MODEL_DIR"
+
+  for f in "$MODEL_DIR"/*.ttf; do
+    "$RUST_BIN"/woff2_compress "$f" &
+  done
+  wait
+
+  # Compare woff2 files against original
+  for f in "$MODEL_DIR"/*.woff2; do
+    base=$(basename "$f" .woff2)
+    diff "$f" "$TMP_DIR/original/$base.woff2" \
+      || { echo "FAIL [$model]: woff2 mismatch on $base"; exit 1; }
+  done
+
+  # Decompress and compare ttf roundtrip
+  for f in "$MODEL_DIR"/*.woff2; do
+    "$RUST_BIN"/woff2_decompress "$f" &
+  done
+  wait
+
+  for f in "$MODEL_DIR"/*.ttf; do
+    base=$(basename "$f" .ttf)
+    diff "$f" "$TMP_DIR/original/$base.ttf" \
+      || { echo "FAIL [$model]: ttf mismatch on $base"; exit 1; }
+  done
+
+  # Compare woff2_info output
+  for f in "$MODEL_DIR"/*.woff2; do
+    base=$(basename "$f" .woff2)
+    "$RUST_BIN"/woff2_info "$f" | tail -n2 > "$MODEL_DIR/$base.info"
+    diff "$MODEL_DIR/$base.info" "$TMP_DIR/original/$base.info" \
+      || { echo "FAIL [$model]: info mismatch on $base"; exit 1; }
+  done
+
+  echo "WOFF2 $model tests passed!"
+done
+
+rm -fr "$TMP_DIR"

woff2/out/refcount/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "woff2_refcount"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+brotli-sys = "0.3.2"
+libcc2rs = { version = "0.1.0", path = "../../../../cpp2rust/libcc2rs" }

woff2/out/refcount/rust-toolchain.toml

@@ -0,0 +1,2 @@
+[toolchain]
+channel = "1.95.0"