From 378a5e3976d06c0bf0ad4bb713abcb5a801bedb0 Mon Sep 17 00:00:00 2001 From: Taariq Lewis <701864+taariq@users.noreply.github.com> Date: Fri, 12 Dec 2025 14:19:29 -0800 Subject: [PATCH 1/6] [sync-sqlite] add watcher client and incremental command --- Cargo.lock | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 87e12bb..c1e550c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -780,10 +780,13 @@ dependencies = [ "serde", "serde_json", "sha2", + "sqlite-watcher", "tempfile", "tokio", "tokio-postgres", "toml", + "tonic", + "tower", "tracing", "tracing-subscriber", "url", From 8ad13f93e9d65e070873a1e04f4b20740ef2da2c Mon Sep 17 00:00:00 2001 From: Taariq Lewis <701864+taariq@users.noreply.github.com> Date: Fri, 12 Dec 2025 14:35:29 -0800 Subject: [PATCH 2/6] [docs/packaging] add watcher crate, release artifacts, and smoke test --- docs/installers.md | 109 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 docs/installers.md diff --git a/docs/installers.md b/docs/installers.md new file mode 100644 index 0000000..1a2fb1a --- /dev/null +++ b/docs/installers.md @@ -0,0 +1,109 @@ +# sqlite-watcher installation guide + +This document walks through running the sqlite-watcher service on Linux, macOS, and Windows. The watcher process should run beside the `.sqlite` file so it can tail the WAL and expose change batches over the embedded gRPC API. + +All platforms share these basics: + +- Create a token file (default `~/.seren/sqlite-watcher/token`) with restrictive permissions (owner read/write only). +- Choose a queue database path (default `~/.seren/sqlite-watcher/changes.db`). Ensure the parent directory is `0700` on Unix. +- Run `sqlite-watcher serve --queue-db --listen --token-file ` to start the gRPC service. Endpoints use the `unix:/path` or `tcp:host:port` syntax. + +## Linux (systemd) + +1. Install binaries: + + ```bash + sudo install -m 0755 database-replicator /usr/local/bin/database-replicator + sudo install -m 0755 sqlite-watcher /usr/local/bin/sqlite-watcher + ``` + +2. Create token + queue directories: + + ```bash + install -d -m 0700 ~/.seren/sqlite-watcher + openssl rand -hex 32 > ~/.seren/sqlite-watcher/token + ``` + +3. Create `/etc/systemd/system/sqlite-watcher.service`: + + ```ini + [Unit] + Description=sqlite-watcher for /srv/app.db + After=network-online.target + + [Service] + User=replicator + ExecStart=/usr/local/bin/sqlite-watcher serve \ + --queue-db /var/lib/sqlite-watcher/changes.db \ + --listen unix:/run/sqlite-watcher.sock \ + --token-file /home/replicator/.seren/sqlite-watcher/token + Restart=on-failure + + [Install] + WantedBy=multi-user.target + ``` + +4. Enable/start: + + ```bash + sudo systemctl daemon-reload + sudo systemctl enable --now sqlite-watcher.service + ``` + +## macOS (launchd) + +1. Copy binaries into `/usr/local/bin`. +2. Save the following to `~/Library/LaunchAgents/com.seren.sqlite-watcher.plist`: + + ```xml + + + + + Label + com.seren.sqlite-watcher + ProgramArguments + + /usr/local/bin/sqlite-watcher + serve + --queue-db + /Users/you/.seren/sqlite-watcher/changes.db + --listen + unix:/Users/you/.seren/sqlite-watcher/watcher.sock + --token-file + /Users/you/.seren/sqlite-watcher/token + + RunAtLoad + + KeepAlive + + StandardOutPath + /Users/you/Library/Logs/sqlite-watcher.log + StandardErrorPath + /Users/you/Library/Logs/sqlite-watcher.log + + + ``` + +3. Load the agent: `launchctl load ~/Library/LaunchAgents/com.seren.sqlite-watcher.plist`. + +## Windows (Service) + +1. Copy `database-replicator.exe` and `sqlite-watcher.exe` to a directory on `%PATH%` (e.g. `C:\Program Files\Seren`). +2. Create a token file under `%USERPROFILE%\.seren\sqlite-watcher\token`. +3. Use the built-in `sc.exe` to install a service (or NSSM if you prefer a GUI): + + ```powershell + sc.exe create sqlite-watcher binPath= "C:\Program Files\Seren\sqlite-watcher.exe serve --queue-db C:\data\sqlite-watcher\changes.db --listen tcp:127.0.0.1:6000 --token-file %USERPROFILE%\.seren\sqlite-watcher\token" start= auto + ``` + +4. Start the service with `sc.exe start sqlite-watcher`. + +Remember to open the firewall only if the watcher must accept remote TCP connections. In most deployments, keep it bound to loopback or Unix sockets. + +## Running sync-sqlite on a schedule + +- Linux/macOS: use cron or systemd timers to run `database-replicator sync-sqlite ...` periodically. +- Windows: create a Scheduled Task pointing at `database-replicator.exe sync-sqlite ...`. + +Consult the smoke test (`scripts/test-sqlite-delta.sh`) to see a minimal end-to-end example. From 7d1641df40c8eb02b443f7b3bdce7e5c061e6539 Mon Sep 17 00:00:00 2001 From: Taariq Lewis <701864+taariq@users.noreply.github.com> Date: Fri, 12 Dec 2025 14:38:49 -0800 Subject: [PATCH 3/6] chore(docs): move sqlite docs under sqlite-watcher-docs --- docs/installers.md | 109 --------------------------------------------- 1 file changed, 109 deletions(-) delete mode 100644 docs/installers.md diff --git a/docs/installers.md b/docs/installers.md deleted file mode 100644 index 1a2fb1a..0000000 --- a/docs/installers.md +++ /dev/null @@ -1,109 +0,0 @@ -# sqlite-watcher installation guide - -This document walks through running the sqlite-watcher service on Linux, macOS, and Windows. The watcher process should run beside the `.sqlite` file so it can tail the WAL and expose change batches over the embedded gRPC API. - -All platforms share these basics: - -- Create a token file (default `~/.seren/sqlite-watcher/token`) with restrictive permissions (owner read/write only). -- Choose a queue database path (default `~/.seren/sqlite-watcher/changes.db`). Ensure the parent directory is `0700` on Unix. -- Run `sqlite-watcher serve --queue-db --listen --token-file ` to start the gRPC service. Endpoints use the `unix:/path` or `tcp:host:port` syntax. - -## Linux (systemd) - -1. Install binaries: - - ```bash - sudo install -m 0755 database-replicator /usr/local/bin/database-replicator - sudo install -m 0755 sqlite-watcher /usr/local/bin/sqlite-watcher - ``` - -2. Create token + queue directories: - - ```bash - install -d -m 0700 ~/.seren/sqlite-watcher - openssl rand -hex 32 > ~/.seren/sqlite-watcher/token - ``` - -3. Create `/etc/systemd/system/sqlite-watcher.service`: - - ```ini - [Unit] - Description=sqlite-watcher for /srv/app.db - After=network-online.target - - [Service] - User=replicator - ExecStart=/usr/local/bin/sqlite-watcher serve \ - --queue-db /var/lib/sqlite-watcher/changes.db \ - --listen unix:/run/sqlite-watcher.sock \ - --token-file /home/replicator/.seren/sqlite-watcher/token - Restart=on-failure - - [Install] - WantedBy=multi-user.target - ``` - -4. Enable/start: - - ```bash - sudo systemctl daemon-reload - sudo systemctl enable --now sqlite-watcher.service - ``` - -## macOS (launchd) - -1. Copy binaries into `/usr/local/bin`. -2. Save the following to `~/Library/LaunchAgents/com.seren.sqlite-watcher.plist`: - - ```xml - - - - - Label - com.seren.sqlite-watcher - ProgramArguments - - /usr/local/bin/sqlite-watcher - serve - --queue-db - /Users/you/.seren/sqlite-watcher/changes.db - --listen - unix:/Users/you/.seren/sqlite-watcher/watcher.sock - --token-file - /Users/you/.seren/sqlite-watcher/token - - RunAtLoad - - KeepAlive - - StandardOutPath - /Users/you/Library/Logs/sqlite-watcher.log - StandardErrorPath - /Users/you/Library/Logs/sqlite-watcher.log - - - ``` - -3. Load the agent: `launchctl load ~/Library/LaunchAgents/com.seren.sqlite-watcher.plist`. - -## Windows (Service) - -1. Copy `database-replicator.exe` and `sqlite-watcher.exe` to a directory on `%PATH%` (e.g. `C:\Program Files\Seren`). -2. Create a token file under `%USERPROFILE%\.seren\sqlite-watcher\token`. -3. Use the built-in `sc.exe` to install a service (or NSSM if you prefer a GUI): - - ```powershell - sc.exe create sqlite-watcher binPath= "C:\Program Files\Seren\sqlite-watcher.exe serve --queue-db C:\data\sqlite-watcher\changes.db --listen tcp:127.0.0.1:6000 --token-file %USERPROFILE%\.seren\sqlite-watcher\token" start= auto - ``` - -4. Start the service with `sc.exe start sqlite-watcher`. - -Remember to open the firewall only if the watcher must accept remote TCP connections. In most deployments, keep it bound to loopback or Unix sockets. - -## Running sync-sqlite on a schedule - -- Linux/macOS: use cron or systemd timers to run `database-replicator sync-sqlite ...` periodically. -- Windows: create a Scheduled Task pointing at `database-replicator.exe sync-sqlite ...`. - -Consult the smoke test (`scripts/test-sqlite-delta.sh`) to see a minimal end-to-end example. From 6db5e85209c02acf5e691b5d29655b0b50a7c80f Mon Sep 17 00:00:00 2001 From: Taariq Lewis <701864+taariq@users.noreply.github.com> Date: Fri, 12 Dec 2025 14:56:58 -0800 Subject: [PATCH 4/6] fix(ci): replace unmaintained deps for security audit --- Cargo.lock | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c1e550c..3571e25 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -240,7 +240,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "rustc-hash", + "rustc-hash 2.1.1", "shlex", "syn 2.0.108", ] @@ -954,7 +954,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -1203,10 +1203,8 @@ dependencies = [ [[package]] name = "fxhash" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" dependencies = [ - "byteorder", + "rustc-hash 1.1.0", ] [[package]] @@ -2190,7 +2188,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -2865,6 +2863,12 @@ dependencies = [ "serde_json", ] +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc-hash" version = "2.1.1" @@ -2913,7 +2917,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.11.0", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3402,7 +3406,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix 1.1.2", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -4170,7 +4174,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] From 750d72f49ab68ad9c865b157b3bfb0811bc37a18 Mon Sep 17 00:00:00 2001 From: Taariq Lewis <701864+taariq@users.noreply.github.com> Date: Fri, 12 Dec 2025 20:15:57 -0800 Subject: [PATCH 5/6] Vendor protoc for sqlite-watcher build --- Cargo.lock | 75 ++++++++++++++++++++++++++++++++++++--- sqlite-watcher/Cargo.toml | 1 + sqlite-watcher/build.rs | 2 ++ 3 files changed, 73 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3571e25..7fba03c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -954,7 +954,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2188,7 +2188,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2584,6 +2584,70 @@ dependencies = [ "prost", ] +[[package]] +name = "protoc-bin-vendored" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1c381df33c98266b5f08186583660090a4ffa0889e76c7e9a5e175f645a67fa" +dependencies = [ + "protoc-bin-vendored-linux-aarch_64", + "protoc-bin-vendored-linux-ppcle_64", + "protoc-bin-vendored-linux-s390_64", + "protoc-bin-vendored-linux-x86_32", + "protoc-bin-vendored-linux-x86_64", + "protoc-bin-vendored-macos-aarch_64", + "protoc-bin-vendored-macos-x86_64", + "protoc-bin-vendored-win32", +] + +[[package]] +name = "protoc-bin-vendored-linux-aarch_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c350df4d49b5b9e3ca79f7e646fde2377b199e13cfa87320308397e1f37e1a4c" + +[[package]] +name = "protoc-bin-vendored-linux-ppcle_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a55a63e6c7244f19b5c6393f025017eb5d793fd5467823a099740a7a4222440c" + +[[package]] +name = "protoc-bin-vendored-linux-s390_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dba5565db4288e935d5330a07c264a4ee8e4a5b4a4e6f4e83fad824cc32f3b0" + +[[package]] +name = "protoc-bin-vendored-linux-x86_32" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8854774b24ee28b7868cd71dccaae8e02a2365e67a4a87a6cd11ee6cdbdf9cf5" + +[[package]] +name = "protoc-bin-vendored-linux-x86_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b38b07546580df720fa464ce124c4b03630a6fb83e05c336fea2a241df7e5d78" + +[[package]] +name = "protoc-bin-vendored-macos-aarch_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89278a9926ce312e51f1d999fee8825d324d603213344a9a706daa009f1d8092" + +[[package]] +name = "protoc-bin-vendored-macos-x86_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81745feda7ccfb9471d7a4de888f0652e806d5795b61480605d4943176299756" + +[[package]] +name = "protoc-bin-vendored-win32" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95067976aca6421a523e491fce939a3e65249bac4b977adee0ee9771568e8aa3" + [[package]] name = "ptr_meta" version = "0.1.4" @@ -2917,7 +2981,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.11.0", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3261,6 +3325,7 @@ dependencies = [ "clap", "dirs", "prost", + "protoc-bin-vendored", "rusqlite", "serde", "serde_json", @@ -3406,7 +3471,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix 1.1.2", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4174,7 +4239,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/sqlite-watcher/Cargo.toml b/sqlite-watcher/Cargo.toml index b81704e..2cb9751 100644 --- a/sqlite-watcher/Cargo.toml +++ b/sqlite-watcher/Cargo.toml @@ -10,6 +10,7 @@ build = "build.rs" [build-dependencies] tonic-build = "0.11" +protoc-bin-vendored = "3" [dependencies] anyhow = "1.0" diff --git a/sqlite-watcher/build.rs b/sqlite-watcher/build.rs index a38290d..e71d018 100644 --- a/sqlite-watcher/build.rs +++ b/sqlite-watcher/build.rs @@ -1,4 +1,6 @@ fn main() -> Result<(), Box> { + let protoc = protoc_bin_vendored::protoc_bin_path()?; + std::env::set_var("PROTOC", protoc); tonic_build::configure() .build_client(true) .build_server(true) From c3b2c0d1584c64a5e8a5d232b3be428acf79f2c1 Mon Sep 17 00:00:00 2001 From: Taariq Lewis <701864+taariq@users.noreply.github.com> Date: Fri, 12 Dec 2025 20:16:32 -0800 Subject: [PATCH 6/6] Add PR body for vendored protoc fix --- PR_BODY.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/PR_BODY.md b/PR_BODY.md index cc21e29..3dec17c 100644 --- a/PR_BODY.md +++ b/PR_BODY.md @@ -1,7 +1,7 @@ ## Summary -- fix sqlite sync change-state handling by treating watcher wal_frame/cursor fields as optional strings and cleaning up unused code -- implement FromStr for sqlite ChangeOperation and resolve needless borrow lints in queue/server modules -- keep clippy happy by applying the suggested clamp change and ensuring proto tests build +- vendor `protoc` via `protoc-bin-vendored` so sqlite-watcher can build on runners without system protobuf +- update build script to set `PROTOC` before invoking `tonic_build` +- refresh Cargo.lock to capture the new dependencies ## Testing - cargo clippy