diff --git a/README.md b/README.md index 05bd058..13002af 100644 --- a/README.md +++ b/README.md @@ -93,14 +93,25 @@ Examples: ## Sync +Quick git-like sync (recommended): + +```bash +./sim_db push /path/to/remote.sqlite3 +./sim_db pull /path/to/remote.sqlite3 +``` + +- `push ` copies local changes to the remote DB path. +- `pull ` brings remote changes into your local DB. +- merge policy is per `job_id`: newer `updated_at` wins. + +Artifact-based sync (advanced/manual flow): + ```bash ./sim_db sync-status --table ./sim_db sync-export --out ./sync-out.json ./sim_db sync-import --in ./sync-out.json ``` -Sync merge policy is per `job_id`: newer `updated_at` wins. - ## Tests ```bash diff --git a/sim_db.py b/sim_db.py index 4d5e0eb..891ec0c 100755 --- a/sim_db.py +++ b/sim_db.py @@ -12,6 +12,7 @@ import socket import sqlite3 import sys +import tempfile import threading import webbrowser from datetime import datetime @@ -581,7 +582,7 @@ def sync_status(db_path: str = DEFAULT_DB_PATH) -> dict[str, Any]: conn.close() -def sync_export(db_path: str, out_path: str, include_all: bool = False) -> dict[str, Any]: +def sync_export(db_path: str, out_path: str, include_all: bool = False, mark_synced: bool = True) -> dict[str, Any]: conn, sqlite_path = _connect_db(db_path) exported_at = _now_iso() source_host = socket.gethostname() @@ -594,11 +595,12 @@ def sync_export(db_path: str, out_path: str, include_all: bool = False) -> dict[ out_file = Path(out_path).expanduser() out_file.parent.mkdir(parents=True, exist_ok=True) out_file.write_text(json.dumps(artifact, indent=2, ensure_ascii=False, sort_keys=True) + '\n', encoding='utf-8') - for row in rows: - job_id = str(row.get('job_id', '')) - if job_id: - conn.execute('INSERT OR REPLACE INTO sim_sync_state(job_id, last_synced_updated_at, last_exported_at, last_imported_at) VALUES (?, ?, ?, COALESCE((SELECT last_imported_at FROM sim_sync_state WHERE job_id = ?), \"\"))', (job_id, str(row.get('updated_at', '')), exported_at, job_id)) - conn.commit() + if mark_synced: + for row in rows: + job_id = str(row.get('job_id', '')) + if job_id: + conn.execute('INSERT OR REPLACE INTO sim_sync_state(job_id, last_synced_updated_at, last_exported_at, last_imported_at) VALUES (?, ?, ?, COALESCE((SELECT last_imported_at FROM sim_sync_state WHERE job_id = ?), \"\"))', (job_id, str(row.get('updated_at', '')), exported_at, job_id)) + conn.commit() return {'ok': True, 'path': str(out_file), 'exported': len(rows), 'exported_at': exported_at} finally: conn.close() @@ -911,6 +913,36 @@ def log_message(self, format: str, *args: Any) -> None: # noqa: A003 server.server_close() +def sync_push(db_path: str, remote: str) -> dict[str, Any]: + remote_db = os.path.expanduser(remote) + with tempfile.NamedTemporaryFile(prefix='mini_sim_db_push_', suffix='.json', delete=False) as tmp: + artifact_path = tmp.name + try: + sync_export(db_path, artifact_path, include_all=True, mark_synced=False) + out = sync_import(remote_db, artifact_path) + sync_export(db_path, artifact_path, include_all=False, mark_synced=True) + out['remote'] = remote_db + out['direction'] = 'push' + return out + finally: + Path(artifact_path).unlink(missing_ok=True) + + +def sync_pull(db_path: str, remote: str) -> dict[str, Any]: + remote_db = os.path.expanduser(remote) + with tempfile.NamedTemporaryFile(prefix='mini_sim_db_pull_', suffix='.json', delete=False) as tmp: + artifact_path = tmp.name + try: + sync_export(remote_db, artifact_path, include_all=True, mark_synced=False) + out = sync_import(db_path, artifact_path) + sync_export(remote_db, artifact_path, include_all=False, mark_synced=True) + out['remote'] = remote_db + out['direction'] = 'pull' + return out + finally: + Path(artifact_path).unlink(missing_ok=True) + + def _format_table(rows: list[dict[str, str]]) -> str: cols = ['case', 'status', 'job_id', 'bin', 'inp', 'updated_at', 'run_host', 'note'] widths = {c: len(c) for c in cols} @@ -933,6 +965,8 @@ def _build_cli() -> argparse.ArgumentParser: ' ./sim_db add --case case_001 --inp variant.inp --bin solver --status restart\n' ' ./sim_db done --job-id \n' ' ./sim_db list --table\n' + ' ./sim_db push /path/to/remote.sqlite3\n' + ' ./sim_db pull /path/to/remote.sqlite3\n' ), formatter_class=argparse.RawDescriptionHelpFormatter, ) @@ -1036,6 +1070,14 @@ def _build_cli() -> argparse.ArgumentParser: p_sync_import = sub.add_parser('sync-import', help='Import updates from a JSON sync artifact') p_sync_import.add_argument('--in', dest='in_path', required=True, help='Input JSON file path') p_sync_import.add_argument('--db', default=DEFAULT_DB_PATH, help='Path to DB (CSV path auto-maps to SQLite)') + + p_push = sub.add_parser('push', help='Push local updates to a remote DB path') + p_push.add_argument('remote', help='Remote DB path (for example /shared/remote.sqlite3)') + p_push.add_argument('--db', default=DEFAULT_DB_PATH, help='Local DB path (CSV path auto-maps to SQLite)') + + p_pull = sub.add_parser('pull', help='Pull updates from a remote DB path into local DB') + p_pull.add_argument('remote', help='Remote DB path (for example /shared/remote.sqlite3)') + p_pull.add_argument('--db', default=DEFAULT_DB_PATH, help='Local DB path (CSV path auto-maps to SQLite)') return parser @@ -1113,6 +1155,20 @@ def main(argv: list[str] | None = None) -> int: print('Conflicts:') for conflict in out['conflicts']: print(json.dumps(conflict, ensure_ascii=False, sort_keys=True)) + elif args.command == 'push': + out = sync_push(args.db, args.remote) + print(f"Pushed to {out['remote']}: {out['created']} created, {out['updated']} updated, {out['skipped']} unchanged") + if out['conflicts']: + print('Conflicts:') + for conflict in out['conflicts']: + print(json.dumps(conflict, ensure_ascii=False, sort_keys=True)) + elif args.command == 'pull': + out = sync_pull(args.db, args.remote) + print(f"Pulled from {out['remote']}: {out['created']} created, {out['updated']} updated, {out['skipped']} unchanged") + if out['conflicts']: + print('Conflicts:') + for conflict in out['conflicts']: + print(json.dumps(conflict, ensure_ascii=False, sort_keys=True)) else: parser.print_help() return 1 diff --git a/test_sim_db.py b/test_sim_db.py index 88cdefe..d549a4d 100644 --- a/test_sim_db.py +++ b/test_sim_db.py @@ -6,7 +6,7 @@ import time import unittest -from sim_db import _view_payload, add_sim_item, derive_job_id, find_items, import_csv, init_sim_db, list_items, list_view, mark_done, mark_start, resolve_job_id, sync_export, sync_import, sync_status +from sim_db import _view_payload, add_sim_item, derive_job_id, find_items, import_csv, init_sim_db, list_items, list_view, mark_done, mark_start, resolve_job_id, sync_export, sync_import, sync_pull, sync_push, sync_status class TestSimpleCliFunctions(unittest.TestCase): @@ -141,6 +141,23 @@ def test_cli_rejects_ambiguous_case_done(self): self.assertNotEqual(done.returncode, 0) self.assertIn('matches multiple rows', done.stderr) + def test_cli_push_and_pull_with_remote_path(self): + local_db = os.path.join(self.home_dir, 'local.sqlite3') + remote_db = os.path.join(self.home_dir, 'remote.sqlite3') + self.assertEqual(self._run('init', '--db', local_db).returncode, 0) + self.assertEqual(self._run('add', '--db', local_db, '--case', 'c-local', '--inp', 'a.inp', '--bin', 'solver', '--status', 'start').returncode, 0) + pushed = self._run('push', '--db', local_db, remote_db) + self.assertEqual(pushed.returncode, 0) + self.assertIn('Pushed to', pushed.stdout) + + self.assertEqual(self._run('add', '--db', remote_db, '--case', 'c-remote', '--inp', 'b.inp', '--bin', 'solver', '--status', 'restart').returncode, 0) + pulled = self._run('pull', '--db', local_db, remote_db) + self.assertEqual(pulled.returncode, 0) + self.assertIn('Pulled from', pulled.stdout) + listed = self._run('list', '--db', local_db) + self.assertIn('c-local', listed.stdout) + self.assertIn('c-remote', listed.stdout) + class TestLocalSync(unittest.TestCase): def setUp(self): @@ -164,6 +181,25 @@ def test_sync_export_and_import(self): imported = sync_import(self.db_path, self.sync_file) self.assertEqual(imported['skipped'], 1) + def test_sync_push_and_pull(self): + remote_db = os.path.join(self.tmp_dir.name, 'remote.sqlite3') + init_sim_db(remote_db) + + add_sim_item(case='local-case', inp='a.inp', bin_name='solver', status='start', db_path=self.db_path) + pushed = sync_push(self.db_path, remote_db) + self.assertEqual(pushed['created'], 1) + self.assertEqual(sync_status(self.db_path)['pending_cases'], 0) + + add_sim_item(case='remote-case', inp='b.inp', bin_name='solver', status='restart', db_path=remote_db) + self.assertEqual(sync_status(remote_db)['pending_cases'], 1) + pulled = sync_pull(self.db_path, remote_db) + self.assertEqual(pulled['created'], 1) + self.assertEqual(sync_status(remote_db)['pending_cases'], 0) + + rows = list_view(self.db_path) + names = {row['case'] for row in rows} + self.assertEqual(names, {'local-case', 'remote-case'}) + if __name__ == '__main__': unittest.main()