Skip to content

Add git-like pull and push sync commands #12

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
hyharry merged 2 commits into from
Apr 11, 2026
Merged

Add git-like pull and push sync commands #12

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7cb17d2
Add git-like pull/push sync commands
hyharry Apr 11, 2026
41e1796
Advance sync state after pull and push
hyharry Apr 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 13 additions & 2 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,14 +93,25 @@ Examples:

## Sync

Quick git-like sync (recommended):

```bash
./sim_db push /path/to/remote.sqlite3
./sim_db pull /path/to/remote.sqlite3
```

- `push <remote>` copies local changes to the remote DB path.
- `pull <remote>` brings remote changes into your local DB.
- merge policy is per `job_id`: newer `updated_at` wins.

Artifact-based sync (advanced/manual flow):

```bash
./sim_db sync-status --table
./sim_db sync-export --out ./sync-out.json
./sim_db sync-import --in ./sync-out.json
```

Sync merge policy is per `job_id`: newer `updated_at` wins.

## Tests

```bash
Expand Down
68 changes: 62 additions & 6 deletions sim_db.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import socket
import sqlite3
import sys
import tempfile
import threading
import webbrowser
from datetime import datetime
Expand Down Expand Up @@ -581,7 +582,7 @@ def sync_status(db_path: str = DEFAULT_DB_PATH) -> dict[str, Any]:
conn.close()


def sync_export(db_path: str, out_path: str, include_all: bool = False) -> dict[str, Any]:
def sync_export(db_path: str, out_path: str, include_all: bool = False, mark_synced: bool = True) -> dict[str, Any]:
conn, sqlite_path = _connect_db(db_path)
exported_at = _now_iso()
source_host = socket.gethostname()
Expand All @@ -594,11 +595,12 @@ def sync_export(db_path: str, out_path: str, include_all: bool = False) -> dict[
out_file = Path(out_path).expanduser()
out_file.parent.mkdir(parents=True, exist_ok=True)
out_file.write_text(json.dumps(artifact, indent=2, ensure_ascii=False, sort_keys=True) + '\n', encoding='utf-8')
for row in rows:
job_id = str(row.get('job_id', ''))
if job_id:
conn.execute('INSERT OR REPLACE INTO sim_sync_state(job_id, last_synced_updated_at, last_exported_at, last_imported_at) VALUES (?, ?, ?, COALESCE((SELECT last_imported_at FROM sim_sync_state WHERE job_id = ?), \"\"))', (job_id, str(row.get('updated_at', '')), exported_at, job_id))
conn.commit()
if mark_synced:
for row in rows:
job_id = str(row.get('job_id', ''))
if job_id:
conn.execute('INSERT OR REPLACE INTO sim_sync_state(job_id, last_synced_updated_at, last_exported_at, last_imported_at) VALUES (?, ?, ?, COALESCE((SELECT last_imported_at FROM sim_sync_state WHERE job_id = ?), \"\"))', (job_id, str(row.get('updated_at', '')), exported_at, job_id))
conn.commit()
return {'ok': True, 'path': str(out_file), 'exported': len(rows), 'exported_at': exported_at}
finally:
conn.close()
Expand Down Expand Up @@ -911,6 +913,36 @@ def log_message(self, format: str, *args: Any) -> None: # noqa: A003
server.server_close()


def sync_push(db_path: str, remote: str) -> dict[str, Any]:
remote_db = os.path.expanduser(remote)
with tempfile.NamedTemporaryFile(prefix='mini_sim_db_push_', suffix='.json', delete=False) as tmp:
artifact_path = tmp.name
try:
sync_export(db_path, artifact_path, include_all=True, mark_synced=False)

@chatgpt-codex-connector chatgpt-codex-connector Bot Apr 11, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Mark source rows synced after successful push

sync_push calls sync_export(..., mark_synced=False), so even when the remote import succeeds the local sim_sync_state is never advanced. In practice, a row that was just pushed still appears in sync-status as pending, which can mislead operators and trigger redundant exports in automation that relies on pending counts. The same pattern is used in sync_pull for the remote source export, so source-side sync metadata is skipped in both directions.

Useful? React with 👍 / 👎.

out = sync_import(remote_db, artifact_path)
sync_export(db_path, artifact_path, include_all=False, mark_synced=True)
out['remote'] = remote_db
out['direction'] = 'push'
return out
finally:
Path(artifact_path).unlink(missing_ok=True)


def sync_pull(db_path: str, remote: str) -> dict[str, Any]:
remote_db = os.path.expanduser(remote)
with tempfile.NamedTemporaryFile(prefix='mini_sim_db_pull_', suffix='.json', delete=False) as tmp:
artifact_path = tmp.name
try:
sync_export(remote_db, artifact_path, include_all=True, mark_synced=False)
out = sync_import(db_path, artifact_path)
sync_export(remote_db, artifact_path, include_all=False, mark_synced=True)
out['remote'] = remote_db
out['direction'] = 'pull'
return out
finally:
Path(artifact_path).unlink(missing_ok=True)


def _format_table(rows: list[dict[str, str]]) -> str:
cols = ['case', 'status', 'job_id', 'bin', 'inp', 'updated_at', 'run_host', 'note']
widths = {c: len(c) for c in cols}
Expand All @@ -933,6 +965,8 @@ def _build_cli() -> argparse.ArgumentParser:
' ./sim_db add --case case_001 --inp variant.inp --bin solver --status restart\n'
' ./sim_db done --job-id <job_id>\n'
' ./sim_db list --table\n'
' ./sim_db push /path/to/remote.sqlite3\n'
' ./sim_db pull /path/to/remote.sqlite3\n'
),
formatter_class=argparse.RawDescriptionHelpFormatter,
)
Expand Down Expand Up @@ -1036,6 +1070,14 @@ def _build_cli() -> argparse.ArgumentParser:
p_sync_import = sub.add_parser('sync-import', help='Import updates from a JSON sync artifact')
p_sync_import.add_argument('--in', dest='in_path', required=True, help='Input JSON file path')
p_sync_import.add_argument('--db', default=DEFAULT_DB_PATH, help='Path to DB (CSV path auto-maps to SQLite)')

p_push = sub.add_parser('push', help='Push local updates to a remote DB path')
p_push.add_argument('remote', help='Remote DB path (for example /shared/remote.sqlite3)')
p_push.add_argument('--db', default=DEFAULT_DB_PATH, help='Local DB path (CSV path auto-maps to SQLite)')

p_pull = sub.add_parser('pull', help='Pull updates from a remote DB path into local DB')
p_pull.add_argument('remote', help='Remote DB path (for example /shared/remote.sqlite3)')
p_pull.add_argument('--db', default=DEFAULT_DB_PATH, help='Local DB path (CSV path auto-maps to SQLite)')
return parser


Expand Down Expand Up @@ -1113,6 +1155,20 @@ def main(argv: list[str] | None = None) -> int:
print('Conflicts:')
for conflict in out['conflicts']:
print(json.dumps(conflict, ensure_ascii=False, sort_keys=True))
elif args.command == 'push':
out = sync_push(args.db, args.remote)
print(f"Pushed to {out['remote']}: {out['created']} created, {out['updated']} updated, {out['skipped']} unchanged")
if out['conflicts']:
print('Conflicts:')
for conflict in out['conflicts']:
print(json.dumps(conflict, ensure_ascii=False, sort_keys=True))
elif args.command == 'pull':
out = sync_pull(args.db, args.remote)
print(f"Pulled from {out['remote']}: {out['created']} created, {out['updated']} updated, {out['skipped']} unchanged")
if out['conflicts']:
print('Conflicts:')
for conflict in out['conflicts']:
print(json.dumps(conflict, ensure_ascii=False, sort_keys=True))
else:
parser.print_help()
return 1
Expand Down
38 changes: 37 additions & 1 deletion test_sim_db.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import time
import unittest

from sim_db import _view_payload, add_sim_item, derive_job_id, find_items, import_csv, init_sim_db, list_items, list_view, mark_done, mark_start, resolve_job_id, sync_export, sync_import, sync_status
from sim_db import _view_payload, add_sim_item, derive_job_id, find_items, import_csv, init_sim_db, list_items, list_view, mark_done, mark_start, resolve_job_id, sync_export, sync_import, sync_pull, sync_push, sync_status


class TestSimpleCliFunctions(unittest.TestCase):
Expand Down Expand Up @@ -141,6 +141,23 @@ def test_cli_rejects_ambiguous_case_done(self):
self.assertNotEqual(done.returncode, 0)
self.assertIn('matches multiple rows', done.stderr)

def test_cli_push_and_pull_with_remote_path(self):
local_db = os.path.join(self.home_dir, 'local.sqlite3')
remote_db = os.path.join(self.home_dir, 'remote.sqlite3')
self.assertEqual(self._run('init', '--db', local_db).returncode, 0)
self.assertEqual(self._run('add', '--db', local_db, '--case', 'c-local', '--inp', 'a.inp', '--bin', 'solver', '--status', 'start').returncode, 0)
pushed = self._run('push', '--db', local_db, remote_db)
self.assertEqual(pushed.returncode, 0)
self.assertIn('Pushed to', pushed.stdout)

self.assertEqual(self._run('add', '--db', remote_db, '--case', 'c-remote', '--inp', 'b.inp', '--bin', 'solver', '--status', 'restart').returncode, 0)
pulled = self._run('pull', '--db', local_db, remote_db)
self.assertEqual(pulled.returncode, 0)
self.assertIn('Pulled from', pulled.stdout)
listed = self._run('list', '--db', local_db)
self.assertIn('c-local', listed.stdout)
self.assertIn('c-remote', listed.stdout)


class TestLocalSync(unittest.TestCase):
def setUp(self):
Expand All @@ -164,6 +181,25 @@ def test_sync_export_and_import(self):
imported = sync_import(self.db_path, self.sync_file)
self.assertEqual(imported['skipped'], 1)

def test_sync_push_and_pull(self):
remote_db = os.path.join(self.tmp_dir.name, 'remote.sqlite3')
init_sim_db(remote_db)

add_sim_item(case='local-case', inp='a.inp', bin_name='solver', status='start', db_path=self.db_path)
pushed = sync_push(self.db_path, remote_db)
self.assertEqual(pushed['created'], 1)
self.assertEqual(sync_status(self.db_path)['pending_cases'], 0)

add_sim_item(case='remote-case', inp='b.inp', bin_name='solver', status='restart', db_path=remote_db)
self.assertEqual(sync_status(remote_db)['pending_cases'], 1)
pulled = sync_pull(self.db_path, remote_db)
self.assertEqual(pulled['created'], 1)
self.assertEqual(sync_status(remote_db)['pending_cases'], 0)

rows = list_view(self.db_path)
names = {row['case'] for row in rows}
self.assertEqual(names, {'local-case', 'remote-case'})


if __name__ == '__main__':
unittest.main()
Loading