Open packaged SQLite DBs in read-only immutable mode to avoid locking errors under SLURM

singlem/metapackage_read_name_store.py

@@ -1,6 +1,8 @@
 import logging
 import tempfile
 import extern
+import os
+from urllib.parse import quote
 
 from sqlalchemy import create_engine
 from sqlalchemy.orm import registry, declarative_base
@@ -14,6 +16,14 @@
 Base = declarative_base()
 
 class MetapackageReadNameStore:
+    @staticmethod
+    def _acquire_readonly_engine(sqlitedb_path):
+        quoted_sqlite_path = quote(os.path.abspath(sqlitedb_path), safe="/")
+        sqlite_uri = f"sqlite+pysqlite:///file:{quoted_sqlite_path}?mode=ro&immutable=1&uri=true"
+        return create_engine(sqlite_uri,
+            echo=logging.getLogger().isEnabledFor(logging.DEBUG),
+            future=True)
+
     @staticmethod
     def generate(singlem_package_paths, sqlitedb_path, taxonomy_marker_counts=None):
         engine = create_engine("sqlite+pysqlite:///{}".format(sqlitedb_path),
@@ -61,9 +71,7 @@ def generate(singlem_package_paths, sqlitedb_path, taxonomy_marker_counts=None):
 
     @staticmethod
     def acquire(sqlitedb_path):
-        engine = create_engine("sqlite+pysqlite:///{}".format(sqlitedb_path),
-            echo=logging.getLogger().isEnabledFor(logging.DEBUG),
-            future=True)
+        engine = MetapackageReadNameStore._acquire_readonly_engine(sqlitedb_path)
 
         m = MetapackageReadNameStore()
         m.engine = engine
@@ -130,4 +138,4 @@ class TaxonomyMarkerCount(Base):
 
     id = Column(Integer, primary_key=True)
     taxonomy = Column(String, nullable=False, unique=True)
-    marker_count = Column(Integer, nullable=False)
+    marker_count = Column(Integer, nullable=False)

singlem/sequence_database.py

@@ -9,6 +9,7 @@
 import csv
 import extern
 import numpy as np
+from urllib.parse import quote
 
 from sqlalchemy import create_engine, select, distinct
 
@@ -54,6 +55,12 @@ class SequenceDatabase:
     _marker_cache = None
     _taxonomy_cache = None
 
+    @staticmethod
+    def _acquire_readonly_engine(sqlite_file):
+        quoted_sqlite_path = quote(os.path.abspath(sqlite_file), safe="/")
+        sqlite_uri = f"sqlite+pysqlite:///file:{quoted_sqlite_path}?mode=ro&immutable=1&uri=true"
+        return create_engine(sqlite_uri)
+
     def get_marker_via_cache(self, marker_id):
         if self._marker_cache is None:
             logging.info('Loading marker cache')
@@ -241,7 +248,7 @@ def acquire(path, min_version=None):
             raise Exception("Unexpected SingleM DB version found: {}".format(found_version))
 
         db.sqlite_file = os.path.join(path, SequenceDatabase.SQLITE_DB_NAME)
-        db.engine = create_engine("sqlite:///" + db.sqlite_file)
+        db.engine = SequenceDatabase._acquire_readonly_engine(db.sqlite_file)
         db.sqlalchemy_connection = db.engine.connect()
 
         nmslib_nucleotide_index_files = glob.glob("%s/nucleotide_indices_nmslib/*.nmslib_index" % path)

test/test_sqlite_readonly_mode.py

@@ -0,0 +1,51 @@
+from singlem.metapackage_read_name_store import MetapackageReadNameStore
+from singlem.sequence_database import SequenceDatabase
+
+
+def test_sequence_database_acquire_uses_readonly_immutable_sqlite(monkeypatch, tmp_path):
+    db_dir = tmp_path / "db"
+    db_dir.mkdir()
+    (db_dir / "CONTENTS.json").write_text('{"singlem_database_version": 5}')
+    (db_dir / "otus.sqlite3").touch()
+
+    captured = {}
+
+    class FakeEngine:
+        def connect(self):
+            return object()
+
+    def fake_create_engine(url):
+        captured["url"] = url
+        return FakeEngine()
+
+    monkeypatch.setattr("singlem.sequence_database.create_engine", fake_create_engine)
+
+    SequenceDatabase.acquire(str(db_dir))
+
+    assert captured["url"].startswith("sqlite+pysqlite:///file:")
+    assert "mode=ro" in captured["url"]
+    assert "immutable=1" in captured["url"]
+    assert "uri=true" in captured["url"]
+
+
+def test_metapackage_read_name_store_acquire_uses_readonly_immutable_sqlite(monkeypatch, tmp_path):
+    sqlite_db = tmp_path / "read_taxonomies.sqlite3"
+    sqlite_db.touch()
+
+    captured = {}
+
+    def fake_create_engine(url, echo=None, future=None):
+        captured["url"] = url
+        captured["echo"] = echo
+        captured["future"] = future
+        return object()
+
+    monkeypatch.setattr("singlem.metapackage_read_name_store.create_engine", fake_create_engine)
+
+    MetapackageReadNameStore.acquire(str(sqlite_db))
+
+    assert captured["url"].startswith("sqlite+pysqlite:///file:")
+    assert "mode=ro" in captured["url"]
+    assert "immutable=1" in captured["url"]
+    assert "uri=true" in captured["url"]
+    assert captured["future"] is True