From 799e12cc2a987cdd3489e21e903725e732371bd5 Mon Sep 17 00:00:00 2001 From: Carsten Hinz Date: Thu, 24 Apr 2025 15:59:46 +0200 Subject: [PATCH 1/5] added option to retrieve metadata sorted to the three levels of the schema extended integration tests code not yet tested --- src/pyfdb/pyfdb.py | 34 ++++++++++++++++++++++++--------- tests/integration/test_pyfdb.py | 17 ++++++++++++++++- 2 files changed, 41 insertions(+), 10 deletions(-) diff --git a/src/pyfdb/pyfdb.py b/src/pyfdb/pyfdb.py index 430bed1..62f6cf5 100644 --- a/src/pyfdb/pyfdb.py +++ b/src/pyfdb/pyfdb.py @@ -171,7 +171,7 @@ class ListIterator: __iterator = None __key = False - def __init__(self, fdb, request, duplicates, key=False, expand=True): + def __init__(self, fdb, request, duplicates, key=False, expand=True, schema=False): iterator = ffi.new("fdb_listiterator_t**") if request: req = Request(request) @@ -183,10 +183,12 @@ def __init__(self, fdb, request, duplicates, key=False, expand=True): self.__iterator = ffi.gc(iterator[0], lib.fdb_delete_listiterator) self.__key = key + self.__schema = schema self.path = ffi.new("const char**") self.off = ffi.new("size_t*") self.len = ffi.new("size_t*") + def __next__(self) -> dict: err = lib.fdb_listiterator_next(self.__iterator) @@ -201,7 +203,7 @@ def __next__(self) -> dict: length=self.len[0], ) - if self.__key: + if self.__key or self.__schema: splitkey = ffi.new("fdb_split_key_t**") lib.fdb_new_splitkey(splitkey) key = ffi.gc(splitkey[0], lib.fdb_delete_splitkey) @@ -213,9 +215,22 @@ def __next__(self) -> dict: level = ffi.new("size_t*") meta = dict() - while lib.fdb_splitkey_next_metadata(key, k, v, level) == 0: - meta[ffi.string(k[0]).decode("utf-8")] = ffi.string(v[0]).decode("utf-8") - el["keys"] = meta + if self.__schema: + schema = dict() + for lvl in range(1,4): + schema[lvl] = dict() + while lib.fdb_splitkey_next_metadata(key, k, v, level) == 0: + mKey = ffi.string(k[0]).decode('utf-8') + val = ffi.string(v[0]).decode('utf-8') + meta[mKey] = val + schema[int(level[0])+1][mKey] = val + el['schema'] = schema + else: # key=True and schema=False + while lib.fdb_splitkey_next_metadata(key, k, v, level) == 0: + meta[ffi.string(k[0]).decode("utf-8")] = ffi.string(v[0]).decode("utf-8") + + if self.__key: + el["keys"] = meta return el @@ -411,18 +426,19 @@ def flush(self) -> None: """Flush any archived data to disk""" lib.fdb_flush(self.ctype) - def list(self, request=None, duplicates=False, keys=False) -> ListIterator: + def list(self, request=None, duplicates=False, keys=False, schema=False) -> ListIterator: """List entries in the FDB5 database Args: request (dict): dictionary representing the request. duplicates (bool) = false : whether to include duplicate entries. keys (bool) = false : whether to include the keys for each entry in the output. + schema (bool) = false : whether to include the metadata sorted according to the three FDB schema levels for each entry in the output. Returns: ListIterator: an iterator over the entries. """ - return ListIterator(self, request, duplicates, keys) + return ListIterator(self, request, duplicates, keys, schema) def retrieve(self, request) -> DataRetriever: """Retrieve data as a stream. @@ -465,11 +481,11 @@ def archive( @wraps(FDB.list) -def list(request, duplicates=False, keys=False) -> ListIterator: +def list(request, duplicates=False, keys=False, schema=False) -> ListIterator: global fdb if not fdb: fdb = FDB() - return ListIterator(fdb, request, duplicates, keys) + return ListIterator(fdb, request, duplicates, keys, schema) @wraps(FDB.retrieve) diff --git a/tests/integration/test_pyfdb.py b/tests/integration/test_pyfdb.py index 019771d..92f8c89 100644 --- a/tests/integration/test_pyfdb.py +++ b/tests/integration/test_pyfdb.py @@ -76,7 +76,7 @@ def test_archival_read(setup_fdb_tmp_dir, tmp_path_factory): request["param"] = "138" print("") print("direct function, updated dictionary:", request) - it = fdb.list(request, True, True) + it = fdb.list(request, True, True, True) el = next(it) assert el["path"] @@ -85,6 +85,14 @@ def test_archival_read(setup_fdb_tmp_dir, tmp_path_factory): keys = el["keys"] assert keys["class"] == "rd" assert keys["levelist"] == "300" + assert el["schema"] + schema = el["schema"] + assert "class" in schema[1] + assert schema[1]["class"] == keys["class"] + assert "levtype" in schema[2] + assert schema[2]["levtype"] == keys["leytype"] + assert "levelist" in schema[3] + assert schema[3]["levelist"] == keys["levelist"] el = next(it) assert el["path"] @@ -93,6 +101,13 @@ def test_archival_read(setup_fdb_tmp_dir, tmp_path_factory): keys = el["keys"] assert keys["class"] == "rd" assert keys["levelist"] == "400" + schema = el["schema"] + assert "class" in schema[1] + assert schema[1]["class"] == keys["class"] + assert "levtype" in schema[2] + assert schema[2]["levtype"] == keys["leytype"] + assert "levelist" in schema[3] + assert schema[3]["levelist"] == keys["levelist"] try: el = next(it) From e96dff0688d0efee5b533ce21a0627465d6421f2 Mon Sep 17 00:00:00 2001 From: Carsten Hinz Date: Thu, 31 Jul 2025 15:06:49 +0200 Subject: [PATCH 2/5] updated pyproject.toml to allow installation with pip install -e .[test] Otherwise pyfdb.FDB could not be found! --- pyproject.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b5c5920..24d0588 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,8 +49,9 @@ dev = [ "tox>=4.19", ] -[tool.setuptools.packages.find] -where = ["src", "tests"] +[tool.setuptools] +packages = ["pyfdb"] +package-dir = {"" = "src" } [tool.setuptools.package-data] pyfdb = ["processed_fdb.h"] From d1de7c4bb7776a1a94345c85aa6065cf6bd117a3 Mon Sep 17 00:00:00 2001 From: Carsten Hinz Date: Tue, 5 Aug 2025 15:42:07 +0200 Subject: [PATCH 3/5] fixed bug in argument forwarding to ListIterator trying to debug crashed of tests --- src/pyfdb/pyfdb.py | 3 ++- tests/integration/test_pyfdb.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/pyfdb/pyfdb.py b/src/pyfdb/pyfdb.py index 80f56b6..30c184d 100644 --- a/src/pyfdb/pyfdb.py +++ b/src/pyfdb/pyfdb.py @@ -474,7 +474,8 @@ def list(self, request=None, duplicates=False, keys=False, schema=False) -> List Returns: ListIterator: an iterator over the entries. """ - return ListIterator(self, request, duplicates, keys, schema) + expand = True + return ListIterator(self, request, duplicates, keys, expand, schema) def retrieve(self, request) -> DataRetriever: """Retrieve data as a stream. diff --git a/tests/integration/test_pyfdb.py b/tests/integration/test_pyfdb.py index 92f8c89..7c2b169 100644 --- a/tests/integration/test_pyfdb.py +++ b/tests/integration/test_pyfdb.py @@ -8,6 +8,8 @@ # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. +import faulthandler +faulthandler.enable() import shutil From 1d72f5ec637e40208d82b6b0b501f23eaa603b6c Mon Sep 17 00:00:00 2001 From: Carsten Hinz Date: Tue, 5 Aug 2025 16:42:54 +0200 Subject: [PATCH 4/5] fixed parameter forwarding for list function --- src/pyfdb/pyfdb.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pyfdb/pyfdb.py b/src/pyfdb/pyfdb.py index 30c184d..bd9f3f6 100644 --- a/src/pyfdb/pyfdb.py +++ b/src/pyfdb/pyfdb.py @@ -568,9 +568,10 @@ def archive( @wraps(FDB.list) def list(request, duplicates=False, keys=False, schema=False) -> ListIterator: global fdb + expand = True if not fdb: fdb = FDB() - return ListIterator(fdb, request, duplicates, keys, schema) + return ListIterator(fdb, request, duplicates, keys, expand, schema) @wraps(FDB.retrieve) From 5a48b788a66cb2e1fa8c5368591c200434acfdef Mon Sep 17 00:00:00 2001 From: Carsten Hinz Date: Tue, 5 Aug 2025 16:46:45 +0200 Subject: [PATCH 5/5] updated comment --- src/pyfdb/pyfdb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pyfdb/pyfdb.py b/src/pyfdb/pyfdb.py index bd9f3f6..194e904 100644 --- a/src/pyfdb/pyfdb.py +++ b/src/pyfdb/pyfdb.py @@ -568,7 +568,7 @@ def archive( @wraps(FDB.list) def list(request, duplicates=False, keys=False, schema=False) -> ListIterator: global fdb - expand = True + expand = True # as not public to the user if not fdb: fdb = FDB() return ListIterator(fdb, request, duplicates, keys, expand, schema)