chore: Merge branch 'release/v4.0.0'

Author: Manuthor

.github/workflows/ci.yml

@@ -18,7 +18,7 @@ jobs:
           pip install flake8
       - name: Linter
         run: |
-          flake8 --max-line-length=100
+          flake8 --max-line-length=100 --ignore E501
       - name: Build package
         run: |
           python -m build
@@ -62,9 +62,10 @@ jobs:
           pip install cloudproof_py*.whl
       - name: Type-check
         run: |
-          mypy src/cloudproof_py/findex/
+          mypy src/cloudproof_py/anonymization/
           mypy src/cloudproof_py/cover_crypt/
-          mypy src/cloudproof_py/cloudproof_fpe/
+          mypy src/cloudproof_py/findex/
+          mypy src/cloudproof_py/fpe/
           mypy tests/
           mypy examples/cli_demo
           mypy examples/findex_upsert_search
@@ -88,12 +89,13 @@ jobs:
     needs: test
     uses: Cosmian/reusable_workflows/.github/workflows/cloudproof_java_in_docker.yml@develop
     with:
-      branch: develop
+      branch: v6.0.0
       target: x86_64-unknown-linux-gnu
       extension: so
       destination: linux-x86-64
       os: ubuntu-20.04
       kms-version: 4.3.3
+      findex-cloud-version: 0.1.0
       copy_fresh_build: false
       copy_regression_files: |
         cp ./cloudproof_python/non_regression_vector.json src/test/resources/cover_crypt/non_regression/python_non_regression_vector.json
@@ -116,7 +118,7 @@ jobs:
       - test
     uses: Cosmian/reusable_workflows/.github/workflows/cloudproof_flutter.yml@develop
     with:
-      branch: develop
+      branch: v7.0.0
       target: x86_64-unknown-linux-gnu
       extension: so
       copy_fresh_build: false

.github/workflows/publish.yml

@@ -1,17 +1,48 @@
-name: Publish
+name: Build doc, release and publish
 
-on:
-  push:
-    tags:
-      - '*'
+on: [push]
 
 jobs:
+  build_doc:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v3
+        with:
+          python-version: 3.7
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          scripts/ci_install_pyo3_builds.sh
+          pip install -r docs/requirements.txt
+      - name: Build package
+        run: |
+          python scripts/extract_lib_types.py
+          cd docs
+          make html
+          cd _build
+          zip -r ../../Documentation.zip html/
+      - uses: actions/upload-artifact@v3
+        with:
+          name: html_doc
+          path: Documentation.zip
+          retention-days: 1
+
   release:
+    needs: build_doc
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - name: Release
+      - name: Download artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: html_doc
+          path: .
+      - name: Release on tags, attach asset on release
+        if: startsWith(github.ref, 'refs/tags/')
         uses: softprops/action-gh-release@v1
+        with:
+          files: Documentation.zip
 
   publish:
     runs-on: ubuntu-latest
@@ -27,29 +58,8 @@ jobs:
           pip install twine
           python -m build
       - name: Upload to Pypi
+        if: startsWith(github.ref, 'refs/tags/')
         run: twine upload -u "${PYPI_USERNAME}" -p "${PYPI_PASSWORD}" dist/cloudproof_py*.whl
         env:
           PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }}
           PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
-
-  docs:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v3
-        with:
-          python-version: 3.7
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          scripts/ci_install_pyo3_builds.sh
-          pip install -r docs/requirements.txt
-      - name: Build package
-        run: |
-          python scripts/extract_lib_types.py
-          cd docs && make html
-      - uses: actions/upload-artifact@v3
-        with:
-          name: html_doc
-          path: docs/_build/html
-          retention-days: 60

.pre-commit-config.yaml

@@ -33,37 +33,37 @@ repos:
       - id: trailing-whitespace
 
   - repo: https://github.com/compilerla/conventional-pre-commit
-    rev: v2.1.1
+    rev: v2.2.0
     hooks:
       - id: conventional-pre-commit
         stages: [commit-msg]
         args: [] # optional: list of Conventional Commits types to allow e.g. [feat, fix, ci, chore, test]
 
   - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: v3.0.0-alpha.6
+    rev: v3.0.0-alpha.9-for-vscode
     hooks:
       - id: prettier
         types_or: [css, javascript, jsx, markdown, bash, java, sh]
 
   - repo: https://github.com/psf/black
-    rev: 23.1.0
+    rev: 23.3.0
     hooks:
       - id: black
 
   - repo: https://github.com/pycqa/flake8
     rev: 6.0.0
     hooks:
       - id: flake8
-        args: [--max-line-length=100]
+        args: ["--max-line-length=100", "--ignore", "E501"]
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: 'v1.1.1'
+    rev: 'v1.3.0'
     hooks:
     - id: mypy
       additional_dependencies: [types_termcolor==1.1.6, types_redis==4.3.21.6, types-requests==2.28.11.15, types-urllib3==1.26.25.8]
 
   - repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: v0.33.0
+    rev: v0.34.0
     hooks:
       - id: markdownlint-fix
         args:

CHANGELOG.md

@@ -2,6 +2,31 @@
 
 All notable changes to this project will be documented in this file.
 
+## [4.0.0] - 2023-06-01
+
+### Features
+
+- upgrade Findex (v3.0.0 -> v4.0.0):
+  - change indexed values size (require a reset of the index database)
+  - change search interface
+    - change parameter order
+    - remove `max_result_per_keyword`
+    - remove `max_depth`
+    - remove `fetch_chains_batch_size`
+    - searching for a non indexed keyword leads to an empty array for this
+      keyword in the search results instead of this keyword being missing from
+      the results.
+  - change upsert interface:
+    - change parameter order
+    - add deletions
+  - change compact interface:
+    - change parameter order
+- add data anonymization methods such as:
+  - noise methods
+  - hash methods
+  - number methods
+  - word methods
+
 ---
 
 ## [3.1.0] - 2023-04-06

README.md

@@ -61,17 +61,23 @@ examples/cli_demo/run_demo.sh
 To build from source:
 
 ```sh
-scripts/build.sh [-i]
+scripts/build.sh
 ```
 
-**Note**: add `-i` to install after build.
-
 To build and run the tests:
 
+```sh
+scripts/build.sh -t
+```
+
+With PyO3 packages from [package.cosmian.com](https://package.cosmian.com):
+
 ```sh
 scripts/build.sh -it
 ```
 
+**Note**: edit [scripts/ci_install_pyo3_builds.sh](./scripts/ci_install_pyo3_builds.sh) with the right build name.
+
 To build the documentation:
 
 ```sh
@@ -88,6 +94,7 @@ This table shows the minimum version correspondence between the various componen
 
 | `cloudproof_py` | CoverCrypt | Findex      | KMS   |
 | --------------- | ---------- | ----------- | ----- |
+| >=4.0.0         | 11.0.0     | 4.0.0,4.0.1 | 4.3.3 |
 | >=3.0.0         | 11.0.0     | 3.0.0       | 4.3.3 |
 | >=2.0.0         | 10.0.0     | 2.0.1,2.1.0 | 4.2.0 |
 | >=1.0.0         | 8.0.1      | 2.0.0       | -     |

examples/cli_demo/findex_db.py

@@ -1,8 +1,8 @@
 # -*- coding: utf-8 -*-
 import sqlite3
-from cloudproof_py.findex import Findex
+from typing import Dict, List, Set, Tuple, Sequence
 
-from typing import Dict, List, Set, Tuple
+from cloudproof_py.findex import Findex
 
 
 class FindexSQLite(Findex.FindexUpsert, Findex.FindexSearch):
@@ -12,25 +12,27 @@ def __init__(self, db_conn: sqlite3.Connection) -> None:
         super().__init__()
         self.conn = db_conn
 
-    def fetch_entry_table(self, entry_uids: List[bytes]) -> Dict[bytes, bytes]:
+    def fetch_entry_table(
+        self, entry_uids: List[bytes]
+    ) -> Sequence[Tuple[bytes, bytes]]:
         """Query the Entry Table.
 
         Args:
             entry_uids (List[bytes], optional): uids to query. if None, return the entire table
 
         Returns:
-            Dict[bytes, bytes]: uid -> value mapping
+            Sequence[Tuple[bytes, bytes]]: uid -> value mapping
         """
         str_uids = ",".join("?" * len(entry_uids))
         cur = self.conn.execute(
             f"SELECT uid, value FROM entry_table WHERE uid IN ({str_uids})",
             entry_uids,
         )
         values = cur.fetchall()
-        output_dict = {}
+        output_array = []
         for value in values:
-            output_dict[value[0]] = value[1]
-        return output_dict
+            output_array.append((value[0], value[1]))
+        return output_array
 
     def fetch_all_entry_table_uids(self) -> Set[bytes]:
         """Return all UIDs in the Entry Table.

examples/cli_demo/main.py

@@ -132,7 +132,7 @@
     }
     # Upsert keywords
     findex_interface.upsert(
-        mapping_indexed_values_to_keywords, findex_master_key, findex_label
+        findex_master_key, findex_label, mapping_indexed_values_to_keywords, {}
     )
     print("Findex: Done indexing", len(users), "users")
     print(
@@ -142,9 +142,10 @@
     if activate_auto_completion:
         keywords = [keyword.lower() for user in users for keyword in user.values()]
         findex_interface.upsert(
-            findex.utils.generate_auto_completion(keywords),
             findex_master_key,
             findex_label,
+            findex.utils.generate_auto_completion(keywords),
+            {},
         )
 
     cc_user_keys = {"Alice": key_Alice, "Bob": key_Bob, "Charlie": key_Charlie}
@@ -168,7 +169,7 @@
 
         # 1. Findex search
         found_users = findex_interface.search(
-            [keyword], findex_master_key, findex_label
+            findex_master_key, findex_label, [keyword]
         )
         if len(found_users) == 0:
             print(colored("No user found!", "red", attrs=["bold"]))

examples/findex_upsert_search/findex_dict.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 from cloudproof_py.findex import Findex
 
-from typing import Dict, List, Tuple
+from typing import Dict, List, Tuple, Sequence
 
 
 class FindexDict(Findex.FindexUpsert, Findex.FindexSearch):
@@ -14,19 +14,21 @@ def __init__(self) -> None:
         self.chain_table: Dict[bytes, bytes] = {}
 
     # Implement callback functions
-    def fetch_entry_table(self, entry_uids: List[bytes]) -> Dict[bytes, bytes]:
+    def fetch_entry_table(
+        self, entry_uids: List[bytes]
+    ) -> Sequence[Tuple[bytes, bytes]]:
         """Query the Entry Table.
 
         Args:
             entry_uids (List[bytes], optional): uids to query. if None, return the entire table
 
         Returns:
-            Dict[bytes, bytes]: uid -> value mapping
+            Sequence[Tuple[bytes, bytes]]: uid -> value mapping
         """
-        res = {}
+        res = []
         for uid in entry_uids:
             if uid in self.entry_table:
-                res[uid] = self.entry_table[uid]
+                res.append((uid, self.entry_table[uid]))
         return res
 
     def fetch_chain_table(self, chain_uids: List[bytes]) -> Dict[bytes, bytes]:

examples/findex_upsert_search/findex_redis.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 from cloudproof_py.findex import Findex
 
-from typing import Dict, List, Tuple
+from typing import Dict, List, Tuple, Sequence
 import redis
 
 
@@ -16,20 +16,22 @@ def __init__(self) -> None:
         self.prefix_chain = b"chain:"
 
     # Implement callback functions
-    def fetch_entry_table(self, entry_uids: List[bytes]) -> Dict[bytes, bytes]:
+    def fetch_entry_table(
+        self, entry_uids: List[bytes]
+    ) -> Sequence[Tuple[bytes, bytes]]:
         """Query the Entry Table.
 
         Args:
             entry_uids (List[bytes], optional): uids to query. if None, return the entire table
 
         Returns:
-            Dict[bytes, bytes]: uid -> value mapping
+            Sequence[Tuple[bytes, bytes]]: uid -> value mapping
         """
-        res = {}
+        res = []
         for uid in entry_uids:
             existing_value = self.redis.get(self.prefix_entry + uid)
             if existing_value:
-                res[uid] = existing_value
+                res.append((uid, existing_value))
         return res
 
     def fetch_chain_table(self, chain_uids: List[bytes]) -> Dict[bytes, bytes]: