feat: add exception handling for Findex callbacks

Author: ThibaudDauce

.github/workflows/maven.yml

@@ -45,3 +45,137 @@ jobs:
 
       - name: Build with Maven
         run: mvn -B package --file pom.xml
+
+  build_in_docker:
+    services:
+      kms:
+        image: cosmian/kms:4.0.1
+        env:
+          COSMIAN_SERVER_URL: http://localhost:9998
+          KMS_PUBLIC_PATH: /tmp
+          KMS_PRIVATE_PATH: /tmp
+          KMS_SHARED_PATH: /tmp
+        ports:
+          - 9998:9998
+
+      redis:
+        image: redis
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 6379:6379
+
+    runs-on: ${{ matrix.os }}
+    container: centos:centos7.4.1708
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ubuntu-20.04
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Cache build
+        uses: actions/cache@v3
+        continue-on-error: false
+        with:
+          path: |
+            ~/.m2/repository
+            target
+          key: ${{ runner.os }}-maven-docker-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-docker-${{ hashFiles('**/pom.xml') }}
+      - run: yum -y install java-1.8.0-openjdk maven python3 python3-pip
+      - run: python3 scripts/get_native_libraries.py
+
+      - name: Build with Maven
+        run: mvn compile
+        env:
+          COSMIAN_SERVER_URL: http://kms:9998
+          REDIS_HOSTNAME: redis
+          REDIS_PORT: 6379
+          LANG: en_US.UTF-8
+          LANGUAGE: en_US:en
+          LC_ALL: en_US.UTF-8
+
+      - name: Test with Maven
+        run: mvn test
+        env:
+          COSMIAN_SERVER_URL: http://kms:9998
+          REDIS_HOSTNAME: redis
+          REDIS_PORT: 6379
+          LANG: en_US.UTF-8
+          LANGUAGE: en_US:en
+          LC_ALL: en_US.UTF-8
+
+  maven_deploy:
+    needs:
+      - build
+      - build_in_docker
+    runs-on: ubuntu-20.04
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Cache build
+        uses: actions/cache@v3
+        continue-on-error: false
+        with:
+          path: |
+            ~/.m2/repository
+            target
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+      - name: Set up JDK 8
+        uses: actions/setup-java@v3
+        with:
+          java-version: 8
+          distribution: temurin
+          cache: maven
+
+      - uses: actions/setup-python@v3
+        with:
+          python-version: 3.7
+      - run: python scripts/get_native_libraries.py
+
+      - name: Import GPG key
+        id: import_gpg
+        uses: crazy-max/ghaction-import-gpg@v5
+        with:
+          gpg_private_key: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }}
+          passphrase: ${{ secrets.GPG_PASSPHRASE }}
+
+      - name: Maven deploy
+        if: startsWith(github.ref, 'refs/tags/')
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y gnupg2
+          mkdir -p ~/.m2 ~/.gpg
+          echo "$M2_SETTINGS" > ~/.m2/settings.xml
+          echo "$MAVEN_GPG_PRIVATE_KEY" > ~/.gpg/maven_gpg_private_key.asc
+          mvn clean deploy -Dmaven.test.skip
+        env:
+          M2_SETTINGS: ${{ secrets.M2_SETTINGS }}
+          MAVEN_GPG_PRIVATE_KEY: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }}
+
+  release:
+    needs:
+      - maven_deploy
+    name: release
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - name: Release
+        if: startsWith(github.ref, 'refs/tags/')
+        uses: softprops/action-gh-release@v1
+
+  cleanup:
+    needs:
+      - release
+    uses: Cosmian/reusable_workflows/.github/workflows/cleanup_cache.yml@findex_2.0_cover_crypt_8.0
+    secrets: inherit

.github/workflows/release_on_tags.yml

@@ -2,6 +2,16 @@
 
 All notable changes to this project will be documented in this file.
 
+## [3.0.4] - 2023-04-25
+
+### Features
+
+- Add exception handling for Findex callbacks
+
+### Ci
+
+- Test on docker too
+
 ## [3.0.3] - 2023-01-16
 
 ### Deleted

CHANGELOG.md

@@ -40,7 +40,7 @@ This library is open-source software and is available on Maven Central.
 <dependency>
     <groupId>com.cosmian</groupId>
     <artifactId>cloudproof_java</artifactId>
-    <version>3.0.3</version>
+    <version>3.0.4</version>
 </dependency>
 ```
 
@@ -76,3 +76,4 @@ This table shows the compatible versions of the various components
 |----------|------------|------------|--------|
 | 3.0.0    | 4.0.1      | 8.0.1      | 1.0.1  |
 | 3.0.2    | 4.0.1      | 8.0.1      | 2.0.0  |
+| 3.0.4    | 4.0.1      | 8.0.2      | 2.0.2  |

README.md

@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.cosmian</groupId>
     <artifactId>cloudproof_java</artifactId>
-    <version>3.0.3</version>
+    <version>3.0.4</version>
 
     <name>cloudproof_java</name>
     <description>The Cloudproof Java Lib secures data repositories in the cloud with attributes-based access control encryption and encrypted search</description>

pom.xml

@@ -56,7 +56,7 @@ def download_native_libraries(name: str, version: str, destination: str):
 
 
 if __name__ == '__main__':
-    download_native_libraries('findex', 'v2.0.0', 'src/main/resources')
+    download_native_libraries('findex', 'v2.0.2', 'src/main/resources')
     download_native_libraries('findex', 'last_build', 'src/main/resources')
     download_native_libraries('cover_crypt', 'v8.0.2', 'src/main/resources')
     download_native_libraries('cover_crypt', 'last_build', 'src/main/resources')

scripts/get_native_libraries.py

@@ -80,13 +80,14 @@ public static void upsert(
             labelPointer.write(0, label, 0, label.length);
 
             // Indexes creation + insertion/update
+            long start = System.currentTimeMillis();
             unwrap(Findex.INSTANCE.h_upsert(
                 keyPointer, key.length,
                 labelPointer, label.length,
                 indexedValuesToJson(indexedValuesAndWords),
                 db.fetchEntryCallback(),
                 db.upsertEntryCallback(),
-                db.upsertChainCallback()));
+                db.upsertChainCallback()), start);
         }
     }
 
@@ -178,6 +179,7 @@ public static Map<Keyword, Set<Location>> search(byte[] key,
             }
 
             // Indexes creation + insertion/update
+            long start = System.currentTimeMillis();
             int ffiCode = Findex.INSTANCE.h_search(
                 indexedValuesBuffer, indexedValuesBufferSize,
                 keyPointer, key.length,
@@ -189,10 +191,13 @@ public static Map<Keyword, Set<Location>> search(byte[] key,
                 db.progressCallback(),
                 db.fetchEntryCallback(),
                 db.fetchChainCallback());
+            FindexCallbackException.rethrowOnErrorCode(ffiCode, start, System.currentTimeMillis());
+
             if (ffiCode != 0) {
                 // Retry with correct allocated size
                 indexedValuesBuffer = new byte[indexedValuesBufferSize.getValue()];
-                ffiCode = Findex.INSTANCE.h_search(
+                long startRetry = System.currentTimeMillis();
+                unwrap(Findex.INSTANCE.h_search(
                     indexedValuesBuffer, indexedValuesBufferSize,
                     keyPointer, key.length,
                     labelPointer, label.length,
@@ -202,10 +207,7 @@ public static Map<Keyword, Set<Location>> search(byte[] key,
                     insecureFetchChainsBatchSize,
                     db.progressCallback(),
                     db.fetchEntryCallback(),
-                    db.fetchChainCallback());
-                if (ffiCode != 0) {
-                    throw new CloudproofException(get_last_error(4095));
-                }
+                    db.fetchChainCallback()), startRetry);
             }
 
             byte[] indexedValuesBytes = Arrays.copyOfRange(indexedValuesBuffer, 0, indexedValuesBufferSize.getValue());
@@ -236,6 +238,7 @@ public static void compact(int numberOfReindexingPhasesBeforeFullSet,
             labelPointer.write(0, label, 0, label.length);
 
             // Indexes creation + insertion/update
+            long start = System.currentTimeMillis();
             unwrap(Findex.INSTANCE.h_compact(
                 numberOfReindexingPhasesBeforeFullSet,
                 existingKeyPointer, existingKey.length,
@@ -245,21 +248,34 @@ public static void compact(int numberOfReindexingPhasesBeforeFullSet,
                 database.fetchEntryCallback(),
                 database.fetchChainCallback(),
                 database.updateLinesCallback(),
-                database.listRemoveLocationsCallback()));
+                database.listRemoveLocationsCallback()), start);
         }
     }
 
     /**
      * If the result of the last FFI call is in Error, recover the last error from the native code and throw an
      * exception wrapping it.
      *
-     * @param result the result of the FFI call
+     * @param errorCode the errorCode of the FFI call
      * @throws CloudproofException in case of native library error
      */
-    public static void unwrap(int result) throws CloudproofException {
-        if (result == 1) {
+    public static void unwrap(int errorCode) throws CloudproofException {
+        if (errorCode != 0) {
             throw new CloudproofException(get_last_error(4095));
         }
     }
 
+    /**
+     * If the result of the last FFI call is in Error, recover the last error from the native code and throw an
+     * exception wrapping it.
+     *
+     * @param errorCode the errorCode of the FFI call
+     * @throws CloudproofException in case of native library error
+     */
+    public static void unwrap(int errorCode, long start) throws CloudproofException {
+        FindexCallbackException.rethrowOnErrorCode(errorCode, start, System.currentTimeMillis());
+
+        unwrap(errorCode);
+    }
+
 }

src/main/java/com/cosmian/jna/findex/Findex.java

@@ -0,0 +1,46 @@
+package com.cosmian.jna.findex;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import com.cosmian.utils.CloudproofException;
+
+public class FindexCallbackException {
+
+    private final static int CALLBACK_ERROR_CODE_WHEN_THROWING = 42;
+    public static int purgeExceptionsWhenCountIsGreaterThan = 50;
+    public static int purgeExceptionsAfterMillis = 1000 * 60 * 5; // 5 minutes
+
+    public static List<FindexCallbackException> exceptions = new ArrayList<>();
+
+    private long timestamp = System.currentTimeMillis();
+    private CloudproofException e;
+
+    public FindexCallbackException(CloudproofException e) {
+        this.e = e;
+    }
+
+    public static int record(CloudproofException e) {
+        if (exceptions.size() > purgeExceptionsWhenCountIsGreaterThan) {
+            exceptions.removeIf(exception -> exception.timestamp < System.currentTimeMillis() - purgeExceptionsAfterMillis);
+        }
+
+        exceptions.add(new FindexCallbackException(e));
+
+        return CALLBACK_ERROR_CODE_WHEN_THROWING;
+    }
+
+    public static void rethrowOnErrorCode(int errorCode, long start, long end) throws CloudproofException {
+        if (errorCode != CALLBACK_ERROR_CODE_WHEN_THROWING) {
+            return;
+        }
+
+        for (FindexCallbackException e : exceptions) {
+            if (e.timestamp >= start && e.timestamp <= end) {
+                throw e.e;
+            }
+        }
+
+        throw new CloudproofException("Findex returned an error code " + errorCode + " reserved for exceptions but no exception was recorded during the callbacks.");
+    }
+}

src/main/java/com/cosmian/jna/findex/FindexCallbackException.java

@@ -3,6 +3,7 @@
 import java.util.Set;
 import java.util.logging.Logger;
 
+import com.cosmian.jna.findex.FindexCallbackException;
 import com.cosmian.jna.findex.ffi.FindexNativeWrapper.FetchAllEntryTableUidsCallback;
 import com.cosmian.jna.findex.ffi.FindexUserCallbacks.DBFetchAllEntryTableUids;
 import com.cosmian.jna.findex.structs.Uid32;
@@ -24,16 +25,19 @@ public FetchAllEntryTableUids(DBFetchAllEntryTableUids fetch) {
     public int apply(Pointer uidsPointer,
                      IntByReference uidsSize)
         throws CloudproofException {
-
-        //
-        // Select uids and values in EntryTable
-        //
-        Set<Uid32> uidsAndValues = this.fetch.fetchAll();
-
-        //
-        // Serialize results
-        //
-        return FFiUtils.setToOutputPointer(uidsAndValues, uidsPointer, uidsSize);
+        try {
+            //
+            // Select uids and values in EntryTable
+            //
+            Set<Uid32> uidsAndValues = this.fetch.fetchAll();
+    
+            //
+            // Serialize results
+            //
+            return FFiUtils.setToOutputPointer(uidsAndValues, uidsPointer, uidsSize);
+        } catch (CloudproofException e) {
+            return FindexCallbackException.record(e);
+        }
     }
 
 }