Version 3 with cached cross chunk edges

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.21.1
+current_version = 3.1.7
 commit = True
 tag = True
 

.coveragerc

@@ -5,6 +5,13 @@ source = pychunkedgraph
 omit =
     *test*
     *benchmarking/*
+    pychunkedgraph/debug/*
+    pychunkedgraph/export/*
+    pychunkedgraph/jobs/*
+    pychunkedgraph/logging/*
+    pychunkedgraph/repair/*
+    pychunkedgraph/meshing/*
+    pychunkedgraph/app/*
 
 [report]
 # Regexes for lines to exclude from consideration

.github/workflows/main.yml

@@ -4,19 +4,49 @@ on:
   push:
     branches:
       - "main"
+      - "pcgv3"
   pull_request:
     branches:
       - "main"
+      - "pcgv3"
 
 jobs:
   unit-tests:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
-      - name: Build image and run tests
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          load: true
+          tags: seunglab/pychunkedgraph:${{ github.sha }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Run tests with coverage
         run: |
-          docker build --tag seunglab/pychunkedgraph:$GITHUB_SHA .
-          docker run --rm seunglab/pychunkedgraph:$GITHUB_SHA /bin/sh -c "pytest --cov-config .coveragerc --cov=pychunkedgraph ./pychunkedgraph/tests && codecov"
+          docker run --name pcg-tests seunglab/pychunkedgraph:${{ github.sha }} \
+            /bin/sh -c "pytest --cov-config .coveragerc --cov=pychunkedgraph --cov-report=xml:/app/coverage.xml ./pychunkedgraph/tests"
+
+      - name: Copy coverage report from container
+        if: always()
+        run: docker cp pcg-tests:/app/coverage.xml ./coverage.xml
+
+      - name: Upload coverage to Codecov
+        if: always()
+        uses: codecov/codecov-action@v5
+        with:
+          files: ./coverage.xml
+          token: ${{ secrets.CODECOV_TOKEN }}
+          slug: CAVEconnectome/PyChunkedGraph
+          fail_ci_if_error: true
 
+      - name: Cleanup
+        if: always()
+        run: docker rm pcg-tests || true

.github/workflows/release.yml

@@ -45,7 +45,7 @@ jobs:
       - name: Install Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: "3.12"
       - name: Install bumpversion
         run: pip install bumpversion
       - name: Bump version with bumpversion

.gitignore

@@ -115,6 +115,7 @@ venv.bak/
 
 
 # local dev stuff
+.claude/
 .devcontainer/
 *.ipynb
 *.rdb

.pre-commit-config.yaml

@@ -0,0 +1,5 @@
+repos:
+  - repo: https://github.com/psf/black
+    rev: 26.1.0
+    hooks:
+      - id: black

Dockerfile

@@ -1,11 +1,66 @@
-FROM caveconnectome/pychunkedgraph:base_042124
+# syntax=docker/dockerfile:1
+ARG PYTHON_VERSION=3.12
+ARG BASE_IMAGE=tiangolo/uwsgi-nginx-flask:python${PYTHON_VERSION}
+
+
+######################################################
+# Stage 1: Conda environment
+######################################################
+FROM ${BASE_IMAGE} AS conda-deps
+ENV PATH="/root/miniconda3/bin:${PATH}"
+
+RUN apt-get update && apt-get install build-essential wget -y \
+  && wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \
+  && bash Miniconda3-latest-Linux-x86_64.sh -b \
+  && rm Miniconda3-latest-Linux-x86_64.sh \
+  && conda config --add channels conda-forge \
+  && conda update -y --override-channels -c conda-forge conda \
+  && conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/main \
+  && conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/r \
+  && conda install -y --override-channels -c conda-forge conda-pack
+
+COPY requirements.yml requirements.txt requirements-dev.txt ./
+
+RUN --mount=type=cache,target=/opt/conda/pkgs \
+    conda env create -n pcg -f requirements.yml
+
+RUN conda-pack -n pcg --ignore-missing-files -o /tmp/env.tar \
+  && mkdir -p /app/venv && cd /app/venv \
+  && tar xf /tmp/env.tar && rm /tmp/env.tar \
+  && /app/venv/bin/conda-unpack
+
+
+######################################################
+# Stage 2: Bigtable emulator
+######################################################
+FROM golang:bullseye AS bigtable-emulator
+ARG GOOGLE_CLOUD_GO_VERSION=bigtable/v1.19.0
+RUN --mount=type=cache,target=/go/pkg/mod \
+    --mount=type=cache,target=/root/.cache/go-build \
+    git clone --depth=1 --branch="$GOOGLE_CLOUD_GO_VERSION" \
+      https://github.com/googleapis/google-cloud-go.git /usr/src \
+  && cd /usr/src/bigtable && go install -v ./cmd/emulator
+
+
+######################################################
+# Stage 3: Production
+######################################################
+FROM ${BASE_IMAGE}
 ENV VIRTUAL_ENV=/app/venv
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 
+COPY --from=conda-deps /app/venv /app/venv
+COPY --from=bigtable-emulator /go/bin/emulator /app/venv/bin/cbtemulator
 COPY override/gcloud /app/venv/bin/gcloud
 COPY override/timeout.conf /etc/nginx/conf.d/timeout.conf
 COPY override/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+RUN pip install --no-cache-dir --no-deps --force-reinstall zstandard>=0.23.0 \
+  && mkdir -p /home/nginx/.cloudvolume/secrets \
+  && chown -R nginx /home/nginx \
+  && usermod -d /home/nginx -s /bin/bash nginx
 
 COPY requirements.txt .
-RUN pip install --upgrade -r requirements.txt
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip install --upgrade -r requirements.txt
+
 COPY . /app

README.md

@@ -1,8 +1,8 @@
 # PyChunkedGraph
 
 
-[![Build Status](https://travis-ci.org/seung-lab/PyChunkedGraph.svg?branch=master)](https://travis-ci.org/seung-lab/PyChunkedGraph)
-[![codecov](https://codecov.io/gh/seung-lab/PyChunkedGraph/branch/master/graph/badge.svg)](https://codecov.io/gh/seung-lab/PyChunkedGraph)
+[![Tests](https://github.com/CAVEconnectome/PyChunkedGraph/actions/workflows/main.yml/badge.svg)](https://github.com/CAVEconnectome/PyChunkedGraph/actions/workflows/main.yml)
+[![codecov](https://codecov.io/gh/CAVEconnectome/PyChunkedGraph/branch/main/graph/badge.svg)](https://codecov.io/gh/CAVEconnectome/PyChunkedGraph)
 
 The PyChunkedGraph is a proofreading and segmentation data management backend powering FlyWire and other proofreading platforms. It builds on an initial agglomeration of supervoxels and facilitates fast and parallel editing of connected components in the agglomeration graph by many users.
 

cloudbuild.yaml

@@ -10,7 +10,7 @@ steps:
     args:
       - "-c"
       - |
-        docker build -t $$USERNAME/pychunkedgraph:$TAG_NAME .
+        DOCKER_BUILDKIT=1 docker build -t $$USERNAME/pychunkedgraph:$TAG_NAME .
     timeout: 600s
     secretEnv: ["USERNAME"]
 

codecov.yml

@@ -0,0 +1,19 @@
+codecov:
+  require_ci_to_pass: true
+
+coverage:
+  status:
+    project:
+      default:
+        target: auto
+        threshold: 1%
+        informational: true
+    patch:
+      default:
+        target: 1%
+        informational: true
+
+comment:
+  layout: "reach,diff,flags,files"
+  behavior: default
+  require_changes: false

docs/Readme.md

@@ -10,7 +10,7 @@ pip install -r requirements.txt
 
 ## Multiprocessing
 
-Check out [multiprocessing.md](https://github.com/seung-lab/PyChunkedGraph/blob/master/src/pychunkedgraph/multiprocessing.md) for how to use the multiprocessing tools implemented for the ChunkedGraph
+Check out [multiprocessing.md](https://github.com/CAVEconnectome/PyChunkedGraph/blob/master/src/pychunkedgraph/multiprocessing.md) for how to use the multiprocessing tools implemented for the ChunkedGraph
 
 
 ## Credentials
@@ -30,7 +30,7 @@ The current version of the ChunkedGraph contains supervoxels from `gs://nkem/bas
 
 ### Building the graph
 
-[buildgraph.md](https://github.com/seung-lab/PyChunkedGraph/blob/master/src/pychunkedgraph/buildgraph.md) explains how to build a graph from scratch.
+[buildgraph.md](https://github.com/CAVEconnectome/PyChunkedGraph/blob/master/src/pychunkedgraph/buildgraph.md) explains how to build a graph from scratch.
 
 
 ### Initialization

docs/edges.md

@@ -2,7 +2,7 @@
 
 PyChunkedgraph uses protobuf for serialization and zstandard for compression.
 
-Edges and connected components per chunk are stored using the protobuf definitions in [`pychunkedgraph.io.protobuf`](https://github.com/seung-lab/PyChunkedGraph/pychunkedgraph/io/protobuf/chunkEdges.proto).
+Edges and connected components per chunk are stored using the protobuf definitions in [`pychunkedgraph.io.protobuf`](https://github.com/CAVEconnectome/PyChunkedGraph/pychunkedgraph/io/protobuf/chunkEdges.proto).
 This format is a result of performance tests.
 It provided the best tradeoff between deserialzation speed and storage size.
 

docs/edges_and_components.md

@@ -2,7 +2,7 @@
 
 PyChunkedgraph uses protobuf for serialization and zstandard for compression.
 
-Edges and connected components per chunk are stored using the protobuf definitions in [`pychunkedgraph.io.protobuf`](https://github.com/seung-lab/PyChunkedGraph/pychunkedgraph/io/protobuf/chunkEdges.proto).
+Edges and connected components per chunk are stored using the protobuf definitions in [`pychunkedgraph.io.protobuf`](https://github.com/CAVEconnectome/PyChunkedGraph/pychunkedgraph/io/protobuf/chunkEdges.proto).
 This format is a result of performance tests.
 It provided the best tradeoff between deserialzation speed and storage size.
 

docs/segmentation_preprocessing.md

@@ -32,10 +32,10 @@ There are three types of edges:
 2. `cross_chunk`: edges between parts of "the same" supervoxel in the unchunked segmentation that has been split across chunk boundary 
 3. `between_chunk`: edges between supervoxels across chunks
 
-Every pair of touching supervoxels has an edge between them. All edges are stored using [protobuf](https://github.com/seung-lab/PyChunkedGraph/blob/pcgv2/pychunkedgraph/io/protobuf/chunkEdges.proto). During ingest only edges of type 2. and 3. are copied into BigTable, whereas edges of type 1. are always read from storage to reduce cost. Similar to the supervoxel segmentation, we recommed storing these on GCloud in the same zone the ChunkedGraph server will be deployed in to reduce latency. 
+Every pair of touching supervoxels has an edge between them. All edges are stored using [protobuf](https://github.com/CAVEconnectome/PyChunkedGraph/blob/pcgv2/pychunkedgraph/io/protobuf/chunkEdges.proto). During ingest only edges of type 2. and 3. are copied into BigTable, whereas edges of type 1. are always read from storage to reduce cost. Similar to the supervoxel segmentation, we recommed storing these on GCloud in the same zone the ChunkedGraph server will be deployed in to reduce latency. 
 
 To denote which edges form a connected component within a chunk, a component mapping needs to be created. This mapping is only used during ingest. 
 
-More details on how to create these protobuf files can be found [here](https://github.com/seung-lab/PyChunkedGraph/blob/pcgv2/docs/storage.md).
+More details on how to create these protobuf files can be found [here](https://github.com/CAVEconnectome/PyChunkedGraph/blob/pcgv2/docs/storage.md).