Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
26 changes: 1 addition & 25 deletions .envrc
Original file line number Diff line number Diff line change
Expand Up @@ -7,22 +7,13 @@ use_flake_subdir() {
watch_file $file
done
mkdir -p "$(direnv_layout_dir)"
if [ -z "$ARTIFACTORY_PASSWORD" ] || [ "${OSS_ONLY:-}" == 1 ] || [ "${OSS_ONLY:-}" == "true" ]; then
eval "$(nix print-dev-env --profile "$(direnv_layout_dir)/flake-profile" "path:nix#oss" "$@")"
else
eval "$(nix print-dev-env --profile "$(direnv_layout_dir)/flake-profile" "path:nix" "$@")"
fi
eval "$(nix print-dev-env --profile "$(direnv_layout_dir)/flake-profile" "path:nix" "$@")"
}

source_envrc_private() {
[[ -f .envrc.private ]] && [[ -z "$IGNORE_PRIVATE_ENVRC" ]] && source_env .envrc.private || true
}

# Source .envrc.private before use_flake_subdir because it provides ARTIFACTORY_PASSWORD
# which determines whether the enterprise or OSS nix shell is used.
# This workaround can be removed once the enterprise edition is removed.
source_envrc_private

# TODO(DACH-NY/canton-network-node#3876) work around for $TMPDIR is removed. #3876 to investigate more
OLD_TMPDIR=${TMPDIR-unset}

Expand All @@ -42,8 +33,6 @@ source "${TOOLS_LIB}/libcli.source"

source_env .envrc.validate

export ENTERPRISE_ARTIFACTORY_DOCKER_REGISTRY=digitalasset-canton-enterprise-docker.jfrog.io

# Configure docker access for cluster and integration tests
## Read current credentials
function read_docker_creds() {
Expand All @@ -58,13 +47,6 @@ function read_docker_creds() {
fi
}

function check_docker_creds() {
local docker_creds="$1"
[[ -z "$docker_creds" || "$docker_creds" == "null" || "$docker_creds" != "$ARTIFACTORY_USER:$ARTIFACTORY_PASSWORD" ]] && \
## And artifactory user and password are set
[[ -n "$ARTIFACTORY_USER" && -n "$ARTIFACTORY_PASSWORD" ]]
}

function check_docker_creds_ghcr() {
local docker_creds="$1"
[[ -z "$docker_creds" || "$docker_creds" == "null" || "$docker_creds" != "$GH_USER:$GH_TOKEN" ]] && \
Expand All @@ -77,12 +59,6 @@ if check_docker_creds_ghcr "$DOCKER_CREDS"; then
echo $GH_TOKEN | docker login "$GHCR" -u "$GH_USER" --password-stdin
fi

DOCKER_CREDS=$(read_docker_creds "$ENTERPRISE_ARTIFACTORY_DOCKER_REGISTRY")
if check_docker_creds "$DOCKER_CREDS"; then
echo "Logging into $ENTERPRISE_ARTIFACTORY_DOCKER_REGISTRY"
echo $ARTIFACTORY_PASSWORD | docker login "$ENTERPRISE_ARTIFACTORY_DOCKER_REGISTRY" -u "$ARTIFACTORY_USER" --password-stdin
fi

# re-export CIRCLECI_TOKEN (which we use in many places) as CIRCLECI_CLI_TOKEN (which `circleci` cli tool picks up)
export CIRCLECI_CLI_TOKEN="${CIRCLECI_TOKEN}"

Expand Down
9 changes: 1 addition & 8 deletions .envrc.validate
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,5 @@ if [[ -z "$IGNORE_PRIVATE_ENVRC" ]]; then
AUTH0_VALIDATOR_MANAGEMENT_API_CLIENT_ID \
AUTH0_VALIDATOR_MANAGEMENT_API_CLIENT_SECRET \
AUTH0_TESTS_MANAGEMENT_API_CLIENT_ID \
AUTH0_TESTS_MANAGEMENT_API_CLIENT_SECRET \
ARTIFACTORY_USER \
ARTIFACTORY_PASSWORD
fi

if [ "$IS_ENTERPRISE" != "true" ]; then
echo ""
_warning "Note: Using Canton community instead of Canton enterprise. Certain tests which rely on Enterprise features will fail locally."
AUTH0_TESTS_MANAGEMENT_API_CLIENT_SECRET
fi
2 changes: 1 addition & 1 deletion .envrc.vars
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ export SPLICE_OAUTH_SV_TEST_CLIENT_ID_VALIDATOR=bUfFRpl2tEfZBB7wzIo9iRNGTj8wMeIn
export USE_GKE_GCLOUD_AUTH_PLUGIN=true

# CometBFT settings
export COMETBFT_DOCKER_IMAGE="digitalasset-canton-enterprise-docker.jfrog.io/cometbft-canton-network:${COMETBFT_RELEASE_VERSION}"
export COMETBFT_DOCKER_IMAGE="europe-docker.pkg.dev/da-images/public/docker/cometbft-canton-network:${COMETBFT_RELEASE_VERSION}"

#Test containers config
## Speed up runs
Expand Down
33 changes: 7 additions & 26 deletions .github/actions/nix/setup_nix/action.yml
Original file line number Diff line number Diff line change
@@ -1,19 +1,14 @@
name: "Setup Nix"
description: "Setup Nix"
inputs:
artifactory_user:
description: "The Artifactory user"
required: true
artifactory_password:
description: "The Artifactory password"
required: true
nix_path:
description: "The path to nix flake directory"
required: false
default: ${{ format('{0}/nix', github.repository == 'DACH-NY/canton-network-internal' && 'splice' || '.') }}
target:
description: "Choose nix target: oss - restrict upstream dependencies (e.g. Canton) to OSS versions (the equivalent of OSS_ONLY=1 in local checkouts), static_tests - for static tests, default - for enterprise dependencies"
required: true
description: "Choose nix target: static_tests - for static tests, default - for full dependencies"
required: false
default: "default"
cache_version:
description: "Cache version"
required: true
Expand All @@ -37,12 +32,12 @@ runs:
using: "composite"
steps:
- name: Validate input
if: "${{!( inputs.target == 'default' || inputs.target == 'oss' || inputs.target == 'static_tests')}}"
if: "${{!( inputs.target == 'default' || inputs.target == 'static_tests')}}"
shell: bash
id: validate_input
run: |
echo "Target invalid: ${{ inputs.target }}"
echo "Target needs to be one of: 'default', 'oss', 'static_tests'"
echo "Target needs to be one of: 'default', 'static_tests'"
exit 1
- name: Compute cache Key
id: cache_key
Expand All @@ -60,8 +55,8 @@ runs:
echo "home: $HOME" >> /tmp/nix-cache-key # important when restoring simlinks from cache, apparently
echo "nix binary version: $NIX_BINARY_VERSION" >> /tmp/nix-cache-key # different nix versions might behave differently and corrupt the caches
echo "target: ${{ inputs.target }}" >> /tmp/nix-cache-key
if [ "${{ inputs.target }}" != 'default' ]; then
echo "Using OSS only dependencies"
if [ "${{ inputs.target }}" == 'static_tests' ]; then
echo "Using minimal nix dependencies for static tests"
fi
cat /tmp/nix-cache-key
cache_key=($(md5sum "/tmp/nix-cache-key"))
Expand All @@ -76,11 +71,6 @@ runs:
run: |
set -euxo pipefail

if [[ ${{ inputs.target }} == 'default' ]]; then
echo "Must use OSS only dependencies in GitHub-hosted runners"
exit 1
fi

echo "Latest nix cache:"

wget -q "https://storage.googleapis.com/splice-nix-cache-public/${cache_key}.tar.gz" -O cache.tar.gz || true
Expand Down Expand Up @@ -143,15 +133,6 @@ runs:
sh <(curl -fsSL --retry 8 "https://releases.nixos.org/nix/nix-$NIX_BINARY_VERSION/install") --no-daemon
sudo mkdir -p /etc/nix
sudo chmod a+rw /etc/nix
if [[ "${{ inputs.target }}" != 'default' ]]; then
echo "Using OSS only dependencies, not setting up Artifactory credentials"
else
cat <<EOF > /etc/nix/netrc
machine digitalasset.jfrog.io
login ${{ inputs.artifactory_user }}
password ${{ inputs.artifactory_password }}
EOF
fi
export USER=$(whoami)
echo "Running nix.sh"
. ~/.nix-profile/etc/profile.d/nix.sh
Expand Down
10 changes: 1 addition & 9 deletions .github/actions/sbt/execute_sbt_command/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,6 @@ inputs:
# The caller needs to quote commands that contain spaces, e.g. "\"testOnly myTest\"".
description: "The SBT command to run"
required: true
artifactory_user:
description: "Artifactory user"
required: false
artifactory_password:
description: "Artifactory password"
required: false
extra_env_vars:
description: "Extra environment variables to set before running the SBT command"
required: false
Expand Down Expand Up @@ -51,7 +45,7 @@ runs:
- name: Execute SBT command"
uses: ./.github/actions/nix/run_bash_command_in_nix
with:
additional_nix_args: "--keep GITHUB_ACTION_PATH --keep ARTIFACTORY_USER --keep ARTIFACTORY_PASSWORD ${{ inputs.additional_nix_args }}"
additional_nix_args: "--keep GITHUB_ACTION_PATH ${{ inputs.additional_nix_args }}"
cmd: |
# This might help resolve https://github.com/DACH-NY/canton-network-node/issues/8146
export PROTOCBRIDGE_NO_CLEANUP="1"
Expand Down Expand Up @@ -91,8 +85,6 @@ runs:
$GITHUB_ACTION_PATH/../../scripts/check-sbt-output.sh "sbt_output"
fi
}
export ARTIFACTORY_USER="${{ inputs.artifactory_user }}"
export ARTIFACTORY_PASSWORD="${{ inputs.artifactory_password }}"

# Ensure that we're in the root of splice before execution
pushd ${{ inputs.splice_root }} &> /dev/null
Expand Down
16 changes: 1 addition & 15 deletions .github/actions/tests/common_test_setup/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,8 @@ inputs:
description: "Whether to save the Nix cache to GCP"
required: false
default: "false"
artifactory_user:
description: "The Artifactory user"
required: false
artifactory_password:
description: "The Artifactory password"
required: false
target:
description: "Choose nix target: oss - restrict upstream dependencies (e.g. Canton) to OSS versions (the equivalent of OSS_ONLY=1 in local checkouts), static_tests - for static tests, default - for enterprise dependencies"
description: "Choose nix target: static_tests - for static tests, default - for full dependencies"
default: 'default'
# type: choice
# options:
Expand Down Expand Up @@ -51,12 +45,6 @@ outputs:
runs:
using: "composite"
steps:
- name: Validate input
if: inputs.target == 'default' && (inputs.artifactory_password == '' || inputs.artifactory_user == '')
shell: bash
run: |
echo "artifactory_user and artifactory_password must be provided if not using OSS only dependencies."
exit 1

- name: Publish test name metric
uses: miguelteixeiraa/action-run-in-background@e28f036c202e9066287e6a50ce8b80749627cc7d # v1.0.0
Expand All @@ -70,8 +58,6 @@ runs:
- name: Set up Nix (Self hosted)
uses: ./.github/actions/nix/setup_nix
with:
artifactory_user: ${{ inputs.artifactory_user }}
artifactory_password: ${{ inputs.artifactory_password }}
cache_version: ${{ inputs.cache_version }}
should_save: ${{ inputs.save_nix_cache }}
should_save_gcp: ${{ inputs.save_nix_cache_to_gcp }}
Expand Down
21 changes: 1 addition & 20 deletions .github/actions/tests/scala_test/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,6 @@ inputs:
start_canton_options:
description: "Options for start-canton.sh"
required: true
artifactory_user:
description: "The Artifactory user"
required: true
artifactory_password:
description: "The Artifactory password"
required: true
test_suite_name:
description: "Name of the test suite"
required: true
Expand Down Expand Up @@ -93,10 +87,6 @@ inputs:
protocol_version:
description: "Synchronizer Protocol Version"
required: true
oss_only:
description: "Restrict upstream dependencies (e.g. Canton) to OSS versions (the equivalent of OSS_ONLY=1 in local checkouts)"
required: false
default: "false"
cache_version:
description: "Cache version"
required: true
Expand All @@ -109,12 +99,7 @@ runs:
with:
test_name: ${{ inputs.test_suite_name }}
with_sbt: false # we setup SBT later while canton is starting up
artifactory_user: ${{ inputs.artifactory_user }}
artifactory_password: ${{ inputs.artifactory_password }}
target: ${{ inputs.oss_only == true && 'oss' || 'default' }}
# The docs job saves the oss nix cache, here we save the non-oss one, but only in one runner to reduce contention
# TODO(#1296): When this runner stops using non-oss, move this to one that does
save_nix_cache: ${{ inputs.runner_index == 0 && inputs.test_suite_name == 'canton-enterprise' }}
target: 'default'

- name: Wait for postgres
uses: ./.github/actions/nix/run_bash_command_in_nix
Expand Down Expand Up @@ -230,8 +215,6 @@ runs:
uses: ./.github/actions/nix/run_bash_command_in_nix
with:
cmd: |
export ARTIFACTORY_USER="${{ inputs.artifactory_user }}"
export ARTIFACTORY_PASSWORD="${{ inputs.artifactory_password }}"
export CIRCLE_REPOSITORY_URL="${{ github.repositoryUrl }}"
export CIRCLE_SHA1="${{ github.sha }}"
/usr/bin/sudo mkdir -p ~/.docker/buildx
Expand All @@ -258,8 +241,6 @@ runs:
uses: ./.github/actions/sbt/execute_sbt_command
with:
extra_env_vars: "POSTGRES_DB=postgres POSTGRES_HOST=localhost POSTGRES_USER=postgres POSTGRES_PASSWORD=postgres INITIAL_PACKAGE_VERSIONS=${{ steps.daml_package_versions.outputs.initial_package_versions }} PROTOCOL_VERSION=${{ inputs.protocol_version }}"
artifactory_user: ${{ inputs.artifactory_user }}
artifactory_password: ${{ inputs.artifactory_password }}
cmd: ${{ steps.list_tests.outputs.RUN_SPLITTED_TESTS_CMD }}
additional_nix_args: "--keep GITHUB_ACTION"
extra_parameters: -DAUTH0_MANAGEMENT_API_CLIENT_ID=${{ inputs.auth0_management_api_client_id }} -DAUTH0_MANAGEMENT_API_CLIENT_SECRET=${{ inputs.auth0_management_api_client_secret }}
Expand Down
15 changes: 11 additions & 4 deletions .github/actions/tests/skip_on_static/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,17 @@ runs:
# as the latter is fixed when the job starts which for `env_hold` jobs
# is _before_ the approval already e.g. when an external contributor
# created the PR and not when the maintainer approved it after adding the static label.
pr_labels=$(curl -sSL --fail-with-body -H "Authorization: Bearer ${{ inputs.gh_token }}" \
--retry 10 --retry-delay 10 --retry-all-errors \
-H "Accept: application/vnd.github.v3+json" \
"${{ github.event.pull_request.url }}" | jq '.labels')
# Write the response to a file instead of piping into jq: curl resets an
# -o output file between retries, but a pipe keeps the failed-attempt
# bodies (breaking jq) and dies once jq exits (curl error 23).
pr_json="$RUNNER_TEMP/pr.json"
curl -sSL --fail-with-body -o "$pr_json" \
-H "Authorization: Bearer ${{ inputs.gh_token }}" \
--retry 10 --retry-delay 10 --retry-all-errors \
-H "Accept: application/vnd.github.v3+json" \
"${{ github.event.pull_request.url }}" \
|| { echo "PR fetch failed; last response body:"; cat "$pr_json"; exit 1; }
pr_labels=$(jq '.labels' "$pr_json")
echo "Pull request labels: $pr_labels"
static_label=$(echo "$pr_labels" | jq -r '.[] | select(.name == "static") | .name' | grep -c 'static' || true)
if [[ "$last_commit_msg" == *"[static]"* ]] || [[ "$static_label" -gt 0 ]]; then
Expand Down
30 changes: 15 additions & 15 deletions .github/store-perf-thresholds.json
Original file line number Diff line number Diff line change
@@ -1,52 +1,52 @@
{
"SvDsoStoreIngestionPerformanceTest": {
"_comment_avg_item_time_ns": "median*(1+ noise_margin + safety_margin) of last 30d. noise_margin=(p95-median)/median, safety_margin=10%. calculated on 2026-May-07",
"_comment_avg_item_time_ns": "median*(1+ noise_margin + safety_margin) of last 30d. noise_margin=(p95-median)/median, safety_margin=10%. calculated on 2026-June-25",
"splice_perf_ingestion_avg_item_time_ns": {
"max": 2500000
"max": 3700000
},
"_comment_total_time_ns": "our hard max is 1h, but we set it to 10m based on the past data collected on 2026-May-07 to detect the trends earlier",
"_comment_total_time_ns": "our hard max is 1h, but we set it to 10m based on the past data collected on 2026-June-25 to detect the trends earlier",
"splice_perf_ingestion_total_time_ns": {
"max": 600000000000
}
},
"ScanStoreIngestionPerformanceTest": {
"_comment_avg_item_time_ns": "median*(1+ noise_margin + safety_margin) of last 30d. noise_margin=(p95-median)/median, safety_margin=10%. calculated on 2026-May-07",
"_comment_avg_item_time_ns": "median*(1+ noise_margin + safety_margin) of last 30d. noise_margin=(p95-median)/median, safety_margin=10%. calculated on 2026-June-25",
"splice_perf_ingestion_avg_item_time_ns": {
"max": 2400000
"max": 3000000
},
"_comment_total_time_ns": "our hard max is 1h, but we set it to 10m based on the past data collected on 2026-May-07 to detect the trends earlier",
"_comment_total_time_ns": "our hard max is 1h, but we set it to 10m based on the past data collected on 2026-June-25 to detect the trends earlier",
"splice_perf_ingestion_total_time_ns": {
"max": 600000000000
}
},
"UpdateHistoryIngestionPerformanceTest": {
"_comment_avg_item_time_ns": "median*(1+ noise_margin + safety_margin) of last 30d. noise_margin=(p95-median)/median, safety_margin=10%. calculated on 2026-May-07",
"_comment_avg_item_time_ns": "median*(1+ noise_margin + safety_margin) of last 30d. noise_margin=(p95-median)/median, safety_margin=10%. calculated on 2026-June-25",
"splice_perf_ingestion_avg_item_time_ns": {
"max": 16800000
"max": 27400000
},
"_comment_total_time_ns": "our hard max is 1h, but we set it to 20m based on the past data collected on 2026-May-07 to detect the trends earlier",
"_comment_total_time_ns": "our hard max is 1h, but we set it to 20m based on the past data collected on 2026-June-25 to detect the trends earlier",
"splice_perf_ingestion_total_time_ns": {
"max": 1200000000000
}
},
"UpdateHistoryReadPerformanceTest-getUpdate": {
"_comment_avg_item_time_ns": "median*(1+ noise_margin + safety_margin) of last 30d. noise_margin=(p95-median)/median, safety_margin=10%. calculated on 2026-May-07",
"_comment_avg_item_time_ns": "median*(1+ noise_margin + safety_margin) of last 30d. noise_margin=(p95-median)/median, safety_margin=10%. calculated on 2026-June-25",
"splice_perf_read_avg_item_time_ns": {
"max": 753600000
"max": 775400000
},
"_comment_total_time_ns": "hard max same as avg_item_time_ns, as we read 1 item",
"splice_perf_read_total_time_ns": {
"max": 753600000
"max": 775400000
}
},
"UpdateHistoryReadPerformanceTest-encodeUpdate": {
"_comment_avg_item_time_ns": "median*(1+ noise_margin + safety_margin) of last 30d. noise_margin=(p95-median)/median, safety_margin=10%. calculated on 2026-May-07",
"_comment_avg_item_time_ns": "median*(1+ noise_margin + safety_margin) of last 30d. noise_margin=(p95-median)/median, safety_margin=10%. calculated on 2026-June-25",
"splice_perf_read_avg_item_time_ns": {
"max": 337000000
"max": 397400000
},
"_comment_total_time_ns": "hard max same as avg_item_time_ns, as we read 1 item",
"splice_perf_read_total_time_ns": {
"max": 337000000
"max": 397400000
}
}
}
1 change: 0 additions & 1 deletion .github/workflows/build.daml_test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@ jobs:
with:
cache_version: 8
test_name: daml_test
target: 'oss'

- name: Run Daml tests
if: steps.skip.outputs.skip != 'true'
Expand Down
3 changes: 1 addition & 2 deletions .github/workflows/build.deployment_test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,9 @@ jobs:
- name: Setup
uses: ./.github/actions/tests/common_test_setup
with:
cache_version: 8
cache_version: 9
test_name: deployment_test
with_sbt: false
target: 'oss'

- name: Helm tests
uses: ./.github/actions/nix/run_bash_command_in_nix
Expand Down
Loading
Loading