From 77a15756a04be3831a0395be2e5d44829d36db7b Mon Sep 17 00:00:00 2001 From: willcl-ark Date: Mon, 10 Feb 2025 12:19:35 +0000 Subject: [PATCH 01/48] bench: add shell.nix --- shell.nix | 101 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 shell.nix diff --git a/shell.nix b/shell.nix new file mode 100644 index 000000000000..b1914a75d99a --- /dev/null +++ b/shell.nix @@ -0,0 +1,101 @@ +# Copyright 0xB10C, willcl-ark +{ pkgs ? import + (fetchTarball "https://github.com/nixos/nixpkgs/archive/nixos-24.11.tar.gz") + { }, }: +let + inherit (pkgs.lib) optionals strings; + inherit (pkgs) stdenv; + + # Override the default cargo-flamegraph with a custom fork + cargo-flamegraph = pkgs.rustPlatform.buildRustPackage rec { + pname = + "flamegraph"; # Match the name in Cargo.toml, doesn't seem to work otherwise + version = "bitcoin-core"; + + src = pkgs.fetchFromGitHub { + owner = "willcl-ark"; + repo = "flamegraph"; + rev = "bitcoin-core"; + sha256 = "sha256-tQbr3MYfAiOxeT12V9au5KQK5X5JeGuV6p8GR/Sgen4="; + }; + + doCheck = false; + cargoHash = "sha256-QWPqTyTFSZNJNayNqLmsQSu0rX26XBKfdLROZ9tRjrg="; + + useFetchCargoVendor = true; + + nativeBuildInputs = + pkgs.lib.optionals stdenv.hostPlatform.isLinux [ pkgs.makeWrapper ]; + buildInputs = pkgs.lib.optionals stdenv.hostPlatform.isDarwin + [ pkgs.darwin.apple_sdk.frameworks.Security ]; + + postFixup = pkgs.lib.optionalString stdenv.hostPlatform.isLinux '' + wrapProgram $out/bin/cargo-flamegraph \ + --set-default PERF ${pkgs.linuxPackages.perf}/bin/perf + wrapProgram $out/bin/flamegraph \ + --set-default PERF ${pkgs.linuxPackages.perf}/bin/perf + ''; + }; + +in pkgs.mkShell { + nativeBuildInputs = with pkgs; [ + autoconf + automake + boost + ccache + clang_18 + cmake + libevent + libtool + pkg-config + sqlite + zeromq + ]; + buildInputs = with pkgs; [ + just + bash + git + shellcheck + python310 + uv + + # Benchmarking + cargo-flamegraph + flamegraph + hyperfine + jq + linuxKernel.packages.linux_6_6.perf + perf-tools + util-linux + + # Binary patching + patchelf + + # Guix + curl + getent + ]; + + shellHook = '' + echo "Bitcoin Core build nix-shell" + echo "" + echo "Setting up python venv" + + # fixes libstdc++ issues and libgl.so issues + export LD_LIBRARY_PATH=${stdenv.cc.cc.lib}/lib/:$LD_LIBRARY_PATH + + uv venv --python 3.10 + source .venv/bin/activate + uv pip install -r pyproject.toml + + patch-binary() { + if [ -z "$1" ]; then + echo "Usage: patch-binary " + return 1 + fi + patchelf --set-interpreter "$(cat $NIX_CC/nix-support/dynamic-linker)" "$1" + } + echo "Added patch-binary command" + echo " Usage: 'patch-binary '" + ''; +} From bbd9ee3b5809c98d2c2a71b427626f87ca276775 Mon Sep 17 00:00:00 2001 From: willcl-ark Date: Mon, 10 Feb 2025 12:19:52 +0000 Subject: [PATCH 02/48] bench: add uv + python deps --- pyproject.toml | 15 +++ requirements.txt | 28 ++++++ uv.lock | 251 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 294 insertions(+) create mode 100644 pyproject.toml create mode 100644 requirements.txt create mode 100644 uv.lock diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000000..26605fc84930 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,15 @@ +[project] +name = "bitcoin-core-deps" +version = "0.1.0" +dependencies = [ + "codespell==2.2.6", + "lief==0.13.2", + "mypy==1.4.1", + "pyzmq==25.1.0", + # Removing in favour of packaged nixpkgs bin which is not dynamically linked + # "ruff==0.5.5", + "vulture==2.6", + "pyperf==2.8.0", + "matplotlib==3.8.0", + "numpy==1.26.0" +] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000000..c9b220b6fe46 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,28 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile pyproject.toml -o requirements.txt +codespell==2.2.6 + # via bitcoin-core-deps (pyproject.toml) +lief==0.13.2 + # via bitcoin-core-deps (pyproject.toml) +matplotlib==3.8.0 + # via bitcoin-core-deps (pyproject.toml) +mypy==1.4.1 + # via bitcoin-core-deps (pyproject.toml) +mypy-extensions==1.0.0 + # via mypy +numpy==1.26.0 + # via bitcoin-core-deps (pyproject.toml) +psutil==6.1.0 + # via pyperf +pyperf==2.8.0 + # via bitcoin-core-deps (pyproject.toml) +pyzmq==25.1.0 + # via bitcoin-core-deps (pyproject.toml) +toml==0.10.2 + # via vulture +tomli==2.0.2 + # via mypy +typing-extensions==4.12.2 + # via mypy +vulture==2.6 + # via bitcoin-core-deps (pyproject.toml) diff --git a/uv.lock b/uv.lock new file mode 100644 index 000000000000..090e5f1cb4f1 --- /dev/null +++ b/uv.lock @@ -0,0 +1,251 @@ +version = 1 +requires-python = ">=3.10" + +[[package]] +name = "bitcoin-core-deps" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "codespell" }, + { name = "lief" }, + { name = "mypy" }, + { name = "pyperf" }, + { name = "pyzmq" }, + { name = "vulture" }, +] + +[package.metadata] +requires-dist = [ + { name = "codespell", specifier = "==2.2.6" }, + { name = "lief", specifier = "==0.13.2" }, + { name = "mypy", specifier = "==1.4.1" }, + { name = "pyperf" }, + { name = "pyzmq", specifier = "==25.1.0" }, + { name = "vulture", specifier = "==2.6" }, +] + +[[package]] +name = "cffi" +version = "1.17.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fc/97/c783634659c2920c3fc70419e3af40972dbaf758daa229a7d6ea6135c90d/cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824", size = 516621 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/07/f44ca684db4e4f08a3fdc6eeb9a0d15dc6883efc7b8c90357fdbf74e186c/cffi-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14", size = 182191 }, + { url = "https://files.pythonhosted.org/packages/08/fd/cc2fedbd887223f9f5d170c96e57cbf655df9831a6546c1727ae13fa977a/cffi-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67", size = 178592 }, + { url = "https://files.pythonhosted.org/packages/de/cc/4635c320081c78d6ffc2cab0a76025b691a91204f4aa317d568ff9280a2d/cffi-1.17.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:edae79245293e15384b51f88b00613ba9f7198016a5948b5dddf4917d4d26382", size = 426024 }, + { url = "https://files.pythonhosted.org/packages/b6/7b/3b2b250f3aab91abe5f8a51ada1b717935fdaec53f790ad4100fe2ec64d1/cffi-1.17.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45398b671ac6d70e67da8e4224a065cec6a93541bb7aebe1b198a61b58c7b702", size = 448188 }, + { url = "https://files.pythonhosted.org/packages/d3/48/1b9283ebbf0ec065148d8de05d647a986c5f22586b18120020452fff8f5d/cffi-1.17.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ad9413ccdeda48c5afdae7e4fa2192157e991ff761e7ab8fdd8926f40b160cc3", size = 455571 }, + { url = "https://files.pythonhosted.org/packages/40/87/3b8452525437b40f39ca7ff70276679772ee7e8b394934ff60e63b7b090c/cffi-1.17.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5da5719280082ac6bd9aa7becb3938dc9f9cbd57fac7d2871717b1feb0902ab6", size = 436687 }, + { url = "https://files.pythonhosted.org/packages/8d/fb/4da72871d177d63649ac449aec2e8a29efe0274035880c7af59101ca2232/cffi-1.17.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bb1a08b8008b281856e5971307cc386a8e9c5b625ac297e853d36da6efe9c17", size = 446211 }, + { url = "https://files.pythonhosted.org/packages/ab/a0/62f00bcb411332106c02b663b26f3545a9ef136f80d5df746c05878f8c4b/cffi-1.17.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:045d61c734659cc045141be4bae381a41d89b741f795af1dd018bfb532fd0df8", size = 461325 }, + { url = "https://files.pythonhosted.org/packages/36/83/76127035ed2e7e27b0787604d99da630ac3123bfb02d8e80c633f218a11d/cffi-1.17.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6883e737d7d9e4899a8a695e00ec36bd4e5e4f18fabe0aca0efe0a4b44cdb13e", size = 438784 }, + { url = "https://files.pythonhosted.org/packages/21/81/a6cd025db2f08ac88b901b745c163d884641909641f9b826e8cb87645942/cffi-1.17.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6b8b4a92e1c65048ff98cfe1f735ef8f1ceb72e3d5f0c25fdb12087a23da22be", size = 461564 }, + { url = "https://files.pythonhosted.org/packages/f8/fe/4d41c2f200c4a457933dbd98d3cf4e911870877bd94d9656cc0fcb390681/cffi-1.17.1-cp310-cp310-win32.whl", hash = "sha256:c9c3d058ebabb74db66e431095118094d06abf53284d9c81f27300d0e0d8bc7c", size = 171804 }, + { url = "https://files.pythonhosted.org/packages/d1/b6/0b0f5ab93b0df4acc49cae758c81fe4e5ef26c3ae2e10cc69249dfd8b3ab/cffi-1.17.1-cp310-cp310-win_amd64.whl", hash = "sha256:0f048dcf80db46f0098ccac01132761580d28e28bc0f78ae0d58048063317e15", size = 181299 }, + { url = "https://files.pythonhosted.org/packages/6b/f4/927e3a8899e52a27fa57a48607ff7dc91a9ebe97399b357b85a0c7892e00/cffi-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a45e3c6913c5b87b3ff120dcdc03f6131fa0065027d0ed7ee6190736a74cd401", size = 182264 }, + { url = "https://files.pythonhosted.org/packages/6c/f5/6c3a8efe5f503175aaddcbea6ad0d2c96dad6f5abb205750d1b3df44ef29/cffi-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:30c5e0cb5ae493c04c8b42916e52ca38079f1b235c2f8ae5f4527b963c401caf", size = 178651 }, + { url = "https://files.pythonhosted.org/packages/94/dd/a3f0118e688d1b1a57553da23b16bdade96d2f9bcda4d32e7d2838047ff7/cffi-1.17.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f75c7ab1f9e4aca5414ed4d8e5c0e303a34f4421f8a0d47a4d019ceff0ab6af4", size = 445259 }, + { url = "https://files.pythonhosted.org/packages/2e/ea/70ce63780f096e16ce8588efe039d3c4f91deb1dc01e9c73a287939c79a6/cffi-1.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1ed2dd2972641495a3ec98445e09766f077aee98a1c896dcb4ad0d303628e41", size = 469200 }, + { url = "https://files.pythonhosted.org/packages/1c/a0/a4fa9f4f781bda074c3ddd57a572b060fa0df7655d2a4247bbe277200146/cffi-1.17.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:46bf43160c1a35f7ec506d254e5c890f3c03648a4dbac12d624e4490a7046cd1", size = 477235 }, + { url = "https://files.pythonhosted.org/packages/62/12/ce8710b5b8affbcdd5c6e367217c242524ad17a02fe5beec3ee339f69f85/cffi-1.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a24ed04c8ffd54b0729c07cee15a81d964e6fee0e3d4d342a27b020d22959dc6", size = 459721 }, + { url = "https://files.pythonhosted.org/packages/ff/6b/d45873c5e0242196f042d555526f92aa9e0c32355a1be1ff8c27f077fd37/cffi-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:610faea79c43e44c71e1ec53a554553fa22321b65fae24889706c0a84d4ad86d", size = 467242 }, + { url = "https://files.pythonhosted.org/packages/1a/52/d9a0e523a572fbccf2955f5abe883cfa8bcc570d7faeee06336fbd50c9fc/cffi-1.17.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a9b15d491f3ad5d692e11f6b71f7857e7835eb677955c00cc0aefcd0669adaf6", size = 477999 }, + { url = "https://files.pythonhosted.org/packages/44/74/f2a2460684a1a2d00ca799ad880d54652841a780c4c97b87754f660c7603/cffi-1.17.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:de2ea4b5833625383e464549fec1bc395c1bdeeb5f25c4a3a82b5a8c756ec22f", size = 454242 }, + { url = "https://files.pythonhosted.org/packages/f8/4a/34599cac7dfcd888ff54e801afe06a19c17787dfd94495ab0c8d35fe99fb/cffi-1.17.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b", size = 478604 }, + { url = "https://files.pythonhosted.org/packages/34/33/e1b8a1ba29025adbdcda5fb3a36f94c03d771c1b7b12f726ff7fef2ebe36/cffi-1.17.1-cp311-cp311-win32.whl", hash = "sha256:85a950a4ac9c359340d5963966e3e0a94a676bd6245a4b55bc43949eee26a655", size = 171727 }, + { url = "https://files.pythonhosted.org/packages/3d/97/50228be003bb2802627d28ec0627837ac0bf35c90cf769812056f235b2d1/cffi-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:caaf0640ef5f5517f49bc275eca1406b0ffa6aa184892812030f04c2abf589a0", size = 181400 }, + { url = "https://files.pythonhosted.org/packages/5a/84/e94227139ee5fb4d600a7a4927f322e1d4aea6fdc50bd3fca8493caba23f/cffi-1.17.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:805b4371bf7197c329fcb3ead37e710d1bca9da5d583f5073b799d5c5bd1eee4", size = 183178 }, + { url = "https://files.pythonhosted.org/packages/da/ee/fb72c2b48656111c4ef27f0f91da355e130a923473bf5ee75c5643d00cca/cffi-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:733e99bc2df47476e3848417c5a4540522f234dfd4ef3ab7fafdf555b082ec0c", size = 178840 }, + { url = "https://files.pythonhosted.org/packages/cc/b6/db007700f67d151abadf508cbfd6a1884f57eab90b1bb985c4c8c02b0f28/cffi-1.17.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1257bdabf294dceb59f5e70c64a3e2f462c30c7ad68092d01bbbfb1c16b1ba36", size = 454803 }, + { url = "https://files.pythonhosted.org/packages/1a/df/f8d151540d8c200eb1c6fba8cd0dfd40904f1b0682ea705c36e6c2e97ab3/cffi-1.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da95af8214998d77a98cc14e3a3bd00aa191526343078b530ceb0bd710fb48a5", size = 478850 }, + { url = "https://files.pythonhosted.org/packages/28/c0/b31116332a547fd2677ae5b78a2ef662dfc8023d67f41b2a83f7c2aa78b1/cffi-1.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d63afe322132c194cf832bfec0dc69a99fb9bb6bbd550f161a49e9e855cc78ff", size = 485729 }, + { url = "https://files.pythonhosted.org/packages/91/2b/9a1ddfa5c7f13cab007a2c9cc295b70fbbda7cb10a286aa6810338e60ea1/cffi-1.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f79fc4fc25f1c8698ff97788206bb3c2598949bfe0fef03d299eb1b5356ada99", size = 471256 }, + { url = "https://files.pythonhosted.org/packages/b2/d5/da47df7004cb17e4955df6a43d14b3b4ae77737dff8bf7f8f333196717bf/cffi-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b62ce867176a75d03a665bad002af8e6d54644fad99a3c70905c543130e39d93", size = 479424 }, + { url = "https://files.pythonhosted.org/packages/0b/ac/2a28bcf513e93a219c8a4e8e125534f4f6db03e3179ba1c45e949b76212c/cffi-1.17.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:386c8bf53c502fff58903061338ce4f4950cbdcb23e2902d86c0f722b786bbe3", size = 484568 }, + { url = "https://files.pythonhosted.org/packages/d4/38/ca8a4f639065f14ae0f1d9751e70447a261f1a30fa7547a828ae08142465/cffi-1.17.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4ceb10419a9adf4460ea14cfd6bc43d08701f0835e979bf821052f1805850fe8", size = 488736 }, + { url = "https://files.pythonhosted.org/packages/86/c5/28b2d6f799ec0bdecf44dced2ec5ed43e0eb63097b0f58c293583b406582/cffi-1.17.1-cp312-cp312-win32.whl", hash = "sha256:a08d7e755f8ed21095a310a693525137cfe756ce62d066e53f502a83dc550f65", size = 172448 }, + { url = "https://files.pythonhosted.org/packages/50/b9/db34c4755a7bd1cb2d1603ac3863f22bcecbd1ba29e5ee841a4bc510b294/cffi-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:51392eae71afec0d0c8fb1a53b204dbb3bcabcb3c9b807eedf3e1e6ccf2de903", size = 181976 }, + { url = "https://files.pythonhosted.org/packages/8d/f8/dd6c246b148639254dad4d6803eb6a54e8c85c6e11ec9df2cffa87571dbe/cffi-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f3a2b4222ce6b60e2e8b337bb9596923045681d71e5a082783484d845390938e", size = 182989 }, + { url = "https://files.pythonhosted.org/packages/8b/f1/672d303ddf17c24fc83afd712316fda78dc6fce1cd53011b839483e1ecc8/cffi-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0984a4925a435b1da406122d4d7968dd861c1385afe3b45ba82b750f229811e2", size = 178802 }, + { url = "https://files.pythonhosted.org/packages/0e/2d/eab2e858a91fdff70533cab61dcff4a1f55ec60425832ddfdc9cd36bc8af/cffi-1.17.1-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d01b12eeeb4427d3110de311e1774046ad344f5b1a7403101878976ecd7a10f3", size = 454792 }, + { url = "https://files.pythonhosted.org/packages/75/b2/fbaec7c4455c604e29388d55599b99ebcc250a60050610fadde58932b7ee/cffi-1.17.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:706510fe141c86a69c8ddc029c7910003a17353970cff3b904ff0686a5927683", size = 478893 }, + { url = "https://files.pythonhosted.org/packages/4f/b7/6e4a2162178bf1935c336d4da8a9352cccab4d3a5d7914065490f08c0690/cffi-1.17.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de55b766c7aa2e2a3092c51e0483d700341182f08e67c63630d5b6f200bb28e5", size = 485810 }, + { url = "https://files.pythonhosted.org/packages/c7/8a/1d0e4a9c26e54746dc08c2c6c037889124d4f59dffd853a659fa545f1b40/cffi-1.17.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c59d6e989d07460165cc5ad3c61f9fd8f1b4796eacbd81cee78957842b834af4", size = 471200 }, + { url = "https://files.pythonhosted.org/packages/26/9f/1aab65a6c0db35f43c4d1b4f580e8df53914310afc10ae0397d29d697af4/cffi-1.17.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd398dbc6773384a17fe0d3e7eeb8d1a21c2200473ee6806bb5e6a8e62bb73dd", size = 479447 }, + { url = "https://files.pythonhosted.org/packages/5f/e4/fb8b3dd8dc0e98edf1135ff067ae070bb32ef9d509d6cb0f538cd6f7483f/cffi-1.17.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3edc8d958eb099c634dace3c7e16560ae474aa3803a5df240542b305d14e14ed", size = 484358 }, + { url = "https://files.pythonhosted.org/packages/f1/47/d7145bf2dc04684935d57d67dff9d6d795b2ba2796806bb109864be3a151/cffi-1.17.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:72e72408cad3d5419375fc87d289076ee319835bdfa2caad331e377589aebba9", size = 488469 }, + { url = "https://files.pythonhosted.org/packages/bf/ee/f94057fa6426481d663b88637a9a10e859e492c73d0384514a17d78ee205/cffi-1.17.1-cp313-cp313-win32.whl", hash = "sha256:e03eab0a8677fa80d646b5ddece1cbeaf556c313dcfac435ba11f107ba117b5d", size = 172475 }, + { url = "https://files.pythonhosted.org/packages/7c/fc/6a8cb64e5f0324877d503c854da15d76c1e50eb722e320b15345c4d0c6de/cffi-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a", size = 182009 }, +] + +[[package]] +name = "codespell" +version = "2.2.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e1/97/df3e00b4d795c96233e35d269c211131c5572503d2270afb6fed7d859cc2/codespell-2.2.6.tar.gz", hash = "sha256:a8c65d8eb3faa03deabab6b3bbe798bea72e1799c7e9e955d57eca4096abcff9", size = 300968 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/46/e0/5437cc96b74467c4df6e13b7128cc482c48bb43146fb4c11cf2bcd604e1f/codespell-2.2.6-py3-none-any.whl", hash = "sha256:9ee9a3e5df0990604013ac2a9f22fa8e57669c827124a2e961fe8a1da4cacc07", size = 301382 }, +] + +[[package]] +name = "lief" +version = "0.13.2" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/e2/c4125c279eb2a23ecc86cdb188ed06e9d81a9c700e9412f9be866afc2c7d/lief-0.13.2-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:0390cfaaf0e9aed46bebf26f00f34852768f76bc7f90abf7ceb384566200e5f5", size = 3424746 }, + { url = "https://files.pythonhosted.org/packages/5f/d6/72235d648c6630c37ef52b9f6f4e2f3337842bc4b08c75abcae3052b2c17/lief-0.13.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5581bf0072c1e7a9ea2fb2e2252b8582016e8b298804b5461e552b402c9cd4e9", size = 3249141 }, + { url = "https://files.pythonhosted.org/packages/d7/cc/9895dff094cad3e88636195640b4b47caefe3d300d3f37b653bd109348df/lief-0.13.2-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:dbbf2fb3d7807e815f345c77e287da162e081100f059ec03005995befc295d7f", size = 3793938 }, + { url = "https://files.pythonhosted.org/packages/0d/1b/f4bf63bfce187ae210980bdd1a20ea7d8e080381eef09e7d26c585eaa614/lief-0.13.2-cp310-cp310-manylinux_2_24_x86_64.whl", hash = "sha256:d344d37334c2b488dc02f04cb13c22cd61aa065eeb9bca7424588e0c8c23bdfb", size = 4045328 }, + { url = "https://files.pythonhosted.org/packages/2c/2a/abac2e42c3cc56f2b5020e58b99f700c4d3236d49451607add0f628d737b/lief-0.13.2-cp310-cp310-win32.whl", hash = "sha256:bc041b28b94139843a33c014e355822a9276b35f3c5ae10d82da56bf572f8222", size = 2493454 }, + { url = "https://files.pythonhosted.org/packages/ed/14/34a12787dc4328227e0e84a97db8142aa1e2b33e0aabc538e93abf7d6e5a/lief-0.13.2-cp310-cp310-win_amd64.whl", hash = "sha256:01d4075bbc3541e9dd3ef008045fa1eb128294a0c5b0c1f69ce60d8948d248c7", size = 3089949 }, + { url = "https://files.pythonhosted.org/packages/2e/95/9d7377095fb7cf195aca8f64d9696705c71884dcba16663472ce17139b9c/lief-0.13.2-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:6570dacebe107ad60c2ba0968d1a865d316009d43cc85af3719d3eeb0911abf3", size = 3424752 }, + { url = "https://files.pythonhosted.org/packages/00/2b/7ac8e15ca198a5c50397aec32102e81ef97fd573a4285ee889ec9084d110/lief-0.13.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7ce2e3f7c791efba327c2bb3499dbef81e682027109045a9bae696c62e2aeeb0", size = 3249263 }, + { url = "https://files.pythonhosted.org/packages/d6/8d/b50cc4ad91278015e5ac18fc76f32098ed6887c371bef6f4997af4cb97c9/lief-0.13.2-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:11ab900e0644b6735ecdef2bbd04439b4866a527650fc054470c195d6cfe2917", size = 3792343 }, + { url = "https://files.pythonhosted.org/packages/6b/bd/ea25e9c8ff0a55b5534e5881fa6e5eeca0ed3eeb7c772a276984b8c182d9/lief-0.13.2-cp311-cp311-manylinux_2_24_x86_64.whl", hash = "sha256:042ad2105a136b11a7494b9af8178468e8cb32b8fa2a0a55cb659a5605aeb069", size = 4045112 }, + { url = "https://files.pythonhosted.org/packages/d9/06/ddacd724f65fa8e7eca438c335aa77878a260fbc714cdba252387c33a4cc/lief-0.13.2-cp311-cp311-win32.whl", hash = "sha256:1ce289b6ab3cf4be654270007e8a2c0d2e42116180418c29d3ce83762955de63", size = 2493336 }, + { url = "https://files.pythonhosted.org/packages/82/95/1de9a497946fed9d15f847d8a4a0630dfda6d186c044f8731f53d0d3d758/lief-0.13.2-cp311-cp311-win_amd64.whl", hash = "sha256:eccb248ffb598e410fd2ef7c1f171a3cde57a40c9bb8c4fa15d8e7b90eb4eb2d", size = 3090328 }, +] + +[[package]] +name = "mypy" +version = "1.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mypy-extensions" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/28/d8a8233ff167d06108e53b7aefb4a8d7350adbbf9d7abd980f17fdb7a3a6/mypy-1.4.1.tar.gz", hash = "sha256:9bbcd9ab8ea1f2e1c8031c21445b511442cc45c89951e49bbf852cbb70755b1b", size = 2855162 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/3b/1c7363863b56c059f60a1dfdca9ac774a22ba64b7a4da0ee58ee53e5243f/mypy-1.4.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:566e72b0cd6598503e48ea610e0052d1b8168e60a46e0bfd34b3acf2d57f96a8", size = 10451043 }, + { url = "https://files.pythonhosted.org/packages/a7/24/6f0df1874118839db1155fed62a4bd7e80c181367ff8ea07d40fbaffcfb4/mypy-1.4.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ca637024ca67ab24a7fd6f65d280572c3794665eaf5edcc7e90a866544076878", size = 9542079 }, + { url = "https://files.pythonhosted.org/packages/04/5c/deeac94fcccd11aa621e6b350df333e1b809b11443774ea67582cc0205da/mypy-1.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0dde1d180cd84f0624c5dcaaa89c89775550a675aff96b5848de78fb11adabcd", size = 11974913 }, + { url = "https://files.pythonhosted.org/packages/e5/2f/de3c455c54e8cf5e37ea38705c1920f2df470389f8fc051084d2dd8c9c59/mypy-1.4.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8c4d8e89aa7de683e2056a581ce63c46a0c41e31bd2b6d34144e2c80f5ea53dc", size = 12044492 }, + { url = "https://files.pythonhosted.org/packages/e7/d3/6f65357dcb68109946de70cd55bd2e60f10114f387471302f48d54ff5dae/mypy-1.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:bfdca17c36ae01a21274a3c387a63aa1aafe72bff976522886869ef131b937f1", size = 8831655 }, + { url = "https://files.pythonhosted.org/packages/94/01/e34e37a044325af4d4af9825c15e8a0d26d89b5a9624b4d0908449d3411b/mypy-1.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7549fbf655e5825d787bbc9ecf6028731973f78088fbca3a1f4145c39ef09462", size = 10338636 }, + { url = "https://files.pythonhosted.org/packages/92/58/ccc0b714ecbd1a64b34d8ce1c38763ff6431de1d82551904ecc3711fbe05/mypy-1.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:98324ec3ecf12296e6422939e54763faedbfcc502ea4a4c38502082711867258", size = 9444172 }, + { url = "https://files.pythonhosted.org/packages/73/72/dfc0b46e6905eafd598e7c48c0c4f2e232647e4e36547425c64e6c850495/mypy-1.4.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:141dedfdbfe8a04142881ff30ce6e6653c9685b354876b12e4fe6c78598b45e2", size = 11855450 }, + { url = "https://files.pythonhosted.org/packages/66/f4/60739a2d336f3adf5628e7c9b920d16e8af6dc078550d615e4ba2a1d7759/mypy-1.4.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8207b7105829eca6f3d774f64a904190bb2231de91b8b186d21ffd98005f14a7", size = 11928679 }, + { url = "https://files.pythonhosted.org/packages/8c/26/6ff2b55bf8b605a4cc898883654c2ca4dd4feedf0bb04ecaacf60d165cde/mypy-1.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:16f0db5b641ba159eff72cff08edc3875f2b62b2fa2bc24f68c1e7a4e8232d01", size = 8831134 }, + { url = "https://files.pythonhosted.org/packages/3d/9a/e13addb8d652cb068f835ac2746d9d42f85b730092f581bb17e2059c28f1/mypy-1.4.1-py3-none-any.whl", hash = "sha256:45d32cec14e7b97af848bddd97d85ea4f0db4d5a149ed9676caa4eb2f7402bb4", size = 2451741 }, +] + +[[package]] +name = "mypy-extensions" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/98/a4/1ab47638b92648243faf97a5aeb6ea83059cc3624972ab6b8d2316078d3f/mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782", size = 4433 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/e2/5d3f6ada4297caebe1a2add3b126fe800c96f56dbe5d1988a2cbe0b267aa/mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d", size = 4695 }, +] + +[[package]] +name = "psutil" +version = "6.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/26/10/2a30b13c61e7cf937f4adf90710776b7918ed0a9c434e2c38224732af310/psutil-6.1.0.tar.gz", hash = "sha256:353815f59a7f64cdaca1c0307ee13558a0512f6db064e92fe833784f08539c7a", size = 508565 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/01/9e/8be43078a171381953cfee33c07c0d628594b5dbfc5157847b85022c2c1b/psutil-6.1.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:6e2dcd475ce8b80522e51d923d10c7871e45f20918e027ab682f94f1c6351688", size = 247762 }, + { url = "https://files.pythonhosted.org/packages/1d/cb/313e80644ea407f04f6602a9e23096540d9dc1878755f3952ea8d3d104be/psutil-6.1.0-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:0895b8414afafc526712c498bd9de2b063deaac4021a3b3c34566283464aff8e", size = 248777 }, + { url = "https://files.pythonhosted.org/packages/65/8e/bcbe2025c587b5d703369b6a75b65d41d1367553da6e3f788aff91eaf5bd/psutil-6.1.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9dcbfce5d89f1d1f2546a2090f4fcf87c7f669d1d90aacb7d7582addece9fb38", size = 284259 }, + { url = "https://files.pythonhosted.org/packages/58/4d/8245e6f76a93c98aab285a43ea71ff1b171bcd90c9d238bf81f7021fb233/psutil-6.1.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:498c6979f9c6637ebc3a73b3f87f9eb1ec24e1ce53a7c5173b8508981614a90b", size = 287255 }, + { url = "https://files.pythonhosted.org/packages/27/c2/d034856ac47e3b3cdfa9720d0e113902e615f4190d5d1bdb8df4b2015fb2/psutil-6.1.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d905186d647b16755a800e7263d43df08b790d709d575105d419f8b6ef65423a", size = 288804 }, + { url = "https://files.pythonhosted.org/packages/ea/55/5389ed243c878725feffc0d6a3bc5ef6764312b6fc7c081faaa2cfa7ef37/psutil-6.1.0-cp37-abi3-win32.whl", hash = "sha256:1ad45a1f5d0b608253b11508f80940985d1d0c8f6111b5cb637533a0e6ddc13e", size = 250386 }, + { url = "https://files.pythonhosted.org/packages/11/91/87fa6f060e649b1e1a7b19a4f5869709fbf750b7c8c262ee776ec32f3028/psutil-6.1.0-cp37-abi3-win_amd64.whl", hash = "sha256:a8fb3752b491d246034fa4d279ff076501588ce8cbcdbb62c32fd7a377d996be", size = 254228 }, +] + +[[package]] +name = "pycparser" +version = "2.22" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1d/b2/31537cf4b1ca988837256c910a668b553fceb8f069bedc4b1c826024b52c/pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6", size = 172736 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/13/a3/a812df4e2dd5696d1f351d58b8fe16a405b234ad2886a0dab9183fb78109/pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc", size = 117552 }, +] + +[[package]] +name = "pyperf" +version = "2.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "psutil" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/02/2a/758b3c4cc9843bd385bc595b777345fbf4cd00733b7830cdff43e30002c0/pyperf-2.8.0.tar.gz", hash = "sha256:b30a20465819daf102b6543b512f6799a5a879ff2a123981e6cd732d0e6a7a79", size = 225186 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/f7/bb8965520a9b0a3d720b282e67b5cb7f3305b96e4bacaee2794550e67e94/pyperf-2.8.0-py3-none-any.whl", hash = "sha256:1a775b5a09882f18bf876430ef78e07646f773f50774546f5f6a8b34d60e3968", size = 142508 }, +] + +[[package]] +name = "pyzmq" +version = "25.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "implementation_name == 'pypy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/64/9c/2b2614b0b86ff703b3a33ea5e044923bd7d100adc8c829d579a9b71ea9e7/pyzmq-25.1.0.tar.gz", hash = "sha256:80c41023465d36280e801564a69cbfce8ae85ff79b080e1913f6e90481fb8957", size = 1224640 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/53/17/6a70f84b79e361af34f6c99064ecf9e87112c4c48b9c7ea78f8e680b57d8/pyzmq-25.1.0-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:1a6169e69034eaa06823da6a93a7739ff38716142b3596c180363dee729d713d", size = 1826810 }, + { url = "https://files.pythonhosted.org/packages/2f/53/fc7dbdd32e275aee0961e2a5bed1bb64223846f959fd6e0c9a39aab08eed/pyzmq-25.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:19d0383b1f18411d137d891cab567de9afa609b214de68b86e20173dc624c101", size = 1236489 }, + { url = "https://files.pythonhosted.org/packages/04/0b/bff5b6c1680e248bad2df8248a060645709fe2aef9689e9f7c81c587bad4/pyzmq-25.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1e931d9a92f628858a50f5bdffdfcf839aebe388b82f9d2ccd5d22a38a789dc", size = 864304 }, + { url = "https://files.pythonhosted.org/packages/5e/9e/32074bd8bcf2a5cf282d8817458fd5479c68b487b6c3a5d4627711ad38f5/pyzmq-25.1.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:97d984b1b2f574bc1bb58296d3c0b64b10e95e7026f8716ed6c0b86d4679843f", size = 1116061 }, + { url = "https://files.pythonhosted.org/packages/fa/fb/a114ba641eb873c165106d3c8ee75eb49d6ea3204168808708d866de360d/pyzmq-25.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:154bddda2a351161474b36dba03bf1463377ec226a13458725183e508840df89", size = 1065090 }, + { url = "https://files.pythonhosted.org/packages/ca/db/f9976803f1a660e753d0f2426065975bad5db8272fd5284efaf488dc0ce1/pyzmq-25.1.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:cb6d161ae94fb35bb518b74bb06b7293299c15ba3bc099dccd6a5b7ae589aee3", size = 1062464 }, + { url = "https://files.pythonhosted.org/packages/94/3a/c3964c0a86c3535ae240799d3b7c8e13527e7a092080dda9012b1401fa86/pyzmq-25.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:90146ab578931e0e2826ee39d0c948d0ea72734378f1898939d18bc9c823fcf9", size = 1391159 }, + { url = "https://files.pythonhosted.org/packages/a1/87/92556ffa8fbe7dc497d847e39d5c46134f9ad047b23f5bcefc8fbd0c2c9c/pyzmq-25.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:831ba20b660b39e39e5ac8603e8193f8fce1ee03a42c84ade89c36a251449d80", size = 1721009 }, + { url = "https://files.pythonhosted.org/packages/66/96/129706be681649f43bde93811416f566acfefcd3fb18156d5df349c360ab/pyzmq-25.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:3a522510e3434e12aff80187144c6df556bb06fe6b9d01b2ecfbd2b5bfa5c60c", size = 1611290 }, + { url = "https://files.pythonhosted.org/packages/64/db/e19f69fe9b1a4e53f6382274f553358e2e7305d2a2b9d9db36087bf52d5e/pyzmq-25.1.0-cp310-cp310-win32.whl", hash = "sha256:be24a5867b8e3b9dd5c241de359a9a5217698ff616ac2daa47713ba2ebe30ad1", size = 880070 }, + { url = "https://files.pythonhosted.org/packages/32/e4/ce4f94009f84c2a688082c2674d490d2e20e0c9058087f5358a2bf29ddf1/pyzmq-25.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:5693dcc4f163481cf79e98cf2d7995c60e43809e325b77a7748d8024b1b7bcba", size = 1137827 }, + { url = "https://files.pythonhosted.org/packages/bb/80/ae792378f98d6d0e39c975c334603d3d2535f7897707fe91f31d37f94fdb/pyzmq-25.1.0-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:13bbe36da3f8aaf2b7ec12696253c0bf6ffe05f4507985a8844a1081db6ec22d", size = 1816147 }, + { url = "https://files.pythonhosted.org/packages/5a/b6/3c2ddd09aa24352e4f6aade53e9b9a1816c0774c844f11b1a2f508ddc0be/pyzmq-25.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:69511d604368f3dc58d4be1b0bad99b61ee92b44afe1cd9b7bd8c5e34ea8248a", size = 1230845 }, + { url = "https://files.pythonhosted.org/packages/26/bb/80535157e8811095901f98688839092afb6dcaf2ff154aa8fa2e575f540d/pyzmq-25.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a983c8694667fd76d793ada77fd36c8317e76aa66eec75be2653cef2ea72883", size = 866042 }, + { url = "https://files.pythonhosted.org/packages/7c/65/bccec1eae7c0e089d90648f350e6c2ff40ccb8c6d1b929548f4cd304b1f7/pyzmq-25.1.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:332616f95eb400492103ab9d542b69d5f0ff628b23129a4bc0a2fd48da6e4e0b", size = 1116285 }, + { url = "https://files.pythonhosted.org/packages/b7/cb/2a36d3eed310efb342fbb7b4adf6b05f46401c4b937154bd1c9b703314e0/pyzmq-25.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58416db767787aedbfd57116714aad6c9ce57215ffa1c3758a52403f7c68cff5", size = 1066280 }, + { url = "https://files.pythonhosted.org/packages/66/f5/15db4c297957f049cd4dcd35eb7fbe9098a72489e0abdb289c529d7327cc/pyzmq-25.1.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:cad9545f5801a125f162d09ec9b724b7ad9b6440151b89645241d0120e119dcc", size = 1061673 }, + { url = "https://files.pythonhosted.org/packages/fa/40/7729719e38324e5e9f2e77f6131fc253f063a3741eab170ef610196098e8/pyzmq-25.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d6128d431b8dfa888bf51c22a04d48bcb3d64431caf02b3cb943269f17fd2994", size = 1393337 }, + { url = "https://files.pythonhosted.org/packages/fd/12/0324dcb2554cd3f2ebb851ddbfbac27c4bb384394ba4a8978dec093fe71d/pyzmq-25.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:2b15247c49d8cbea695b321ae5478d47cffd496a2ec5ef47131a9e79ddd7e46c", size = 1723679 }, + { url = "https://files.pythonhosted.org/packages/04/15/b8ab292f0b74e0440547185fb67167c87454a2b3be429d64de569f7142a2/pyzmq-25.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:442d3efc77ca4d35bee3547a8e08e8d4bb88dadb54a8377014938ba98d2e074a", size = 1612761 }, + { url = "https://files.pythonhosted.org/packages/22/3e/3670e36c6f42e124492ddd2af550ca13bd4a9f1edd562e1ae7c35a1f230b/pyzmq-25.1.0-cp311-cp311-win32.whl", hash = "sha256:65346f507a815a731092421d0d7d60ed551a80d9b75e8b684307d435a5597425", size = 878704 }, + { url = "https://files.pythonhosted.org/packages/a0/db/4e586c563b48dec09b8f7c2728b905e29db61af89b5c58e4eba9ad36fdec/pyzmq-25.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:8b45d722046fea5a5694cba5d86f21f78f0052b40a4bbbbf60128ac55bfcc7b6", size = 1135692 }, +] + +[[package]] +name = "toml" +version = "0.10.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/be/ba/1f744cdc819428fc6b5084ec34d9b30660f6f9daaf70eead706e3203ec3c/toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f", size = 22253 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", size = 16588 }, +] + +[[package]] +name = "tomli" +version = "2.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/35/b9/de2a5c0144d7d75a57ff355c0c24054f965b2dc3036456ae03a51ea6264b/tomli-2.0.2.tar.gz", hash = "sha256:d46d457a85337051c36524bc5349dd91b1877838e2979ac5ced3e710ed8a60ed", size = 16096 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cf/db/ce8eda256fa131af12e0a76d481711abe4681b6923c27efb9a255c9e4594/tomli-2.0.2-py3-none-any.whl", hash = "sha256:2ebe24485c53d303f690b0ec092806a085f07af5a5aa1464f3931eec36caaa38", size = 13237 }, +] + +[[package]] +name = "typing-extensions" +version = "4.12.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/df/db/f35a00659bc03fec321ba8bce9420de607a1d37f8342eee1863174c69557/typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8", size = 85321 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/26/9f/ad63fc0248c5379346306f8668cda6e2e2e9c95e01216d2b8ffd9ff037d0/typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d", size = 37438 }, +] + +[[package]] +name = "vulture" +version = "2.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "toml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b9/18/e51a6e575047d19dbcd7394f05b2afa6191fe9ce30bd5bcfb3f850501e0c/vulture-2.6.tar.gz", hash = "sha256:2515fa848181001dc8a73aba6a01a1a17406f5d372f24ec7f7191866f9f4997e", size = 53777 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/9d/3c4df0c704ddb5ecf07fcd92cfe6d4a5dc000b7f5459afcb7e98a2ffea1e/vulture-2.6-py2.py3-none-any.whl", hash = "sha256:e792e903ccc063ec4873a8979dcf11b51ea3d65a2d3b31c113d47be48f0cdcae", size = 26494 }, +] From 5dacff496d2babd5e1f865482453d6a7b43560f2 Mon Sep 17 00:00:00 2001 From: willcl-ark Date: Tue, 28 Oct 2025 22:07:46 +0000 Subject: [PATCH 03/48] clone guix into bench-ci To update in a rebase simply do: ``` mkdir -p bench-ci cp -r contrib/guix bench-ci git add -f bench-ci/guix ``` --- bench-ci/guix/INSTALL.md | 814 ++++++++++++++++++ bench-ci/guix/README.md | 430 +++++++++ bench-ci/guix/guix-attest | 263 ++++++ bench-ci/guix/guix-build | 474 ++++++++++ bench-ci/guix/guix-clean | 83 ++ bench-ci/guix/guix-codesign | 384 +++++++++ bench-ci/guix/guix-verify | 174 ++++ bench-ci/guix/libexec/build.sh | 403 +++++++++ bench-ci/guix/libexec/codesign.sh | 153 ++++ bench-ci/guix/libexec/prelude.bash | 102 +++ bench-ci/guix/manifest.scm | 574 ++++++++++++ .../patches/binutils-unaligned-default.patch | 22 + .../guix/patches/gcc-remap-guix-store.patch | 20 + .../guix/patches/glibc-2.42-guix-prefix.patch | 47 + bench-ci/guix/patches/glibc-guix-prefix.patch | 16 + .../guix/patches/glibc-riscv-jumptarget.patch | 57 ++ bench-ci/guix/patches/lief-scikit-0-9.patch | 21 + .../patches/oscrypto-hard-code-openssl.patch | 13 + .../winpthreads-remap-guix-store.patch | 17 + bench-ci/guix/security-check.py | 293 +++++++ bench-ci/guix/symbol-check.py | 335 +++++++ 21 files changed, 4695 insertions(+) create mode 100644 bench-ci/guix/INSTALL.md create mode 100644 bench-ci/guix/README.md create mode 100755 bench-ci/guix/guix-attest create mode 100755 bench-ci/guix/guix-build create mode 100755 bench-ci/guix/guix-clean create mode 100755 bench-ci/guix/guix-codesign create mode 100755 bench-ci/guix/guix-verify create mode 100755 bench-ci/guix/libexec/build.sh create mode 100755 bench-ci/guix/libexec/codesign.sh create mode 100644 bench-ci/guix/libexec/prelude.bash create mode 100644 bench-ci/guix/manifest.scm create mode 100644 bench-ci/guix/patches/binutils-unaligned-default.patch create mode 100644 bench-ci/guix/patches/gcc-remap-guix-store.patch create mode 100644 bench-ci/guix/patches/glibc-2.42-guix-prefix.patch create mode 100644 bench-ci/guix/patches/glibc-guix-prefix.patch create mode 100644 bench-ci/guix/patches/glibc-riscv-jumptarget.patch create mode 100644 bench-ci/guix/patches/lief-scikit-0-9.patch create mode 100644 bench-ci/guix/patches/oscrypto-hard-code-openssl.patch create mode 100644 bench-ci/guix/patches/winpthreads-remap-guix-store.patch create mode 100755 bench-ci/guix/security-check.py create mode 100755 bench-ci/guix/symbol-check.py diff --git a/bench-ci/guix/INSTALL.md b/bench-ci/guix/INSTALL.md new file mode 100644 index 000000000000..f9a79f66349c --- /dev/null +++ b/bench-ci/guix/INSTALL.md @@ -0,0 +1,814 @@ +# Guix Installation and Setup + +This only needs to be done once per machine. If you have already completed the +installation and setup, please proceed to [perform a build](./README.md). + +Otherwise, you may choose from one of the following options to install Guix: + +1. Using the official **shell installer script** [⤓ skip to section][install-script] + - Maintained by Guix developers + - Easiest (automatically performs *most* setup) + - Works on nearly all Linux distributions + - Only installs latest release + - Binary installation only, requires high level of trust + - Note: The script needs to be run as root, so it should be inspected before it's run +2. Using the official **binary tarball** [⤓ skip to section][install-bin-tarball] + - Maintained by Guix developers + - Normal difficulty (full manual setup required) + - Works on nearly all Linux distributions + - Installs any release + - Binary installation only, requires high level of trust +3. Using fanquake's **container image** [↗︎ external instructions][install-fanquake-container] + - Maintained by fanquake + - Easy (automatically performs *some* setup) + - Works wherever container images work (Docker/Podman) + - Installs any release + - Binary installation only, requires high level of trust +4. Using a **distribution-maintained package** [⤓ skip to section][install-distro-pkg] + - Maintained by distribution's Guix package maintainer + - Normal difficulty (manual setup required) + - Works only on distributions with Guix packaged, see: https://repology.org/project/guix/versions + - Installs a release decided on by package maintainer + - Source or binary installation depending on the distribution +5. Building **from source** [⤓ skip to section][install-source] + - Maintained by you + - Hard, but rewarding + - Can be made to work on most Linux distributions + - Installs any commit (more granular) + - Source installation, requires lower level of trust + +## Options 1 and 2: Using the official shell installer script or binary tarball + +The installation instructions for both the official shell installer script and +the binary tarballs can be found in the GNU Guix Manual's [Binary Installation +section](https://guix.gnu.org/manual/en/html_node/Binary-Installation.html). + +Note that running through the binary tarball installation steps is largely +equivalent to manually performing what the shell installer script does. + +Note that at the time of writing (July 5th, 2021), the shell installer script +automatically creates an `/etc/profile.d` entry which the binary tarball +installation instructions do not ask you to create. However, you will likely +need this entry for better desktop integration. Please see [this +section](#add-an-etcprofiled-entry) for instructions on how to add a +`/etc/profile.d/guix.sh` entry. + +Regardless of which installation option you chose, the changes to +`/etc/profile.d` will not take effect until the next shell or desktop session, +so you should log out and log back in. + +## Option 3: Using fanquake's container image + +Please refer to fanquake's instructions +[here](https://github.com/fanquake/core-review/tree/master/guix). + +## Option 4: Using a distribution-maintained package + +Note that this section is based on the distro packaging situation at the time of +writing (July 2021). Guix is expected to be more widely packaged over time. For +an up-to-date view on Guix's package status/version across distros, please see: +https://repology.org/project/guix/versions + +### Debian / Ubuntu + +Guix is available as a distribution package in [Debian +](https://packages.debian.org/search?keywords=guix) and [Ubuntu +](https://packages.ubuntu.com/search?keywords=guix). + +To install: +```sh +sudo apt install guix +``` + +### Arch Linux + +Guix is available in the AUR as +[`guix`](https://aur.archlinux.org/packages/guix/), please follow the +installation instructions in the Arch Linux Wiki ([live +link](https://wiki.archlinux.org/index.php/Guix#AUR_Package_Installation), +[2021/03/30 +permalink](https://wiki.archlinux.org/index.php?title=Guix&oldid=637559#AUR_Package_Installation)) +to install Guix. + +At the time of writing (2021/03/30), the `check` phase will fail if the path to +guix's build directory is longer than 36 characters due to an anachronistic +character limit on the shebang line. Since the `check` phase happens after the +`build` phase, which may take quite a long time, it is recommended that users +either: + +1. Skip the `check` phase + - For `makepkg`: `makepkg --nocheck ...` + - For `yay`: `yay --mflags="--nocheck" ...` + - For `paru`: `paru --nocheck ...` +2. Or, check their build directory's length beforehand + - For those building with `makepkg`: `pwd | wc -c` + +## Option 5: Building from source + +Building Guix from source is a rather involved process but a rewarding one for +those looking to minimize trust and maximize customizability (e.g. building a +particular commit of Guix). Previous experience with using autotools-style build +systems to build packages from source will be helpful. *hic sunt dracones.* + +I strongly urge you to at least skim through the entire section once before you +start issuing commands, as it will save you a lot of unnecessary pain and +anguish. + +### Installing common build tools + +There are a few basic build tools that are required for most things we'll build, +so let's install them now: + +Text transformation/i18n: +- `autopoint` (sometimes packaged in `gettext`) +- `help2man` +- `po4a` +- `texinfo` + +Build system tools: +- `g++` w/ C++11 support +- `libtool` +- `autoconf` +- `automake` +- `pkg-config` (sometimes packaged as `pkgconf`) +- `make` +- `cmake` + +Miscellaneous: +- `git` +- `gnupg` +- `python3` + +### Building and Installing Guix's dependencies + +In order to build Guix itself from source, we need to first make sure that the +necessary dependencies are installed and discoverable. The most up-to-date list +of Guix's dependencies is kept in the ["Requirements" +section](https://guix.gnu.org/manual/en/html_node/Requirements.html) of the Guix +Reference Manual. + +Depending on your distribution, most or all of these dependencies may already be +packaged and installable without manually building and installing. + +For reference, the graphic below outlines Guix v1.3.0's dependency graph: + +![bootstrap map](https://user-images.githubusercontent.com/6399679/125064185-a9a59880-e0b0-11eb-82c1-9b8e5dc9950d.png) + +If you do not care about building each dependency from source, and Guix is +already packaged for your distribution, you can easily install only the build +dependencies of Guix. For example, to enable deb-src and install the Guix build +dependencies on Ubuntu/Debian: + +```sh +sed -i 's|# deb-src|deb-src|g' /etc/apt/sources.list +apt update +apt-get build-dep -y guix +``` + +If this succeeded, you can likely skip to section +["Building and Installing Guix itself"](#building-and-installing-guix-itself). + +#### Guile + +###### Corner case: Multiple versions of Guile on one system + +It is recommended to only install the required version of Guile, so that build systems do +not get confused about which Guile to use. + +However, if you insist on having more versions of Guile installed on +your system, then you need to **consistently** specify +`GUILE_EFFECTIVE_VERSION=3.0` to all +`./configure` invocations for Guix and its dependencies. + +##### Installing Guile + +If your distribution splits packages into `-dev`-suffixed and +non-`-dev`-suffixed sub-packages (as is the case for Debian-derived +distributions), please make sure to install both. For example, to install Guile +v3.0 on Debian/Ubuntu: + +```sh +apt install guile-3.0 guile-3.0-dev +``` + +#### Mixing distribution packages and source-built packages + +At the time of writing, most distributions have _some_ of Guix's dependencies +packaged, but not all. This means that you may want to install the distribution +package for some dependencies, and manually build-from-source for others. + +Distribution packages usually install to `/usr`, which is different from the +default `./configure` prefix of source-built packages: `/usr/local`. + +This means that if you mix-and-match distribution packages and source-built +packages and do not specify exactly `--prefix=/usr` to `./configure` for +source-built packages, you will need to augment the `GUILE_LOAD_PATH` and +`GUILE_LOAD_COMPILED_PATH` environment variables so that Guile will look +under the right prefix and find your source-built packages. + +For example, if you are using Guile v3.0, and have Guile packages in the +`/usr/local` prefix, either add the following lines to your `.profile` or +`.bash_profile` so that the environment variable is properly set for all future +shell logins, or paste the lines into a POSIX-style shell to temporarily modify +the environment variables of your current shell session. + +```sh +# Help Guile v3.0.x find packages in /usr/local +export GUILE_LOAD_PATH="/usr/local/share/guile/site/3.0${GUILE_LOAD_PATH:+:}$GUILE_LOAD_PATH" +export GUILE_LOAD_COMPILED_PATH="/usr/local/lib/guile/3.0/site-ccache${GUILE_LOAD_COMPILED_PATH:+:}$GUILE_COMPILED_LOAD_PATH" +``` + +Note that these environment variables are used to check for packages during +`./configure`, so they should be set as soon as possible should you want to use +a prefix other than `/usr`. + +#### Building and installing source-built packages + +***IMPORTANT**: A few dependencies have non-obvious quirks/errata which are +documented in the sub-sections immediately below. Please read these sections +before proceeding to build and install these packages.* + +Although you should always refer to the README or INSTALL files for the most +accurate information, most of these dependencies use autoconf-style build +systems (check if there's a `configure.ac` file), and will likely do the right +thing with the following: + +Clone the repository and check out the latest release: +```sh +git clone /.git +cd +git tag -l # check for the latest release +git checkout +``` + +For autoconf-based build systems (if `./autogen.sh` or `configure.ac` exists at +the root of the repository): + +```sh +./autogen.sh || autoreconf -vfi +./configure --prefix= +make +sudo make install +``` + +For CMake-based build systems (if `CMakeLists.txt` exists at the root of the +repository): + +```sh +mkdir build && cd build +cmake .. -DCMAKE_INSTALL_PREFIX= +sudo cmake --build . --target install +``` + +If you choose not to specify exactly `--prefix=/usr` to `./configure`, please +make sure you've carefully read the [previous section] on mixing distribution +packages and source-built packages. + +##### Binding packages require `-dev`-suffixed packages + +Relevant for: +- Everyone + +When building bindings, the `-dev`-suffixed version of the original package +needs to be installed. For example, building `Guile-zlib` on Debian-derived +distributions requires that `zlib1g-dev` is installed. + +When using bindings, the `-dev`-suffixed version of the original package still +needs to be installed. This is particularly problematic when distribution +packages are mispackaged like `guile-sqlite3` is in Ubuntu Focal such that +installing `guile-sqlite3` does not automatically install `libsqlite3-dev` as a +dependency. + +Below is a list of relevant Guile bindings and their corresponding `-dev` +packages in Debian at the time of writing. + +| Guile binding package | -dev Debian package | +|-----------------------|---------------------| +| guile-gcrypt | libgcrypt-dev | +| guile-git | libgit2-dev | +| guile-gnutls | (none) | +| guile-json | (none) | +| guile-lzlib | liblz-dev | +| guile-ssh | libssh-dev | +| guile-sqlite3 | libsqlite3-dev | +| guile-zlib | zlib1g-dev | + +##### `guile-git` actually depends on `libgit2 >= 1.1` + +Relevant for: +- Those building `guile-git` from source against `libgit2 < 1.1` +- Those installing `guile-git` from their distribution where `guile-git` is + built against `libgit2 < 1.1` + +As of v0.5.2, `guile-git` claims to only require `libgit2 >= 0.28.0`, however, +it actually requires `libgit2 >= 1.1`, otherwise, it will be confused by a +reference of `origin/keyring`: instead of interpreting the reference as "the +'keyring' branch of the 'origin' remote", the reference is interpreted as "the +branch literally named 'origin/keyring'" + +This is especially notable because Ubuntu Focal packages `libgit2 v0.28.4`, and +`guile-git` is built against it. + +Should you be in this situation, you need to build both `libgit2 v1.1.x` and +`guile-git` from source. + +Source: https://logs.guix.gnu.org/guix/2020-11-12.log#232527 + +### Building and Installing Guix itself + +Start by cloning Guix: + +``` +git clone https://codeberg.org/guix/guix.git +cd guix +``` + +You will likely want to build the latest release. +At the time of writing (November 2023), the latest release was `v1.4.0`. + +``` +git branch -a -l 'origin/version-*' # check for the latest release +git checkout +``` + +Bootstrap the build system: +``` +./bootstrap +``` + +Configure with the recommended `--localstatedir` flag: +``` +./configure --localstatedir=/var +``` + +Note: If you intend to hack on Guix in the future, you will need to supply the +same `--localstatedir=` flag for all future Guix `./configure` invocations. See +the last paragraph of this +[section](https://guix.gnu.org/manual/en/html_node/Requirements.html) for more +details. + +Build Guix (this will take a while): +``` +make -j$(nproc) +``` + +Install Guix: + +``` +sudo make install +``` + +### Post-"build from source" Setup + +#### Creating and starting a `guix-daemon-original` service with a fixed `argv[0]` + +At this point, guix will be installed to `${bindir}`, which is likely +`/usr/local/bin` if you did not override directory variables at +`./configure`-time. More information on standard Automake directory variables +can be found +[here](https://www.gnu.org/software/automake/manual/html_node/Standard-Directory-Variables.html). + +However, the Guix init scripts and service configurations for Upstart, systemd, +SysV, and OpenRC are installed (in `${libdir}`) to launch +`${localstatedir}/guix/profiles/per-user/root/current-guix/bin/guix-daemon`, +which does not yet exist, and will only exist after [`root` performs their first +`guix pull`](#guix-pull-as-root). + +We need to create a `-original` version of these init scripts that's pointed to +the binaries we just built and `make install`'ed in `${bindir}` (normally, +`/usr/local/bin`). + +Example for `systemd`, run as `root`: + +```sh +# Create guix-daemon-original.service by modifying guix-daemon.service +libdir=# set according to your PREFIX (default is /usr/local/lib) +bindir="$(dirname $(command -v guix-daemon))" +sed -E -e "s|/\S*/guix/profiles/per-user/root/current-guix/bin/guix-daemon|${bindir}/guix-daemon|" "${libdir}"/systemd/system/guix-daemon.service > /etc/systemd/system/guix-daemon-original.service +chmod 664 /etc/systemd/system/guix-daemon-original.service + +# Make systemd recognize the new service +systemctl daemon-reload + +# Make sure that the non-working guix-daemon.service is stopped and disabled +systemctl stop guix-daemon +systemctl disable guix-daemon + +# Make sure that the working guix-daemon-original.service is started and enabled +systemctl enable guix-daemon-original +systemctl start guix-daemon-original +``` + +#### Creating `guix-daemon` users / groups + +Please see the [relevant +section](https://guix.gnu.org/manual/en/html_node/Build-Environment-Setup.html) +in the Guix Reference Manual for more details. + +## Optional setup + +At this point, you are set up to [use Guix to build Bitcoin +Core](./README.md#usage). However, if you want to polish your setup a bit and +make it "what Guix intended", then read the next few subsections. + +### Add an `/etc/profile.d` entry + +This section definitely does not apply to you if you installed Guix using: +1. The shell installer script +2. fanquake's container image +3. Debian's `guix` package + +#### Background + +Although Guix knows how to update itself and its packages, it does so in a +non-invasive way (it does not modify `/usr/local/bin/guix`). + +Instead, it does the following: + +- After a `guix pull`, it updates + `/var/guix/profiles/per-user/$USER/current-guix`, and creates a symlink + targeting this directory at `$HOME/.config/guix/current` + +- After a `guix install`, it updates + `/var/guix/profiles/per-user/$USER/guix-profile`, and creates a symlink + targeting this directory at `$HOME/.guix-profile` + +Therefore, in order for these operations to affect your shell/desktop sessions +(and for the principle of least astonishment to hold), their corresponding +directories have to be added to well-known environment variables like `$PATH`, +`$INFOPATH`, `$XDG_DATA_DIRS`, etc. + +In other words, if `$HOME/.config/guix/current/bin` does not exist in your +`$PATH`, a `guix pull` will have no effect on what `guix` you are using. Same +goes for `$HOME/.guix-profile/bin`, `guix install`, and installed packages. + +Helpfully, after a `guix pull` or `guix install`, a message will be printed like +so: + +``` +hint: Consider setting the necessary environment variables by running: + + GUIX_PROFILE="$HOME/.guix-profile" + . "$GUIX_PROFILE/etc/profile" + +Alternately, see `guix package --search-paths -p "$HOME/.guix-profile"'. +``` + +However, this is somewhat tedious to do for both `guix pull` and `guix install` +for each user on the system that wants to properly use `guix`. I recommend that +you add an entry to `/etc/profile.d` instead. This is done by default +when installing the Debian package later than 1.2.0-4 and when using the shell +script installer. + +#### Instructions + +Create `/etc/profile.d/guix.sh` with the following content: +```sh +# _GUIX_PROFILE: `guix pull` profile +_GUIX_PROFILE="$HOME/.config/guix/current" +if [ -L $_GUIX_PROFILE ]; then + export PATH="$_GUIX_PROFILE/bin${PATH:+:}$PATH" + # Export INFOPATH so that the updated info pages can be found + # and read by both /usr/bin/info and/or $GUIX_PROFILE/bin/info + # When INFOPATH is unset, add a trailing colon so that Emacs + # searches 'Info-default-directory-list'. + export INFOPATH="$_GUIX_PROFILE/share/info:$INFOPATH" +fi + +# GUIX_PROFILE: User's default profile +GUIX_PROFILE="$HOME/.guix-profile" +[ -L $GUIX_PROFILE ] || return +GUIX_LOCPATH="$GUIX_PROFILE/lib/locale" +export GUIX_PROFILE GUIX_LOCPATH + +[ -f "$GUIX_PROFILE/etc/profile" ] && . "$GUIX_PROFILE/etc/profile" + +# set XDG_DATA_DIRS to include Guix installations +export XDG_DATA_DIRS="$GUIX_PROFILE/share:${XDG_DATA_DIRS:-/usr/local/share/:/usr/share/}" +``` + +Please note that this will not take effect until the next shell or desktop +session (log out and log back in). + +### `guix pull` as root + +Before you do this, you need to read the section on [choosing your security +model][security-model] and adjust `guix` and `guix-daemon` flags according to +your choice, as invoking `guix pull` may pull substitutes from substitute +servers (which you may not want). + +As mentioned in a previous section, Guix expects +`${localstatedir}/guix/profiles/per-user/root/current-guix` to be populated with +`root`'s Guix profile, `guix pull`-ed and built by some former version of Guix. +However, this is not the case when we build from source. Therefore, we need to +perform a `guix pull` as `root`: + +```sh +sudo --login guix pull --branch=version- +# or +sudo --login guix pull --commit= +``` + +`guix pull` is quite a long process (especially if you're using +`--no-substitutes`). If you encounter build problems, please refer to the +[troubleshooting section](#troubleshooting). + +Note that running a bare `guix pull` with no commit or branch specified will +pull the latest commit on Guix's master branch, which is likely fine, but not +recommended. + +If you installed Guix from source, you may get an error like the following: +```sh +error: while creating symlink '/root/.config/guix/current' No such file or directory +``` +To resolve this, simply: +``` +sudo mkdir -p /root/.config/guix +``` +Then try the `guix pull` command again. + +After the `guix pull` finishes successfully, +`${localstatedir}/guix/profiles/per-user/root/current-guix` should be populated. + +#### Using the newly-pulled `guix` by restarting the daemon + +Depending on how you installed Guix, you should now make sure that your init +scripts and service configurations point to the newly-pulled `guix-daemon`. + +##### If you built Guix from source + +If you followed the instructions for [fixing argv\[0\]][fix-argv0], you can now +do the following: + +```sh +systemctl stop guix-daemon-original +systemctl disable guix-daemon-original + +systemctl enable guix-daemon +systemctl start guix-daemon +``` + +Remember to set `--no-substitutes` in `$libdir/systemd/system/guix-daemon.service` and other customizations if you used them for `guix-daemon-original.service`. + +##### If you installed Guix via the Debian/Ubuntu distribution packages + +You will need to create a `guix-daemon-latest` service which points to the new +`guix` rather than a pinned one. + +```sh +# Create guix-daemon-latest.service by modifying guix-daemon.service +sed -E -e "s|/usr/bin/guix-daemon|/var/guix/profiles/per-user/root/current-guix/bin/guix-daemon|" /etc/systemd/system/guix-daemon.service > /lib/systemd/system/guix-daemon-latest.service +chmod 664 /lib/systemd/system/guix-daemon-latest.service + +# Make systemd recognize the new service +systemctl daemon-reload + +# Make sure that the old guix-daemon.service is stopped and disabled +systemctl stop guix-daemon +systemctl disable guix-daemon + +# Make sure that the new guix-daemon-latest.service is started and enabled +systemctl enable guix-daemon-latest +systemctl start guix-daemon-latest +``` + +##### If you installed Guix via lantw44's Arch Linux AUR package + +At the time of writing (July 5th, 2021) the systemd unit for "updated Guix" is +`guix-daemon-latest.service`, therefore, you should do the following: + +```sh +systemctl stop guix-daemon +systemctl disable guix-daemon + +systemctl enable guix-daemon-latest +systemctl start guix-daemon-latest +``` + +##### Otherwise... + +Simply do: + +```sh +systemctl restart guix-daemon +``` + +### Checking everything + +If you followed all the steps above to make your Guix setup "prim and proper," +you can check that you did everything properly by running through this +checklist. + +1. `/etc/profile.d/guix.sh` should exist and be sourced at each shell login + +2. `guix describe` should not print `guix describe: error: failed to determine + origin`, but rather something like: + + ``` + Generation 38 Feb 22 2021 16:39:31 (current) + guix f350df4 + repository URL: https://codeberg.org/guix/guix.git + branch: version-1.2.0 + commit: f350df405fbcd5b9e27e6b6aa500da7f101f41e7 + ``` + +3. `guix-daemon` should be running from `${localstatedir}/guix/profiles/per-user/root/current-guix` + +# Troubleshooting + +## Derivation failed to build + +When you see a build failure like below: + +``` +building /gnu/store/...-foo-3.6.12.drv... +/ 'check' phasenote: keeping build directory `/tmp/guix-build-foo-3.6.12.drv-0' +builder for `/gnu/store/...-foo-3.6.12.drv' failed with exit code 1 +build of /gnu/store/...-foo-3.6.12.drv failed +View build log at '/var/log/guix/drvs/../...-foo-3.6.12.drv.bz2'. +cannot build derivation `/gnu/store/...-qux-7.69.1.drv': 1 dependencies couldn't be built +cannot build derivation `/gnu/store/...-bar-3.16.5.drv': 1 dependencies couldn't be built +cannot build derivation `/gnu/store/...-baz-2.0.5.drv': 1 dependencies couldn't be built +guix time-machine: error: build of `/gnu/store/...-baz-2.0.5.drv' failed +``` + +It means that `guix` failed to build a package named `foo`, which was a +dependency of `qux`, `bar`, and `baz`. Importantly, note that the last "failed" +line is not necessarily the root cause, the first "failed" line is. + +Most of the time, the build failure is due to a spurious test failure or the +package's build system/test suite breaking when running multi-threaded. To +rebuild _just_ this derivation in a single-threaded fashion (please don't forget +to add other `guix` flags like `--no-substitutes` as appropriate): + +```sh +$ guix build --cores=1 /gnu/store/...-foo-3.6.12.drv +``` + +If the single-threaded rebuild did not succeed, you may need to dig deeper. +You may view `foo`'s build logs in `less` like so (please replace paths with the +path you see in the build failure output): + +```sh +$ bzcat /var/log/guix/drvs/../...-foo-3.6.12.drv.bz2 | less +``` + +`foo`'s build directory is also preserved and available at +`/tmp/guix-build-foo-3.6.12.drv-0`. However, if you fail to build `foo` multiple +times, it may be `/tmp/...drv-1` or `/tmp/...drv-2`. Always consult the build +failure output for the most accurate, up-to-date information. + +### python(-minimal): [Errno 84] Invalid or incomplete multibyte or wide character + +This error occurs when your `$TMPDIR` (default: /tmp) exists on a filesystem +which rejects characters not present in the UTF-8 character code set. An example +is ZFS with the utf8only=on option set. + +More information: https://github.com/python/cpython/issues/81765 + +### openssl-1.1.1l and openssl-1.1.1n + +OpenSSL includes tests that will fail once some certificate has expired. +The workarounds from the GnuTLS section immediately below can be used. + +For openssl-1.1.1l use 2022-05-01 as the date. + +### GnuTLS: test-suite FAIL: status-request-revoked + +*The derivation is likely identified by: `/gnu/store/vhphki5sg9xkdhh2pbc8gi6vhpfzryf0-gnutls-3.6.12.drv`* + +This unfortunate error is most common for non-substitute builders who installed +Guix v1.2.0. The problem stems from the fact that one of GnuTLS's tests uses a +hardcoded certificate which expired on 2020-10-24. + +What's more unfortunate is that this GnuTLS derivation is somewhat special in +Guix's dependency graph and is not affected by the package transformation flags +like `--without-tests=`. + +The easiest solution for those encountering this problem is to install a newer +version of Guix. However, there are ways to work around this issue: + +#### Workaround 1: Using substitutes for this single derivation + +If you've authorized the official Guix build farm's key (more info +[here](./README.md#step-1-authorize-the-signing-keys)), then you can use +substitutes just for this single derivation by invoking the following: + +```sh +guix build --substitute-urls="https://ci.guix.gnu.org" /gnu/store/vhphki5sg9xkdhh2pbc8gi6vhpfzryf0-gnutls-3.6.12.drv +``` + +See [this section](./README.md#removing-authorized-keys) for instructions on how +to remove authorized keys if you don't want to keep the build farm's key +authorized. + +#### Workaround 2: Temporarily setting the system clock back + +This workaround was described [here](https://issues.guix.gnu.org/44559#5). + +Basically: + +1. Turn off NTP +2. Set system time to 2020-10-01 +3. guix build --no-substitutes /gnu/store/vhphki5sg9xkdhh2pbc8gi6vhpfzryf0-gnutls-3.6.12.drv +4. Set system time back to accurate current time +5. Turn NTP back on + +For example, + +```sh +sudo timedatectl set-ntp no +sudo date --set "01 oct 2020 15:00:00" +guix build /gnu/store/vhphki5sg9xkdhh2pbc8gi6vhpfzryf0-gnutls-3.6.12.drv +sudo timedatectl set-ntp yes +``` + +#### Workaround 3: Disable the tests in the Guix source code for this single derivation + +If all of the above workarounds fail, you can also disable the `tests` phase of +the derivation via the `arguments` option, as described in the official +[`package` +reference](https://guix.gnu.org/manual/en/html_node/package-Reference.html). + +For example, to disable the openssl-1.1 check phase: + +```diff +diff --git a/gnu/packages/tls.scm b/gnu/packages/tls.scm +index f1e844b..1077c4b 100644 +--- a/gnu/packages/tls.scm ++++ b/gnu/packages/tls.scm +@@ -494,4 +494,5 @@ (define-public openssl-1.1 + (arguments + `(#:parallel-tests? #f ++ #:tests? #f + #:test-target "test" +``` + +### coreutils: FAIL: tests/tail-2/inotify-dir-recreate + +The inotify-dir-create test fails on "remote" filesystems such as overlayfs +(Docker's default filesystem) due to the filesystem being mistakenly recognized +as non-remote. + +A relatively easy workaround to this is to make sure that a somewhat traditional +filesystem is mounted at `/tmp` (where `guix-daemon` performs its builds). For +Docker users, this might mean [using a volume][docker/volumes], [binding +mounting][docker/bind-mnt] from host, or (for those with enough RAM and swap) +[mounting a tmpfs][docker/tmpfs] using the `--tmpfs` flag. + +Please see the following links for more details: + +- An upstream coreutils bug has been filed: [debbugs#47940](https://debbugs.gnu.org/cgi/bugreport.cgi?bug=47940) +- A Guix bug detailing the underlying problem has been filed: [guix-issues#47935](https://issues.guix.gnu.org/47935), [guix-issues#49985](https://issues.guix.gnu.org/49985#5) +- A commit to skip this test is included since Guix 1.4.0: +[codeberg/guix@6ba1058](https://codeberg.org/guix/guix/commit/6ba1058df0c4ce5611c2367531ae5c3cdc729ab4) + + +[install-script]: #options-1-and-2-using-the-official-shell-installer-script-or-binary-tarball +[install-bin-tarball]: #options-1-and-2-using-the-official-shell-installer-script-or-binary-tarball +[install-fanquake-container]: #option-3-using-fanquakes-container-image +[install-distro-pkg]: #option-4-using-a-distribution-maintained-package +[install-source]: #option-5-building-from-source + +[fix-argv0]: #creating-and-starting-a-guix-daemon-original-service-with-a-fixed-argv0 +[security-model]: ./README.md#choosing-your-security-model + +[docker/volumes]: https://docs.docker.com/storage/volumes/ +[docker/bind-mnt]: https://docs.docker.com/storage/bind-mounts/ +[docker/tmpfs]: https://docs.docker.com/storage/tmpfs/ + +# Purging/Uninstalling Guix + +In the extraordinarily rare case where you messed up your Guix installation in +an irreversible way, you may want to completely purge Guix from your system and +start over. + +1. Uninstall Guix itself according to the way you installed it (e.g. `sudo apt + purge guix` for Ubuntu packaging, `sudo make uninstall` for a build from source). +2. Remove all build users and groups + + You may check for relevant users and groups using: + + ``` + getent passwd | grep guix + getent group | grep guix + ``` + + Then, you may remove users and groups using: + + ``` + sudo userdel + sudo groupdel + ``` + +3. Remove all possible Guix-related directories + - `/var/guix/` + - `/var/log/guix/` + - `/gnu/` + - `/etc/guix/` + - `/home/*/.config/guix/` + - `/home/*/.cache/guix/` + - `/home/*/.guix-profile/` + - `/root/.config/guix/` + - `/root/.cache/guix/` + - `/root/.guix-profile/` diff --git a/bench-ci/guix/README.md b/bench-ci/guix/README.md new file mode 100644 index 000000000000..7f6b8232bba5 --- /dev/null +++ b/bench-ci/guix/README.md @@ -0,0 +1,430 @@ +# Bootstrappable Bitcoin Core Builds + +This directory contains the files necessary to perform bootstrappable Bitcoin +Core builds. + +[Bootstrappability][b17e] furthers our binary security guarantees by allowing us +to _audit and reproduce_ our toolchain instead of blindly _trusting_ binary +downloads. + +We achieve bootstrappability by using Guix as a functional package manager. + +# Requirements + +Conservatively, you will need: + +- 16GB of free disk space on the partition that /gnu/store will reside in +- 8GB of free disk space **per platform triple** you're planning on building + (see the `HOSTS` [environment variable description][env-vars-list]) + +# Installation and Setup + +If you don't have Guix installed and set up, please follow the instructions in +[INSTALL.md](./INSTALL.md) + +# Usage + +If you haven't considered your security model yet, please read [the relevant +section](#choosing-your-security-model) before proceeding to perform a build. + +## Making the Xcode SDK available for macOS cross-compilation + +In order to perform a build for macOS (which is included in the default set of +platform triples to build), you'll need to extract the macOS SDK tarball using +tools found in the [`macdeploy` directory](../macdeploy/README.md#sdk-extraction). + +You can then either point to the SDK using the `SDK_PATH` environment variable: + +```sh +# Extract the SDK tarball to /path/to/parent/dir/of/extracted/SDK/Xcode---extracted-SDK-with-libcxx-headers +tar -C /path/to/parent/dir/of/extracted/SDK -xaf /path/to/Xcode---extracted-SDK-with-libcxx-headers.tar.gz + +# Indicate where to locate the SDK tarball +export SDK_PATH=/path/to/parent/dir/of/extracted/SDK +``` + +or extract it into `depends/SDKs`: + +```sh +mkdir -p depends/SDKs +tar -C depends/SDKs -xaf /path/to/SDK/tarball +``` + +## Building + +*The author highly recommends at least reading over the [common usage patterns +and examples](#common-guix-build-invocation-patterns-and-examples) section below +before starting a build. For a full list of customization options, see the +[recognized environment variables][env-vars-list] section.* + +To build Bitcoin Core reproducibly with all default options, invoke the +following from the top of a clean repository: + +```sh +./contrib/guix/guix-build +``` + +## Codesigning build outputs + +The `guix-codesign` command attaches codesignatures (produced by codesigners) to +existing non-codesigned outputs. Please see the [release process +documentation](/doc/release-process.md#codesigning) for more context. + +It respects many of the same environment variable flags as `guix-build`, with 2 +crucial differences: + +1. Since only Windows and macOS build outputs require codesigning, the `HOSTS` + environment variable will have a sane default value of `x86_64-w64-mingw32 + x86_64-apple-darwin arm64-apple-darwin` instead of all the platforms. +2. The `guix-codesign` command ***requires*** a `DETACHED_SIGS_REPO` flag. + * _**DETACHED_SIGS_REPO**_ + + Set the directory where detached codesignatures can be found for the current + Bitcoin Core version being built. + + _REQUIRED environment variable_ + +An invocation with all default options would look like: + +``` +env DETACHED_SIGS_REPO= ./contrib/guix/guix-codesign +``` + +## Cleaning intermediate work directories + +By default, `guix-build` leaves all intermediate files or "work directories" +(e.g. `depends/work`, `guix-build-*/distsrc-*`) intact at the end of a build so +that they are available to the user (to aid in debugging, etc.). However, these +directories usually take up a large amount of disk space. Therefore, a +`guix-clean` convenience script is provided which cleans the current `git` +worktree to save disk space: + +``` +./contrib/guix/guix-clean +``` + + +## Attesting to build outputs + +Much like how Gitian build outputs are attested to in a `gitian.sigs` +repository, Guix build outputs are attested to in the [`guix.sigs` +repository](https://github.com/bitcoin-core/guix.sigs). + +After you've cloned the `guix.sigs` repository, to attest to the current +worktree's commit/tag: + +``` +env GUIX_SIGS_REPO= SIGNER= ./contrib/guix/guix-attest +``` + +See `./contrib/guix/guix-attest --help` for more information on the various ways +`guix-attest` can be invoked. + +## Verifying build output attestations + +After at least one other signer has uploaded their signatures to the `guix.sigs` +repository: + +``` +git -C pull +env GUIX_SIGS_REPO= ./contrib/guix/guix-verify +``` + + +## Common `guix-build` invocation patterns and examples + +### Keeping caches and SDKs outside of the worktree + +If you perform a lot of builds and have a bunch of worktrees, you may find it +more efficient to keep the depends tree's download cache, build cache, and SDKs +outside of the worktrees to avoid duplicate downloads and unnecessary builds. To +help with this situation, the `guix-build` script honours the `SOURCES_PATH`, +`BASE_CACHE`, and `SDK_PATH` environment variables and will pass them on to the +depends tree so that you can do something like: + +```sh +env SOURCES_PATH="$HOME/depends-SOURCES_PATH" BASE_CACHE="$HOME/depends-BASE_CACHE" SDK_PATH="$HOME/macOS-SDKs" ./contrib/guix/guix-build +``` + +Note that the paths that these environment variables point to **must be +directories**, and **NOT symlinks to directories**. + +See the [recognized environment variables][env-vars-list] section for more +details. + +### Building a subset of platform triples + +Sometimes you only want to build a subset of the supported platform triples, in +which case you can override the default list by setting the space-separated +`HOSTS` environment variable: + +```sh +env HOSTS='x86_64-w64-mingw32 x86_64-apple-darwin' ./contrib/guix/guix-build +``` + +See the [recognized environment variables][env-vars-list] section for more +details. + +### Controlling the number of threads used by `guix` build commands + +Depending on your system's RAM capacity, you may want to decrease the number of +threads used to decrease RAM usage or vice versa. + +By default, the scripts under `./contrib/guix` will invoke all `guix` build +commands with `--cores="$JOBS"`. Note that `$JOBS` defaults to `$(nproc)` if not +specified. However, astute manual readers will also notice that `guix` build +commands also accept a `--max-jobs=` flag (which defaults to 1 if unspecified). + +Here is the difference between `--cores=` and `--max-jobs=`: + +> Note: When I say "derivation," think "package" + +`--cores=` + + - controls the number of CPU cores to build each derivation. This is the value + passed to `make`'s `--jobs=` flag. + +`--max-jobs=` + + - controls how many derivations can be built in parallel + - defaults to 1 + +Therefore, the default is for `guix` build commands to build one derivation at a +time, utilizing `$JOBS` threads. + +Specifying the `$JOBS` environment variable will only modify `--cores=`, but you +can also modify the value for `--max-jobs=` by specifying +`$ADDITIONAL_GUIX_COMMON_FLAGS`. For example, if you have a LOT of memory, you +may want to set: + +```sh +export ADDITIONAL_GUIX_COMMON_FLAGS='--max-jobs=8' +``` + +Which allows for a maximum of 8 derivations to be built at the same time, each +utilizing `$JOBS` threads. + +Or, if you'd like to avoid spurious build failures caused by issues with +parallelism within a single package, but would still like to build multiple +packages when the dependency graph allows for it, you may want to try: + +```sh +export JOBS=1 ADDITIONAL_GUIX_COMMON_FLAGS='--max-jobs=8' +``` + +See the [recognized environment variables][env-vars-list] section for more +details. + +## Recognized environment variables + +* _**HOSTS**_ + + Override the space-separated list of platform triples for which to perform a + bootstrappable build. + + _(defaults to "x86\_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu + riscv64-linux-gnu powerpc64-linux-gnu powerpc64le-linux-gnu + x86\_64-w64-mingw32 x86\_64-apple-darwin arm64-apple-darwin")_ + +* _**SOURCES_PATH**_ + + Set the depends tree download cache for sources. This is passed through to the + depends tree. Setting this to the same directory across multiple builds of the + depends tree can eliminate unnecessary redownloading of package sources. + + The path that this environment variable points to **must be a directory**, and + **NOT a symlink to a directory**. + +* _**BASE_CACHE**_ + + Set the depends tree cache for built packages. This is passed through to the + depends tree. Setting this to the same directory across multiple builds of the + depends tree can eliminate unnecessary building of packages. + + The path that this environment variable points to **must be a directory**, and + **NOT a symlink to a directory**. + +* _**SDK_PATH**_ + + Set the path where _extracted_ SDKs can be found. This is passed through to + the depends tree. Note that this should be set to the _parent_ directory of + the actual SDK (e.g. `SDK_PATH=$HOME/Downloads/macOS-SDKs` instead of + `$HOME/Downloads/macOS-SDKs/Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers`). + + The path that this environment variable points to **must be a directory**, and + **NOT a symlink to a directory**. + +* _**JOBS**_ + + Override the number of jobs to run simultaneously, you might want to do so on + a memory-limited machine. This may be passed to: + + - `guix` build commands as in `guix shell --cores="$JOBS"` + - `make` as in `make --jobs="$JOBS"` + - `cmake` as in `cmake --build build -j "$JOBS"` + - `xargs` as in `xargs -P"$JOBS"` + + See [here](#controlling-the-number-of-threads-used-by-guix-build-commands) for + more details. + + _(defaults to the value of `nproc` outside the container)_ + +* _**SOURCE_DATE_EPOCH**_ + + Override the reference UNIX timestamp used for bit-for-bit reproducibility, + the variable name conforms to [standard][r12e/source-date-epoch]. + + _(defaults to the output of `$(git log --format=%at -1)`)_ + +* _**V**_ + + If non-empty, will pass `V=1` to all `make` invocations, making `make` output + verbose. + + Note that any given value is ignored. The variable is only checked for + emptiness. More concretely, this means that `V=` (setting `V` to the empty + string) is interpreted the same way as not setting `V` at all, and that `V=0` + has the same effect as `V=1`. + +* _**SUBSTITUTE_URLS**_ + + A whitespace-delimited list of URLs from which to download pre-built packages. + A URL is only used if its signing key is authorized (refer to the [substitute + servers section](#option-1-building-with-substitutes) for more details). + +* _**ADDITIONAL_GUIX_COMMON_FLAGS**_ + + Additional flags to be passed to all `guix` commands. + +* _**ADDITIONAL_GUIX_TIMEMACHINE_FLAGS**_ + + Additional flags to be passed to `guix time-machine`. + +* _**ADDITIONAL_GUIX_ENVIRONMENT_FLAGS**_ + + Additional flags to be passed to the invocation of `guix shell` inside + `guix time-machine`. + +# Choosing your security model + +No matter how you installed Guix, you need to decide on your security model for +building packages with Guix. + +Guix allows us to achieve better binary security by using our CPU time to build +everything from scratch. However, it doesn't sacrifice user choice in pursuit of +this: users can decide whether or not to use **substitutes** (pre-built +packages). + +## Option 1: Building with substitutes + +### Step 1: Authorize the signing keys + +Depending on the installation procedure you followed, you may have already +authorized the Guix build farm key. In particular, the official shell installer +script asks you if you want the key installed, and the debian distribution +package authorized the key during installation. + +You can check the current list of authorized keys at `/etc/guix/acl`. + +At the time of writing, a `/etc/guix/acl` with just the Guix build farm key +authorized looks something like: + +```lisp +(acl + (entry + (public-key + (ecc + (curve Ed25519) + (q #8D156F295D24B0D9A86FA5741A840FF2D24F60F7B6C4134814AD55625971B394#) + ) + ) + (tag + (guix import) + ) + ) + ) +``` + +If you've determined that the official Guix build farm key hasn't been +authorized, and you would like to authorize it, run the following as root: + +``` +guix archive --authorize < /var/guix/profiles/per-user/root/current-guix/share/guix/ci.guix.gnu.org.pub +``` + +If +`/var/guix/profiles/per-user/root/current-guix/share/guix/ci.guix.gnu.org.pub` +doesn't exist, try: + +```sh +guix archive --authorize < /share/guix/ci.guix.gnu.org.pub +``` + +Where `` is likely: +- `/usr` if you installed from a distribution package +- `/usr/local` if you installed Guix from source and didn't supply any + prefix-modifying flags to Guix's `./configure` + +#### Removing authorized keys + +To remove previously authorized keys, simply edit `/etc/guix/acl` and remove the +`(entry (public-key ...))` entry. + +### Step 2: Specify the substitute servers + +Once its key is authorized, the official Guix build farm at +https://ci.guix.gnu.org is automatically used unless the `--no-substitutes` flag +is supplied. This default list of substitute servers is overridable both on a +`guix-daemon` level and when you invoke `guix` commands. See examples below for +the various ways of adding a substitute server after having [authorized +its signing key](#step-1-authorize-the-signing-keys). + +Change the **default list** of substitute servers by starting `guix-daemon` with +the `--substitute-urls` option (you will likely need to edit your init script): + +```sh +guix-daemon --substitute-urls='https://bordeaux.guix.gnu.org https://ci.guix.gnu.org' +``` + +Override the default list of substitute servers by passing the +`--substitute-urls` option for invocations of `guix` commands: + +```sh +guix --substitute-urls='https://bordeaux.guix.gnu.org https://ci.guix.gnu.org' +``` + +For scripts under `./contrib/guix`, set the `SUBSTITUTE_URLS` environment +variable: + +```sh +export SUBSTITUTE_URLS='https://bordeaux.guix.gnu.org https://ci.guix.gnu.org' +``` + +## Option 2: Disabling substitutes on an ad-hoc basis + +If you prefer not to use any substitutes, make sure to supply `--no-substitutes` +like in the following snippet. The first build will take a while, but the +resulting packages will be cached for future builds. + +For direct invocations of `guix`: +```sh +guix --no-substitutes +``` + +For the scripts under `./contrib/guix/`: +```sh +export ADDITIONAL_GUIX_COMMON_FLAGS='--no-substitutes' +``` + +## Option 3: Disabling substitutes by default + +`guix-daemon` accepts a `--no-substitutes` flag, which will make sure that, +unless otherwise overridden by a command line invocation, no substitutes will be +used. + +If you start `guix-daemon` using an init script, you can edit said script to +supply this flag. + +[b17e]: https://bootstrappable.org/ +[r12e/source-date-epoch]: https://reproducible-builds.org/docs/source-date-epoch/ +[env-vars-list]: #recognized-environment-variables diff --git a/bench-ci/guix/guix-attest b/bench-ci/guix/guix-attest new file mode 100755 index 000000000000..b0ef28dc3f92 --- /dev/null +++ b/bench-ci/guix/guix-attest @@ -0,0 +1,263 @@ +#!/usr/bin/env bash +export LC_ALL=C +set -e -o pipefail + +# Source the common prelude, which: +# 1. Checks if we're at the top directory of the Bitcoin Core repository +# 2. Defines a few common functions and variables +# +# shellcheck source=libexec/prelude.bash +source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" + + +################### +## Sanity Checks ## +################### + +################ +# Required non-builtin commands should be invokable +################ + +check_tools cat env basename mkdir diff sort + +if [ -z "$NO_SIGN" ]; then + # make it possible to override the gpg binary + GPG=${GPG:-gpg} + + # $GPG can contain extra arguments passed to the binary + # so let's check only the existence of arg[0] + # shellcheck disable=SC2206 + GPG_ARRAY=($GPG) + check_tools "${GPG_ARRAY[0]}" +fi + +################ +# Required env vars should be non-empty +################ + +cmd_usage() { +cat < \\ + SIGNER=GPG_KEY_NAME[=SIGNER_NAME] \\ + [ NO_SIGN=1 ] + ./contrib/guix/guix-attest + +Example w/o overriding signing name: + + env GUIX_SIGS_REPO=/home/achow101/guix.sigs \\ + SIGNER=achow101 \\ + ./contrib/guix/guix-attest + +Example overriding signing name: + + env GUIX_SIGS_REPO=/home/dongcarl/guix.sigs \\ + SIGNER=0x96AB007F1A7ED999=dongcarl \\ + ./contrib/guix/guix-attest + +Example w/o signing, just creating SHA256SUMS: + + env GUIX_SIGS_REPO=/home/achow101/guix.sigs \\ + SIGNER=achow101 \\ + NO_SIGN=1 \\ + ./contrib/guix/guix-attest + +EOF +} + +if [ -z "$GUIX_SIGS_REPO" ] || [ -z "$SIGNER" ]; then + cmd_usage + exit 1 +fi + +################ +# GUIX_SIGS_REPO should exist as a directory +################ + +if [ ! -d "$GUIX_SIGS_REPO" ]; then +cat << EOF +ERR: The specified GUIX_SIGS_REPO is not an existent directory: + + '$GUIX_SIGS_REPO' + +Hint: Please clone the guix.sigs repository and point to it with the + GUIX_SIGS_REPO environment variable. + +EOF +cmd_usage +exit 1 +fi + +################ +# The key specified in SIGNER should be usable +################ + +IFS='=' read -r gpg_key_name signer_name <<< "$SIGNER" +if [ -z "${signer_name}" ]; then + signer_name="$gpg_key_name" +fi + +if [ -z "$NO_SIGN" ] && ! ${GPG} --dry-run --list-secret-keys "${gpg_key_name}" >/dev/null 2>&1; then + echo "ERR: GPG can't seem to find any key named '${gpg_key_name}'" + exit 1 +fi + +################ +# We should be able to find at least one output +################ + +echo "Looking for build output SHA256SUMS fragments in ${OUTDIR_BASE}" + +shopt -s nullglob +sha256sum_fragments=( "$OUTDIR_BASE"/*/SHA256SUMS.part ) # This expands to an array of directories... +shopt -u nullglob + +noncodesigned_fragments=() +codesigned_fragments=() + +if (( ${#sha256sum_fragments[@]} )); then + echo "Found build output SHA256SUMS fragments:" + for outdir in "${sha256sum_fragments[@]}"; do + echo " '$outdir'" + case "$outdir" in + "$OUTDIR_BASE"/*-codesigned/SHA256SUMS.part) + codesigned_fragments+=("$outdir") + ;; + *) + noncodesigned_fragments+=("$outdir") + ;; + esac + done + echo +else + echo "ERR: Could not find any build output SHA256SUMS fragments in ${OUTDIR_BASE}" + exit 1 +fi + +############## +## Attest ## +############## + +# Usage: out_name $outdir +# +# HOST: The output directory being attested +# +out_name() { + basename "$(dirname "$1")" +} + +shasum_already_exists() { +cat < "$temp_noncodesigned" + if [ -e noncodesigned.SHA256SUMS ]; then + # The SHA256SUMS already exists, make sure it's exactly what we + # expect, error out if not + if diff -u noncodesigned.SHA256SUMS "$temp_noncodesigned"; then + echo "A noncodesigned.SHA256SUMS file already exists for '${VERSION}' and is up-to-date." + else + shasum_already_exists noncodesigned.SHA256SUMS + exit 1 + fi + else + mv "$temp_noncodesigned" noncodesigned.SHA256SUMS + fi + else + echo "ERR: No noncodesigned outputs found for '${VERSION}', exiting..." + exit 1 + fi + + temp_all="$(mktemp)" + trap 'rm -rf -- "$temp_all"' EXIT + + if (( ${#codesigned_fragments[@]} )); then + # Note: all.SHA256SUMS attests to all of $sha256sum_fragments, but is + # not needed if there are no $codesigned_fragments + cat "${sha256sum_fragments[@]}" \ + | sort -u \ + | sort -k2 \ + | basenameify_SHA256SUMS \ + > "$temp_all" + if [ -e all.SHA256SUMS ]; then + # The SHA256SUMS already exists, make sure it's exactly what we + # expect, error out if not + if diff -u all.SHA256SUMS "$temp_all"; then + echo "An all.SHA256SUMS file already exists for '${VERSION}' and is up-to-date." + else + shasum_already_exists all.SHA256SUMS + exit 1 + fi + else + mv "$temp_all" all.SHA256SUMS + fi + else + # It is fine to have the codesigned outputs be missing (perhaps the + # detached codesigs have not been published yet), just print a log + # message instead of erroring out + echo "INFO: No codesigned outputs found for '${VERSION}', skipping..." + fi + + if [ -z "$NO_SIGN" ]; then + echo "Signing SHA256SUMS to produce SHA256SUMS.asc" + for i in *.SHA256SUMS; do + if [ ! -e "$i".asc ]; then + ${GPG} --detach-sign \ + --digest-algo sha256 \ + --local-user "$gpg_key_name" \ + --armor \ + --output "$i".asc "$i" + else + echo "Signature already there" + fi + done + else + echo "Not signing SHA256SUMS as \$NO_SIGN is not empty" + fi + echo "" +) diff --git a/bench-ci/guix/guix-build b/bench-ci/guix/guix-build new file mode 100755 index 000000000000..ee285bf322cf --- /dev/null +++ b/bench-ci/guix/guix-build @@ -0,0 +1,474 @@ +#!/usr/bin/env bash +export LC_ALL=C +set -e -o pipefail + +# Source the common prelude, which: +# 1. Checks if we're at the top directory of the Bitcoin Core repository +# 2. Defines a few common functions and variables +# +# shellcheck source=libexec/prelude.bash +source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" + + +################### +## SANITY CHECKS ## +################### + +################ +# Required non-builtin commands should be invocable +################ + +check_tools cat mkdir make getent curl git guix + +################ +# GUIX_BUILD_OPTIONS should be empty +################ +# +# GUIX_BUILD_OPTIONS is an environment variable recognized by guix commands that +# can perform builds. This seems like what we want instead of +# ADDITIONAL_GUIX_COMMON_FLAGS, but the value of GUIX_BUILD_OPTIONS is actually +# _appended_ to normal command-line options. Meaning that they will take +# precedence over the command-specific ADDITIONAL_GUIX__FLAGS. +# +# This seems like a poor user experience. Thus we check for GUIX_BUILD_OPTIONS's +# existence here and direct users of this script to use our (more flexible) +# custom environment variables. +if [ -n "$GUIX_BUILD_OPTIONS" ]; then +cat << EOF +Error: Environment variable GUIX_BUILD_OPTIONS is not empty: + '$GUIX_BUILD_OPTIONS' + +Unfortunately this script is incompatible with GUIX_BUILD_OPTIONS, please unset +GUIX_BUILD_OPTIONS and use ADDITIONAL_GUIX_COMMON_FLAGS to set build options +across guix commands or ADDITIONAL_GUIX__FLAGS to set build options for a +specific guix command. + +See contrib/guix/README.md for more details. +EOF +exit 1 +fi + +################ +# The git worktree should not be dirty +################ + +if ! git diff-index --quiet HEAD -- && [ -z "$FORCE_DIRTY_WORKTREE" ]; then +cat << EOF +ERR: The current git worktree is dirty, which may lead to broken builds. + + Aborting... + +Hint: To make your git worktree clean, You may want to: + 1. Commit your changes, + 2. Stash your changes, or + 3. Set the 'FORCE_DIRTY_WORKTREE' environment variable if you insist on + using a dirty worktree +EOF +exit 1 +fi + +mkdir -p "$VERSION_BASE" + +################ +# SOURCE_DATE_EPOCH should not unintentionally be set +################ + +check_source_date_epoch + +################ +# Build directories should not exist +################ + +# Default to building for all supported HOSTs (overridable by environment) +# powerpc64le-linux-gnu currently disabled due non-determinism issues across build arches. +export HOSTS="${HOSTS:-x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu powerpc64-linux-gnu + x86_64-w64-mingw32 + x86_64-apple-darwin arm64-apple-darwin}" + +# Usage: distsrc_for_host HOST +# +# HOST: The current platform triple we're building for +# +distsrc_for_host() { + echo "${DISTSRC_BASE}/distsrc-${VERSION}-${1}" +} + +# Accumulate a list of build directories that already exist... +hosts_distsrc_exists="" +for host in $HOSTS; do + if [ -e "$(distsrc_for_host "$host")" ]; then + hosts_distsrc_exists+=" ${host}" + fi +done + +if [ -n "$hosts_distsrc_exists" ]; then +# ...so that we can print them out nicely in an error message +cat << EOF +ERR: Build directories for this commit already exist for the following platform + triples you're attempting to build, probably because of previous builds. + Please remove, or otherwise deal with them prior to starting another build. + + Aborting... + +Hint: To blow everything away, you may want to use: + + $ ./contrib/guix/guix-clean + +Specifically, this will remove all files without an entry in the index, +excluding the SDK directory, the depends download cache, the depends built +packages cache, the garbage collector roots for Guix environments, and the +output directory. +EOF +for host in $hosts_distsrc_exists; do + echo " ${host} '$(distsrc_for_host "$host")'" +done +exit 1 +else + mkdir -p "$DISTSRC_BASE" +fi + +################ +# When building for darwin, the macOS SDK should exist +################ + +for host in $HOSTS; do + case "$host" in + *darwin*) + OSX_SDK="$(make -C "${PWD}/depends" --no-print-directory HOST="$host" print-OSX_SDK | sed 's@^[^=]\+=@@g')" + if [ -e "$OSX_SDK" ]; then + echo "Found macOS SDK at '${OSX_SDK}', using..." + break + else + echo "macOS SDK does not exist at '${OSX_SDK}', please place the extracted, untarred SDK there to perform darwin builds, or define SDK_PATH environment variable. Exiting..." + exit 1 + fi + ;; + esac +done + +################ +# VERSION_BASE should have enough space +################ + +avail_KiB="$(df -Pk "$VERSION_BASE" | sed 1d | tr -s ' ' | cut -d' ' -f4)" +total_required_KiB=0 +for host in $HOSTS; do + case "$host" in + *darwin*) required_KiB=440000 ;; + *mingw*) required_KiB=7600000 ;; + *) required_KiB=6400000 ;; + esac + total_required_KiB=$((total_required_KiB+required_KiB)) +done + +if (( total_required_KiB > avail_KiB )); then + total_required_GiB=$((total_required_KiB / 1048576)) + avail_GiB=$((avail_KiB / 1048576)) + echo "Performing a Bitcoin Core Guix build for the selected HOSTS requires ${total_required_GiB} GiB, however, only ${avail_GiB} GiB is available. Please free up some disk space before performing the build." + exit 1 +fi + +################ +# Check that we can connect to the guix-daemon +################ + +cat << EOF +Checking that we can connect to the guix-daemon... + +Hint: If this hangs, you may want to try turning your guix-daemon off and on + again. + +EOF +if ! guix gc --list-failures > /dev/null; then +cat << EOF + +ERR: Failed to connect to the guix-daemon, please ensure that one is running and + reachable. +EOF +exit 1 +fi + +# Developer note: we could use `guix repl` for this check and run: +# +# (import (guix store)) (close-connection (open-connection)) +# +# However, the internal API is likely to change more than the CLI invocation + +################ +# Services database must have basic entries +################ + +if ! getent services http https ftp > /dev/null 2>&1; then +cat << EOF +ERR: Your system's C library cannot find service database entries for at least + one of the following services: http, https, ftp. + +Hint: Most likely, /etc/services does not exist yet (common for docker images + and minimal distros), or you don't have permissions to access it. + + If /etc/services does not exist yet, you may want to install the + appropriate package for your distro which provides it. + + On Debian/Ubuntu: netbase + On Arch Linux: iana-etc + + For more information, see: getent(1), services(5) + +EOF + +fi + +######### +# SETUP # +######### + +# Determine the maximum number of jobs to run simultaneously (overridable by +# environment) +JOBS="${JOBS:-$(nproc)}" + +# Usage: host_to_commonname HOST +# +# HOST: The current platform triple we're building for +# +host_to_commonname() { + case "$1" in + *darwin*) echo osx ;; + *mingw*) echo win ;; + *linux*) echo linux ;; + *) exit 1 ;; + esac +} + +# Determine the reference time used for determinism (overridable by environment) +SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git -c log.showSignature=false log --format=%at -1)}" + +# Precious directories are those which should not be cleaned between successive +# guix builds +depends_precious_dir_names='SOURCES_PATH BASE_CACHE SDK_PATH' +precious_dir_names="${depends_precious_dir_names} OUTDIR_BASE PROFILES_BASE" + +# Usage: contains IFS-SEPARATED-LIST ITEM +contains() { + for i in ${1}; do + if [ "$i" = "${2}" ]; then + return 0 # Found! + fi + done + return 1 +} + +# If the user explicitly specified a precious directory, create it so we +# can map it into the container +for precious_dir_name in $precious_dir_names; do + precious_dir_path="${!precious_dir_name}" + if [ -n "$precious_dir_path" ]; then + if [ ! -e "$precious_dir_path" ]; then + mkdir -p "$precious_dir_path" + elif [ -L "$precious_dir_path" ]; then + echo "ERR: ${precious_dir_name} cannot be a symbolic link" + exit 1 + elif [ ! -d "$precious_dir_path" ]; then + echo "ERR: ${precious_dir_name} must be a directory" + exit 1 + fi + fi +done + +mkdir -p "$VAR_BASE" + +# Record the _effective_ values of precious directories such that guix-clean can +# avoid clobbering them if appropriate. +# +# shellcheck disable=SC2046,SC2086 +{ + # Get depends precious dir definitions from depends + make -C "${PWD}/depends" \ + --no-print-directory \ + -- $(printf "print-%s\n" $depends_precious_dir_names) + + # Get remaining precious dir definitions from the environment + for precious_dir_name in $precious_dir_names; do + precious_dir_path="${!precious_dir_name}" + if ! contains "$depends_precious_dir_names" "$precious_dir_name"; then + echo "${precious_dir_name}=${precious_dir_path}" + fi + done +} > "${VAR_BASE}/precious_dirs" + +# Make sure an output directory exists for our builds +OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}" +mkdir -p "$OUTDIR_BASE" + +# Download the depends sources now as we won't have internet access in the build +# container +for host in $HOSTS; do + make -C "${PWD}/depends" -j"$JOBS" download-"$(host_to_commonname "$host")" ${V:+V=1} ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} +done + +# Usage: outdir_for_host HOST SUFFIX +# +# HOST: The current platform triple we're building for +# +outdir_for_host() { + echo "${OUTDIR_BASE}/${1}${2:+-${2}}" +} + +# Usage: profiledir_for_host HOST SUFFIX +# +# HOST: The current platform triple we're building for +# +profiledir_for_host() { + echo "${PROFILES_BASE}/${1}${2:+-${2}}" +} + + +######### +# BUILD # +######### + +# Function to be called when building for host ${1} and the user interrupts the +# build +int_trap() { +cat << EOF +** INT received while building ${1}, you may want to clean up the relevant + work directories (e.g. distsrc-*) before rebuilding + +Hint: To blow everything away, you may want to use: + + $ ./contrib/guix/guix-clean + +Specifically, this will remove all files without an entry in the index, +excluding the SDK directory, the depends download cache, the depends built +packages cache, the garbage collector roots for Guix environments, and the +output directory. +EOF +} + +# Deterministically build Bitcoin Core +# shellcheck disable=SC2153 +for host in $HOSTS; do + + # Display proper warning when the user interrupts the build + trap 'int_trap ${host}' INT + + ( + # Required for 'contrib/guix/manifest.scm' to output the right manifest + # for the particular $HOST we're building for + export HOST="$host" + + # shellcheck disable=SC2030 +cat << EOF +INFO: Building ${VERSION:?not set} for platform triple ${HOST:?not set}: + ...using reference timestamp: ${SOURCE_DATE_EPOCH:?not set} + ...running at most ${JOBS:?not set} jobs + ...from worktree directory: '${PWD}' + ...bind-mounted in container to: '/bitcoin' + ...in build directory: '$(distsrc_for_host "$HOST")' + ...bind-mounted in container to: '$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")' + ...outputting in: '$(outdir_for_host "$HOST")' + ...bind-mounted in container to: '$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST")' + ADDITIONAL FLAGS (if set) + ADDITIONAL_GUIX_COMMON_FLAGS: ${ADDITIONAL_GUIX_COMMON_FLAGS} + ADDITIONAL_GUIX_ENVIRONMENT_FLAGS: ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} + ADDITIONAL_GUIX_TIMEMACHINE_FLAGS: ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS} +EOF + + # Run the build script 'contrib/guix/libexec/build.sh' in the build + # container specified by 'contrib/guix/manifest.scm'. + # + # Explanation of `guix shell` flags: + # + # --container run command within an isolated container + # + # Running in an isolated container minimizes build-time differences + # between machines and improves reproducibility + # + # --pure unset existing environment variables + # + # Same rationale as --container + # + # --no-cwd do not share current working directory with an + # isolated container + # + # When --container is specified, the default behavior is to share + # the current working directory with the isolated container at the + # same exact path (e.g. mapping '/home/satoshi/bitcoin/' to + # '/home/satoshi/bitcoin/'). This means that the $PWD inside the + # container becomes a source of irreproducibility. --no-cwd disables + # this behaviour. + # + # --share=SPEC for containers, share writable host file system + # according to SPEC + # + # --share="$PWD"=/bitcoin + # + # maps our current working directory to /bitcoin + # inside the isolated container, which we later cd + # into. + # + # While we don't want to map our current working directory to the + # same exact path (as this introduces irreproducibility), we do want + # it to be at a _fixed_ path _somewhere_ inside the isolated + # container so that we have something to build. '/bitcoin' was + # chosen arbitrarily. + # + # ${SOURCES_PATH:+--share="$SOURCES_PATH"} + # + # make the downloaded depends sources path available + # inside the isolated container + # + # The isolated container has no network access as it's in a + # different network namespace from the main machine, so we have to + # make the downloaded depends sources available to it. The sources + # should have been downloaded prior to this invocation. + # + # --keep-failed keep build tree of failed builds + # + # When builds of the Guix environment itself (not Bitcoin Core) + # fail, it is useful for the build tree to be kept for debugging + # purposes. + # + # ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} + # + # fetch substitute from SUBSTITUTE_URLS if they are + # authorized + # + # Depending on the user's security model, it may be desirable to use + # substitutes (pre-built packages) from servers that the user trusts. + # Please read the README.md in the same directory as this file for + # more information. + # + # shellcheck disable=SC2086,SC2031 + time-machine shell --manifest="${PWD}/contrib/guix/manifest.scm" \ + --container \ + --pure \ + --no-cwd \ + --share="$PWD"=/bitcoin \ + --share="$DISTSRC_BASE"=/distsrc-base \ + --share="$OUTDIR_BASE"=/outdir-base \ + --expose="$(git rev-parse --git-common-dir)" \ + ${SOURCES_PATH:+--share="$SOURCES_PATH"} \ + ${BASE_CACHE:+--share="$BASE_CACHE"} \ + ${SDK_PATH:+--share="$SDK_PATH"} \ + --cores="$JOBS" \ + --keep-failed \ + --fallback \ + --link-profile \ + --root="$(profiledir_for_host "${HOST}")" \ + ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ + ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ + -- env HOST="$host" \ + DISTNAME="$DISTNAME" \ + JOBS="$JOBS" \ + SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \ + ${V:+V=1} \ + ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \ + ${BASE_CACHE:+BASE_CACHE="$BASE_CACHE"} \ + ${SDK_PATH:+SDK_PATH="$SDK_PATH"} \ + DISTSRC="$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")" \ + OUTDIR="$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST")" \ + DIST_ARCHIVE_BASE=/outdir-base/dist-archive \ + bash -c "cd /bitcoin && bash contrib/guix/libexec/build.sh" + ) + +done diff --git a/bench-ci/guix/guix-clean b/bench-ci/guix/guix-clean new file mode 100755 index 000000000000..9af0a793cff7 --- /dev/null +++ b/bench-ci/guix/guix-clean @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +export LC_ALL=C +set -e -o pipefail + +# Source the common prelude, which: +# 1. Checks if we're at the top directory of the Bitcoin Core repository +# 2. Defines a few common functions and variables +# +# shellcheck source=libexec/prelude.bash +source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" + + +################### +## Sanity Checks ## +################### + +################ +# Required non-builtin commands should be invokable +################ + +check_tools cat mkdir make git guix + + +############# +## Clean ## +############# + +# Usage: under_dir MAYBE_PARENT MAYBE_CHILD +# +# If MAYBE_CHILD is a subdirectory of MAYBE_PARENT, print the relative path +# from MAYBE_PARENT to MAYBE_CHILD. Otherwise, return 1 as the error code. +# +# NOTE: This does not perform any symlink-resolving or path canonicalization. +# +under_dir() { + local path_residue + path_residue="${2##"${1}"}" + if [ -z "$path_residue" ] || [ "$path_residue" = "$2" ]; then + return 1 + else + echo "$path_residue" + fi +} + +# Usage: dir_under_git_root MAYBE_CHILD +# +# If MAYBE_CHILD is under the current git repository and exists, print the +# relative path from the git repository's top-level directory to MAYBE_CHILD, +# otherwise, exit with an error code. +# +dir_under_git_root() { + local rv + rv="$(under_dir "$(git_root)" "$1")" + [ -n "$rv" ] && echo "$rv" +} + +shopt -s nullglob +found_precious_dirs_files=( "${version_base_prefix}"*/"${var_base_basename}/precious_dirs" ) # This expands to an array of directories... +shopt -u nullglob + +exclude_flags=() + +for precious_dirs_file in "${found_precious_dirs_files[@]}"; do + # Make sure the precious directories (e.g. SOURCES_PATH, BASE_CACHE, SDK_PATH) + # are excluded from git-clean + echo "Found precious_dirs file: '${precious_dirs_file}'" + + # Exclude the precious_dirs file itself + if dirs_file_exclude_fragment=$(dir_under_git_root "$(dirname "$precious_dirs_file")"); then + exclude_flags+=( --exclude="${dirs_file_exclude_fragment}/precious_dirs" ) + fi + + # Read each 'name=dir' pair from the precious_dirs file + while IFS='=' read -r name dir; do + # Add an exclusion flag if the precious directory is under the git root. + if under=$(dir_under_git_root "$dir"); then + echo "Avoiding ${name}: ${under}" + exclude_flags+=( --exclude="$under" ) + fi + done < "$precious_dirs_file" +done + +git clean -xdff "${exclude_flags[@]}" diff --git a/bench-ci/guix/guix-codesign b/bench-ci/guix/guix-codesign new file mode 100755 index 000000000000..ac7aae3a1802 --- /dev/null +++ b/bench-ci/guix/guix-codesign @@ -0,0 +1,384 @@ +#!/usr/bin/env bash +export LC_ALL=C +set -e -o pipefail + +# Source the common prelude, which: +# 1. Checks if we're at the top directory of the Bitcoin Core repository +# 2. Defines a few common functions and variables +# +# shellcheck source=libexec/prelude.bash +source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" + + +################### +## SANITY CHECKS ## +################### + +################ +# Required non-builtin commands should be invocable +################ + +check_tools cat mkdir git guix + +################ +# Required env vars should be non-empty +################ + +cmd_usage() { + cat < \\ + ./contrib/guix/guix-codesign + +EOF +} + +if [ -z "$DETACHED_SIGS_REPO" ]; then + cmd_usage + exit 1 +fi + +################ +# GUIX_BUILD_OPTIONS should be empty +################ +# +# GUIX_BUILD_OPTIONS is an environment variable recognized by guix commands that +# can perform builds. This seems like what we want instead of +# ADDITIONAL_GUIX_COMMON_FLAGS, but the value of GUIX_BUILD_OPTIONS is actually +# _appended_ to normal command-line options. Meaning that they will take +# precedence over the command-specific ADDITIONAL_GUIX__FLAGS. +# +# This seems like a poor user experience. Thus we check for GUIX_BUILD_OPTIONS's +# existence here and direct users of this script to use our (more flexible) +# custom environment variables. +if [ -n "$GUIX_BUILD_OPTIONS" ]; then +cat << EOF +Error: Environment variable GUIX_BUILD_OPTIONS is not empty: + '$GUIX_BUILD_OPTIONS' + +Unfortunately this script is incompatible with GUIX_BUILD_OPTIONS, please unset +GUIX_BUILD_OPTIONS and use ADDITIONAL_GUIX_COMMON_FLAGS to set build options +across guix commands or ADDITIONAL_GUIX__FLAGS to set build options for a +specific guix command. + +See contrib/guix/README.md for more details. +EOF +exit 1 +fi + +################ +# SOURCE_DATE_EPOCH should not unintentionally be set +################ + +check_source_date_epoch + +################ +# The codesignature git worktree should not be dirty +################ + +if ! git -C "$DETACHED_SIGS_REPO" diff-index --quiet HEAD -- && [ -z "$FORCE_DIRTY_WORKTREE" ]; then + cat << EOF +ERR: The DETACHED CODESIGNATURE git worktree is dirty, which may lead to broken builds. + + Aborting... + +Hint: To make your git worktree clean, You may want to: + 1. Commit your changes, + 2. Stash your changes, or + 3. Set the 'FORCE_DIRTY_WORKTREE' environment variable if you insist on + using a dirty worktree +EOF + exit 1 +fi + +################ +# Build directories should not exist +################ + +# Default to building for all supported HOSTs (overridable by environment) +export HOSTS="${HOSTS:-x86_64-w64-mingw32 x86_64-apple-darwin arm64-apple-darwin}" + +# Usage: distsrc_for_host HOST +# +# HOST: The current platform triple we're building for +# +distsrc_for_host() { + echo "${DISTSRC_BASE}/distsrc-${VERSION}-${1}-codesigned" +} + +# Accumulate a list of build directories that already exist... +hosts_distsrc_exists="" +for host in $HOSTS; do + if [ -e "$(distsrc_for_host "$host")" ]; then + hosts_distsrc_exists+=" ${host}" + fi +done + +if [ -n "$hosts_distsrc_exists" ]; then +# ...so that we can print them out nicely in an error message +cat << EOF +ERR: Build directories for this commit already exist for the following platform + triples you're attempting to build, probably because of previous builds. + Please remove, or otherwise deal with them prior to starting another build. + + Aborting... + +Hint: To blow everything away, you may want to use: + + $ ./contrib/guix/guix-clean + +Specifically, this will remove all files without an entry in the index, +excluding the SDK directory, the depends download cache, the depends built +packages cache, the garbage collector roots for Guix environments, and the +output directory. +EOF +for host in $hosts_distsrc_exists; do + echo " ${host} '$(distsrc_for_host "$host")'" +done +exit 1 +else + mkdir -p "$DISTSRC_BASE" +fi + + +################ +# Codesigning tarballs SHOULD exist +################ + +# Usage: outdir_for_host HOST SUFFIX +# +# HOST: The current platform triple we're building for +# +outdir_for_host() { + echo "${OUTDIR_BASE}/${1}${2:+-${2}}" +} + + +codesigning_tarball_for_host() { + case "$1" in + *mingw*) + echo "$(outdir_for_host "$1")/${DISTNAME}-win64-codesigning.tar.gz" + ;; + *darwin*) + echo "$(outdir_for_host "$1")/${DISTNAME}-${1}-codesigning.tar.gz" + ;; + *) + exit 1 + ;; + esac +} + +# Accumulate a list of build directories that already exist... +hosts_codesigning_tarball_missing="" +for host in $HOSTS; do + if [ ! -e "$(codesigning_tarball_for_host "$host")" ]; then + hosts_codesigning_tarball_missing+=" ${host}" + fi +done + +if [ -n "$hosts_codesigning_tarball_missing" ]; then + # ...so that we can print them out nicely in an error message + cat << EOF +ERR: Codesigning tarballs do not exist +... + +EOF +for host in $hosts_codesigning_tarball_missing; do + echo " ${host} '$(codesigning_tarball_for_host "$host")'" +done +exit 1 +fi + +################ +# Check that we can connect to the guix-daemon +################ + +cat << EOF +Checking that we can connect to the guix-daemon... + +Hint: If this hangs, you may want to try turning your guix-daemon off and on + again. + +EOF +if ! guix gc --list-failures > /dev/null; then + cat << EOF + +ERR: Failed to connect to the guix-daemon, please ensure that one is running and + reachable. +EOF + exit 1 +fi + +# Developer note: we could use `guix repl` for this check and run: +# +# (import (guix store)) (close-connection (open-connection)) +# +# However, the internal API is likely to change more than the CLI invocation + + +######### +# SETUP # +######### + +# Determine the maximum number of jobs to run simultaneously (overridable by +# environment) +JOBS="${JOBS:-$(nproc)}" + +# Determine the reference time used for determinism (overridable by environment) +SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git -c log.showSignature=false log --format=%at -1)}" + +# Make sure an output directory exists for our builds +OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}" +mkdir -p "$OUTDIR_BASE" + +# Usage: profiledir_for_host HOST SUFFIX +# +# HOST: The current platform triple we're building for +# +profiledir_for_host() { + echo "${PROFILES_BASE}/${1}${2:+-${2}}" +} + +######### +# BUILD # +######### + +# Function to be called when codesigning for host ${1} and the user interrupts +# the codesign +int_trap() { +cat << EOF +** INT received while codesigning ${1}, you may want to clean up the relevant + work directories (e.g. distsrc-*) before recodesigning + +Hint: To blow everything away, you may want to use: + + $ ./contrib/guix/guix-clean + +Specifically, this will remove all files without an entry in the index, +excluding the SDK directory, the depends download cache, the depends built +packages cache, the garbage collector roots for Guix environments, and the +output directory. +EOF +} + +# Deterministically build Bitcoin Core +# shellcheck disable=SC2153 +for host in $HOSTS; do + + # Display proper warning when the user interrupts the build + trap 'int_trap ${host}' INT + + ( + # Required for 'contrib/guix/manifest.scm' to output the right manifest + # for the particular $HOST we're building for + export HOST="$host" + + # shellcheck disable=SC2030 +cat << EOF +INFO: Codesigning ${VERSION:?not set} for platform triple ${HOST:?not set}: + ...using reference timestamp: ${SOURCE_DATE_EPOCH:?not set} + ...from worktree directory: '${PWD}' + ...bind-mounted in container to: '/bitcoin' + ...in build directory: '$(distsrc_for_host "$HOST")' + ...bind-mounted in container to: '$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")' + ...outputting in: '$(outdir_for_host "$HOST" codesigned)' + ...bind-mounted in container to: '$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST" codesigned)' + ...using detached signatures in: '${DETACHED_SIGS_REPO:?not set}' + ...bind-mounted in container to: '/detached-sigs' +EOF + + + # Run the build script 'contrib/guix/libexec/build.sh' in the build + # container specified by 'contrib/guix/manifest.scm'. + # + # Explanation of `guix shell` flags: + # + # --container run command within an isolated container + # + # Running in an isolated container minimizes build-time differences + # between machines and improves reproducibility + # + # --pure unset existing environment variables + # + # Same rationale as --container + # + # --no-cwd do not share current working directory with an + # isolated container + # + # When --container is specified, the default behavior is to share + # the current working directory with the isolated container at the + # same exact path (e.g. mapping '/home/satoshi/bitcoin/' to + # '/home/satoshi/bitcoin/'). This means that the $PWD inside the + # container becomes a source of irreproducibility. --no-cwd disables + # this behaviour. + # + # --share=SPEC for containers, share writable host file system + # according to SPEC + # + # --share="$PWD"=/bitcoin + # + # maps our current working directory to /bitcoin + # inside the isolated container, which we later cd + # into. + # + # While we don't want to map our current working directory to the + # same exact path (as this introduces irreproducibility), we do want + # it to be at a _fixed_ path _somewhere_ inside the isolated + # container so that we have something to build. '/bitcoin' was + # chosen arbitrarily. + # + # ${SOURCES_PATH:+--share="$SOURCES_PATH"} + # + # make the downloaded depends sources path available + # inside the isolated container + # + # The isolated container has no network access as it's in a + # different network namespace from the main machine, so we have to + # make the downloaded depends sources available to it. The sources + # should have been downloaded prior to this invocation. + # + # ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} + # + # fetch substitute from SUBSTITUTE_URLS if they are + # authorized + # + # Depending on the user's security model, it may be desirable to use + # substitutes (pre-built packages) from servers that the user trusts. + # Please read the README.md in the same directory as this file for + # more information. + # + # shellcheck disable=SC2086,SC2031 + time-machine shell --manifest="${PWD}/contrib/guix/manifest.scm" \ + --container \ + --pure \ + --no-cwd \ + --share="$PWD"=/bitcoin \ + --share="$DISTSRC_BASE"=/distsrc-base \ + --share="$OUTDIR_BASE"=/outdir-base \ + --share="$DETACHED_SIGS_REPO"=/detached-sigs \ + --expose="$(git rev-parse --git-common-dir)" \ + --expose="$(git -C "$DETACHED_SIGS_REPO" rev-parse --git-common-dir)" \ + ${SOURCES_PATH:+--share="$SOURCES_PATH"} \ + --cores="$JOBS" \ + --keep-failed \ + --fallback \ + --link-profile \ + --root="$(profiledir_for_host "${HOST}" codesigned)" \ + ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ + ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ + -- env HOST="$host" \ + DISTNAME="$DISTNAME" \ + JOBS="$JOBS" \ + SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \ + ${V:+V=1} \ + ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \ + DISTSRC="$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")" \ + OUTDIR="$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST" codesigned)" \ + DIST_ARCHIVE_BASE=/outdir-base/dist-archive \ + DETACHED_SIGS_REPO=/detached-sigs \ + CODESIGNING_TARBALL="$(OUTDIR_BASE=/outdir-base && codesigning_tarball_for_host "$HOST")" \ + bash -c "cd /bitcoin && bash contrib/guix/libexec/codesign.sh" + ) + +done diff --git a/bench-ci/guix/guix-verify b/bench-ci/guix/guix-verify new file mode 100755 index 000000000000..02ae022741ba --- /dev/null +++ b/bench-ci/guix/guix-verify @@ -0,0 +1,174 @@ +#!/usr/bin/env bash +export LC_ALL=C +set -e -o pipefail + +# Source the common prelude, which: +# 1. Checks if we're at the top directory of the Bitcoin Core repository +# 2. Defines a few common functions and variables +# +# shellcheck source=libexec/prelude.bash +source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" + + +################### +## Sanity Checks ## +################### + +################ +# Required non-builtin commands should be invokable +################ + +check_tools cat diff gpg + +################ +# Required env vars should be non-empty +################ + +cmd_usage() { +cat < [ SIGNER= ] ./contrib/guix/guix-verify + +Example overriding signer's manifest to use as base + + env GUIX_SIGS_REPO=/home/dongcarl/guix.sigs SIGNER=achow101 ./contrib/guix/guix-verify + +EOF +} + +if [ -z "$GUIX_SIGS_REPO" ]; then + cmd_usage + exit 1 +fi + +################ +# GUIX_SIGS_REPO should exist as a directory +################ + +if [ ! -d "$GUIX_SIGS_REPO" ]; then +cat << EOF +ERR: The specified GUIX_SIGS_REPO is not an existent directory: + + '$GUIX_SIGS_REPO' + +Hint: Please clone the guix.sigs repository and point to it with the + GUIX_SIGS_REPO environment variable. + +EOF +cmd_usage +exit 1 +fi + +############## +## Verify ## +############## + +OUTSIGDIR_BASE="${GUIX_SIGS_REPO}/${VERSION}" +echo "Looking for signature directories in '${OUTSIGDIR_BASE}'" +echo "" + +# Usage: verify compare_manifest current_manifest +verify() { + local compare_manifest="$1" + local current_manifest="$2" + if ! gpg --quiet --batch --verify "$current_manifest".asc "$current_manifest" 1>&2; then + echo "ERR: Failed to verify GPG signature in '${current_manifest}'" + echo "" + echo "Hint: Either the signature is invalid or the public key is missing" + echo "" + failure=1 + elif ! diff --report-identical "$compare_manifest" "$current_manifest" 1>&2; then + echo "ERR: The SHA256SUMS attestation in these two directories differ:" + echo " '${compare_manifest}'" + echo " '${current_manifest}'" + echo "" + failure=1 + else + echo "Verified: '${current_manifest}'" + echo "" + fi +} + +shopt -s nullglob +all_noncodesigned=( "$OUTSIGDIR_BASE"/*/noncodesigned.SHA256SUMS ) +shopt -u nullglob + +echo "--------------------" +echo "" +if (( ${#all_noncodesigned[@]} )); then + compare_noncodesigned="${all_noncodesigned[0]}" + if [[ -n "$SIGNER" ]]; then + signer_noncodesigned="$OUTSIGDIR_BASE/$SIGNER/noncodesigned.SHA256SUMS" + if [[ -f "$signer_noncodesigned" ]]; then + echo "Using $SIGNER's manifest as the base to compare against" + compare_noncodesigned="$signer_noncodesigned" + else + echo "Unable to find $SIGNER's manifest, using the first one found" + fi + else + echo "No SIGNER provided, using the first manifest found" + fi + + for current_manifest in "${all_noncodesigned[@]}"; do + verify "$compare_noncodesigned" "$current_manifest" + done + + echo "DONE: Checking output signatures for noncodesigned.SHA256SUMS" + echo "" +else + echo "WARN: No signature directories with noncodesigned.SHA256SUMS found" + echo "" +fi + +shopt -s nullglob +all_all=( "$OUTSIGDIR_BASE"/*/all.SHA256SUMS ) +shopt -u nullglob + +echo "--------------------" +echo "" +if (( ${#all_all[@]} )); then + compare_all="${all_all[0]}" + if [[ -n "$SIGNER" ]]; then + signer_all="$OUTSIGDIR_BASE/$SIGNER/all.SHA256SUMS" + if [[ -f "$signer_all" ]]; then + echo "Using $SIGNER's manifest as the base to compare against" + compare_all="$signer_all" + else + echo "Unable to find $SIGNER's manifest, using the first one found" + fi + else + echo "No SIGNER provided, using the first manifest found" + fi + + for current_manifest in "${all_all[@]}"; do + verify "$compare_all" "$current_manifest" + done + + # Sanity check: there should be no entries that exist in + # noncodesigned.SHA256SUMS that doesn't exist in all.SHA256SUMS + if [[ "$(comm -23 <(sort "$compare_noncodesigned") <(sort "$compare_all") | wc -c)" -ne 0 ]]; then + echo "ERR: There are unique lines in noncodesigned.SHA256SUMS which" + echo " do not exist in all.SHA256SUMS, something went very wrong." + exit 1 + fi + + echo "DONE: Checking output signatures for all.SHA256SUMS" + echo "" +else + echo "WARN: No signature directories with all.SHA256SUMS found" + echo "" +fi + +echo "====================" +echo "" +if (( ${#all_noncodesigned[@]} + ${#all_all[@]} == 0 )); then + echo "ERR: Unable to perform any verifications as no signature directories" + echo " were found" + echo "" + exit 1 +fi + +if [ -n "$failure" ]; then + exit 1 +fi diff --git a/bench-ci/guix/libexec/build.sh b/bench-ci/guix/libexec/build.sh new file mode 100755 index 000000000000..16e12d563816 --- /dev/null +++ b/bench-ci/guix/libexec/build.sh @@ -0,0 +1,403 @@ +#!/usr/bin/env bash +# Copyright (c) 2019-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +export LC_ALL=C +set -e -o pipefail +export TZ=UTC + +# Although Guix _does_ set umask when building its own packages (in our case, +# this is all packages in manifest.scm), it does not set it for `guix +# shell`. It does make sense for at least `guix shell --container` +# to set umask, so if that change gets merged upstream and we bump the +# time-machine to a commit which includes the aforementioned change, we can +# remove this line. +# +# This line should be placed before any commands which creates files. +umask 0022 + +if [ -n "$V" ]; then + # Print both unexpanded (-v) and expanded (-x) forms of commands as they are + # read from this file. + set -vx + # Set VERBOSE for CMake-based builds + export VERBOSE="$V" +fi + +# Check that required environment variables are set +cat << EOF +Required environment variables as seen inside the container: + DIST_ARCHIVE_BASE: ${DIST_ARCHIVE_BASE:?not set} + DISTNAME: ${DISTNAME:?not set} + HOST: ${HOST:?not set} + SOURCE_DATE_EPOCH: ${SOURCE_DATE_EPOCH:?not set} + JOBS: ${JOBS:?not set} + DISTSRC: ${DISTSRC:?not set} + OUTDIR: ${OUTDIR:?not set} +EOF + +ACTUAL_OUTDIR="${OUTDIR}" +OUTDIR="${DISTSRC}/output" + +##################### +# Environment Setup # +##################### + +# The depends folder also serves as a base-prefix for depends packages for +# $HOSTs after successfully building. +BASEPREFIX="${PWD}/depends" + +# Given a package name and an output name, return the path of that output in our +# current guix environment +store_path() { + grep --extended-regexp "/[^-]{32}-${1}-[^-]+${2:+-${2}}" "${GUIX_ENVIRONMENT}/manifest" \ + | head --lines=1 \ + | sed --expression='s|\x29*$||' \ + --expression='s|^[[:space:]]*"||' \ + --expression='s|"[[:space:]]*$||' +} + + +# Set environment variables to point the NATIVE toolchain to the right +# includes/libs +NATIVE_GCC="$(store_path gcc-toolchain)" + +unset LIBRARY_PATH +unset CPATH +unset C_INCLUDE_PATH +unset CPLUS_INCLUDE_PATH +unset OBJC_INCLUDE_PATH +unset OBJCPLUS_INCLUDE_PATH + +# Set native toolchain +build_CC="${NATIVE_GCC}/bin/gcc -isystem ${NATIVE_GCC}/include" +build_CXX="${NATIVE_GCC}/bin/g++ -isystem ${NATIVE_GCC}/include/c++ -isystem ${NATIVE_GCC}/include" + +case "$HOST" in + *darwin*) export LIBRARY_PATH="${NATIVE_GCC}/lib" ;; # Required for native packages + *mingw*) export LIBRARY_PATH="${NATIVE_GCC}/lib" ;; + *) + NATIVE_GCC_STATIC="$(store_path gcc-toolchain static)" + export LIBRARY_PATH="${NATIVE_GCC}/lib:${NATIVE_GCC_STATIC}/lib" + ;; +esac + +# Set environment variables to point the CROSS toolchain to the right +# includes/libs for $HOST +case "$HOST" in + *mingw*) + # Determine output paths to use in CROSS_* environment variables + CROSS_GLIBC="$(store_path "mingw-w64-x86_64-winpthreads")" + CROSS_GCC="$(store_path "gcc-cross-${HOST}")" + CROSS_GCC_LIB_STORE="$(store_path "gcc-cross-${HOST}" lib)" + CROSS_GCC_LIBS=( "${CROSS_GCC_LIB_STORE}/lib/gcc/${HOST}"/* ) # This expands to an array of directories... + CROSS_GCC_LIB="${CROSS_GCC_LIBS[0]}" # ...we just want the first one (there should only be one) + + # The search path ordering is generally: + # 1. gcc-related search paths + # 2. libc-related search paths + # 2. kernel-header-related search paths (not applicable to mingw-w64 hosts) + export CROSS_C_INCLUDE_PATH="${CROSS_GCC_LIB}/include:${CROSS_GCC_LIB}/include-fixed:${CROSS_GLIBC}/include" + export CROSS_CPLUS_INCLUDE_PATH="${CROSS_GCC}/include/c++:${CROSS_GCC}/include/c++/${HOST}:${CROSS_GCC}/include/c++/backward:${CROSS_C_INCLUDE_PATH}" + export CROSS_LIBRARY_PATH="${CROSS_GCC_LIB_STORE}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib" + ;; + *darwin*) + # The CROSS toolchain for darwin uses the SDK and ignores environment variables. + # See depends/hosts/darwin.mk for more details. + ;; + *linux*) + CROSS_GLIBC="$(store_path "glibc-cross-${HOST}")" + CROSS_GLIBC_STATIC="$(store_path "glibc-cross-${HOST}" static)" + CROSS_KERNEL="$(store_path "linux-libre-headers-cross-${HOST}")" + CROSS_GCC="$(store_path "gcc-cross-${HOST}")" + CROSS_GCC_LIB_STORE="$(store_path "gcc-cross-${HOST}" lib)" + CROSS_GCC_LIBS=( "${CROSS_GCC_LIB_STORE}/lib/gcc/${HOST}"/* ) # This expands to an array of directories... + CROSS_GCC_LIB="${CROSS_GCC_LIBS[0]}" # ...we just want the first one (there should only be one) + + export CROSS_C_INCLUDE_PATH="${CROSS_GCC_LIB}/include:${CROSS_GCC_LIB}/include-fixed:${CROSS_GLIBC}/include:${CROSS_KERNEL}/include" + export CROSS_CPLUS_INCLUDE_PATH="${CROSS_GCC}/include/c++:${CROSS_GCC}/include/c++/${HOST}:${CROSS_GCC}/include/c++/backward:${CROSS_C_INCLUDE_PATH}" + export CROSS_LIBRARY_PATH="${CROSS_GCC_LIB_STORE}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib:${CROSS_GLIBC_STATIC}/lib" + ;; + *) + exit 1 ;; +esac + +# Sanity check CROSS_*_PATH directories +IFS=':' read -ra PATHS <<< "${CROSS_C_INCLUDE_PATH}:${CROSS_CPLUS_INCLUDE_PATH}:${CROSS_LIBRARY_PATH}" +for p in "${PATHS[@]}"; do + if [ -n "$p" ] && [ ! -d "$p" ]; then + echo "'$p' doesn't exist or isn't a directory... Aborting..." + exit 1 + fi +done + +# Disable Guix ld auto-rpath behavior +export GUIX_LD_WRAPPER_DISABLE_RPATH=yes + +# Make /usr/bin if it doesn't exist +[ -e /usr/bin ] || mkdir -p /usr/bin + +# Symlink env to a conventional path +[ -e /usr/bin/env ] || ln -s --no-dereference "$(command -v env)" /usr/bin/env + +# Determine the correct value for -Wl,--dynamic-linker for the current $HOST +case "$HOST" in + *linux*) + glibc_dynamic_linker=$( + case "$HOST" in + x86_64-linux-gnu) echo /lib64/ld-linux-x86-64.so.2 ;; + arm-linux-gnueabihf) echo /lib/ld-linux-armhf.so.3 ;; + aarch64-linux-gnu) echo /lib/ld-linux-aarch64.so.1 ;; + riscv64-linux-gnu) echo /lib/ld-linux-riscv64-lp64d.so.1 ;; + powerpc64-linux-gnu) echo /lib64/ld64.so.1;; + powerpc64le-linux-gnu) echo /lib64/ld64.so.2;; + *) exit 1 ;; + esac + ) + ;; +esac + +# Environment variables for determinism +export TAR_OPTIONS="--owner=0 --group=0 --numeric-owner --mtime='@${SOURCE_DATE_EPOCH}' --sort=name" +export TZ="UTC" + +#################### +# Depends Building # +#################### + +# Build the depends tree, overriding variables that assume multilib gcc +make -C depends --jobs="$JOBS" HOST="$HOST" \ + ${V:+V=1} \ + ${SOURCES_PATH+SOURCES_PATH="$SOURCES_PATH"} \ + ${BASE_CACHE+BASE_CACHE="$BASE_CACHE"} \ + ${SDK_PATH+SDK_PATH="$SDK_PATH"} \ + ${build_CC+build_CC="$build_CC"} \ + ${build_CXX+build_CXX="$build_CXX"} \ + x86_64_linux_CC=x86_64-linux-gnu-gcc \ + x86_64_linux_CXX=x86_64-linux-gnu-g++ \ + x86_64_linux_AR=x86_64-linux-gnu-gcc-ar \ + x86_64_linux_RANLIB=x86_64-linux-gnu-gcc-ranlib \ + x86_64_linux_NM=x86_64-linux-gnu-gcc-nm \ + x86_64_linux_STRIP=x86_64-linux-gnu-strip + +case "$HOST" in + *darwin*) + # Unset now that Qt is built + unset LIBRARY_PATH + ;; +esac + +########################### +# Source Tarball Building # +########################### + +GIT_ARCHIVE="${DIST_ARCHIVE_BASE}/${DISTNAME}.tar.gz" + +# Create the source tarball if not already there +if [ ! -e "$GIT_ARCHIVE" ]; then + mkdir -p "$(dirname "$GIT_ARCHIVE")" + git archive --prefix="${DISTNAME}/" --output="$GIT_ARCHIVE" HEAD +fi + +mkdir -p "$OUTDIR" + +########################### +# Binary Tarball Building # +########################### + +# CONFIGFLAGS +CONFIGFLAGS="-DREDUCE_EXPORTS=ON -DBUILD_BENCH=OFF -DBUILD_GUI_TESTS=OFF -DBUILD_FUZZ_BINARY=OFF -DCMAKE_SKIP_RPATH=TRUE" + +# CFLAGS +HOST_CFLAGS="-O2 -g" +HOST_CFLAGS+=$(find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;) +case "$HOST" in + *mingw*) HOST_CFLAGS+=" -fno-ident" ;; + *darwin*) unset HOST_CFLAGS ;; +esac + +# CXXFLAGS +HOST_CXXFLAGS="$HOST_CFLAGS" + +case "$HOST" in + arm-linux-gnueabihf) HOST_CXXFLAGS="${HOST_CXXFLAGS} -Wno-psabi" ;; +esac + +# LDFLAGS +case "$HOST" in + *linux*) HOST_LDFLAGS="-Wl,--as-needed -Wl,--dynamic-linker=$glibc_dynamic_linker -static-libstdc++ -Wl,-O2" ;; + *mingw*) HOST_LDFLAGS="-Wl,--no-insert-timestamp" ;; +esac + +mkdir -p "$DISTSRC" +( + cd "$DISTSRC" + + # Extract the source tarball + tar --strip-components=1 -xf "${GIT_ARCHIVE}" + + # Configure this DISTSRC for $HOST + # shellcheck disable=SC2086 + env CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" LDFLAGS="${HOST_LDFLAGS}" \ + cmake -S . -B build \ + --toolchain "${BASEPREFIX}/${HOST}/toolchain.cmake" \ + -DWITH_CCACHE=OFF \ + -Werror=dev \ + ${CONFIGFLAGS} + + # Build Bitcoin Core + cmake --build build -j "$JOBS" ${V:+--verbose} + + # Perform basic security checks on a series of executables. + cmake --build build -j 1 --target check-security ${V:+--verbose} + # Check that executables only contain allowed version symbols. + cmake --build build -j 1 --target check-symbols ${V:+--verbose} + + mkdir -p "$OUTDIR" + + # Make the os-specific installers + case "$HOST" in + *mingw*) + cmake --build build -j "$JOBS" -t deploy ${V:+--verbose} + mv build/bitcoin-win64-setup.exe "${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe" + ;; + esac + + # Setup the directory where our Bitcoin Core build for HOST will be + # installed. This directory will also later serve as the input for our + # binary tarballs. + INSTALLPATH="${PWD}/installed/${DISTNAME}" + mkdir -p "${INSTALLPATH}" + # Install built Bitcoin Core to $INSTALLPATH + case "$HOST" in + *darwin*) + # This workaround can be dropped for CMake >= 3.27. + # See the upstream commit 689616785f76acd844fd448c51c5b2a0711aafa2. + find build -name 'cmake_install.cmake' -exec sed -i 's| -u -r | |g' {} + + + cmake --install build --strip --prefix "${INSTALLPATH}" ${V:+--verbose} + ;; + *) + cmake --install build --prefix "${INSTALLPATH}" ${V:+--verbose} + ;; + esac + + ( + cd installed + + case "$HOST" in + *darwin*) ;; + *) + # Split binaries from their debug symbols + { + find "${DISTNAME}/bin" "${DISTNAME}/libexec" -type f -executable -print0 + } | xargs -0 -P"$JOBS" -I{} "${DISTSRC}/build/split-debug.sh" {} {} {}.dbg + ;; + esac + + case "$HOST" in + *mingw*) + cp "${DISTSRC}/doc/README_windows.txt" "${DISTNAME}/readme.txt" + ;; + *linux*) + cp "${DISTSRC}/README.md" "${DISTNAME}/" + ;; + esac + + # copy over the example bitcoin.conf file. if contrib/devtools/gen-bitcoin-conf.sh + # has not been run before buildling, this file will be a stub + cp "${DISTSRC}/share/examples/bitcoin.conf" "${DISTNAME}/" + + cp -r "${DISTSRC}/share/rpcauth" "${DISTNAME}/share/" + + # Deterministically produce {non-,}debug binary tarballs ready + # for release + case "$HOST" in + *mingw*) + find "${DISTNAME}" -not -name "*.dbg" -print0 \ + | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" + find "${DISTNAME}" -not -name "*.dbg" \ + | sort \ + | zip -X@ "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}-unsigned.zip" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}-unsigned.zip" && exit 1 ) + find "${DISTNAME}" -name "*.dbg" -print0 \ + | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" + find "${DISTNAME}" -name "*.dbg" \ + | sort \ + | zip -X@ "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}-debug.zip" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}-debug.zip" && exit 1 ) + ;; + *linux*) + find "${DISTNAME}" -not -name "*.dbg" -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" && exit 1 ) + find "${DISTNAME}" -name "*.dbg" -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}-debug.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}-debug.tar.gz" && exit 1 ) + ;; + *darwin*) + find "${DISTNAME}" -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}-unsigned.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}-unsigned.tar.gz" && exit 1 ) + ;; + esac + ) # $DISTSRC/installed + + # Finally make tarballs for codesigning + case "$HOST" in + *mingw*) + cp -rf --target-directory=. contrib/windeploy + ( + cd ./windeploy + mkdir -p unsigned + cp --target-directory=unsigned/ "${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe" + cp -r --target-directory=unsigned/ "${INSTALLPATH}" + find unsigned/ -name "*.dbg" -print0 \ + | xargs -0r rm + find . -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-win64-codesigning.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-win64-codesigning.tar.gz" && exit 1 ) + ) + ;; + *darwin*) + cmake --build build --target deploy ${V:+--verbose} + mv build/dist/bitcoin-macos-app.zip "${OUTDIR}/${DISTNAME}-${HOST}-unsigned.zip" + mkdir -p "unsigned-app-${HOST}" + cp --target-directory="unsigned-app-${HOST}" \ + contrib/macdeploy/detached-sig-create.sh + mv --target-directory="unsigned-app-${HOST}" build/dist + cp -r --target-directory="unsigned-app-${HOST}" "${INSTALLPATH}" + ( + cd "unsigned-app-${HOST}" + find . -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}-codesigning.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}-codesigning.tar.gz" && exit 1 ) + ) + ;; + esac +) # $DISTSRC + +rm -rf "$ACTUAL_OUTDIR" +mv --no-target-directory "$OUTDIR" "$ACTUAL_OUTDIR" \ + || ( rm -rf "$ACTUAL_OUTDIR" && exit 1 ) + +( + cd /outdir-base + { + echo "$GIT_ARCHIVE" + find "$ACTUAL_OUTDIR" -type f + } | xargs realpath --relative-base="$PWD" \ + | xargs sha256sum \ + | sort -k2 \ + | sponge "$ACTUAL_OUTDIR"/SHA256SUMS.part +) diff --git a/bench-ci/guix/libexec/codesign.sh b/bench-ci/guix/libexec/codesign.sh new file mode 100755 index 000000000000..fe86065350e9 --- /dev/null +++ b/bench-ci/guix/libexec/codesign.sh @@ -0,0 +1,153 @@ +#!/usr/bin/env bash +# Copyright (c) 2021-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +export LC_ALL=C +set -e -o pipefail + +# Environment variables for determinism +export TAR_OPTIONS="--owner=0 --group=0 --numeric-owner --mtime='@${SOURCE_DATE_EPOCH}' --sort=name" +export TZ=UTC + +# Although Guix _does_ set umask when building its own packages (in our case, +# this is all packages in manifest.scm), it does not set it for `guix +# shell`. It does make sense for at least `guix shell --container` +# to set umask, so if that change gets merged upstream and we bump the +# time-machine to a commit which includes the aforementioned change, we can +# remove this line. +# +# This line should be placed before any commands which creates files. +umask 0022 + +if [ -n "$V" ]; then + # Print both unexpanded (-v) and expanded (-x) forms of commands as they are + # read from this file. + set -vx + # Set VERBOSE for CMake-based builds + export VERBOSE="$V" +fi + +# Check that required environment variables are set +cat << EOF +Required environment variables as seen inside the container: + CODESIGNING_TARBALL: ${CODESIGNING_TARBALL:?not set} + DETACHED_SIGS_REPO: ${DETACHED_SIGS_REPO:?not set} + DIST_ARCHIVE_BASE: ${DIST_ARCHIVE_BASE:?not set} + DISTNAME: ${DISTNAME:?not set} + HOST: ${HOST:?not set} + SOURCE_DATE_EPOCH: ${SOURCE_DATE_EPOCH:?not set} + DISTSRC: ${DISTSRC:?not set} + OUTDIR: ${OUTDIR:?not set} +EOF + +ACTUAL_OUTDIR="${OUTDIR}" +OUTDIR="${DISTSRC}/output" + +git_head_version() { + local recent_tag + if recent_tag="$(git -C "$1" describe --exact-match HEAD 2> /dev/null)"; then + echo "${recent_tag#v}" + else + git -C "$1" rev-parse --short=12 HEAD + fi +} + +CODESIGNATURE_GIT_ARCHIVE="${DIST_ARCHIVE_BASE}/${DISTNAME}-codesignatures-$(git_head_version "$DETACHED_SIGS_REPO").tar.gz" + +# Create the codesignature tarball if not already there +if [ ! -e "$CODESIGNATURE_GIT_ARCHIVE" ]; then + mkdir -p "$(dirname "$CODESIGNATURE_GIT_ARCHIVE")" + git -C "$DETACHED_SIGS_REPO" archive --output="$CODESIGNATURE_GIT_ARCHIVE" HEAD +fi + +mkdir -p "$OUTDIR" + +mkdir -p "$DISTSRC" +( + cd "$DISTSRC" + + tar -xf "$CODESIGNING_TARBALL" + + mkdir -p codesignatures + tar -C codesignatures -xf "$CODESIGNATURE_GIT_ARCHIVE" + + case "$HOST" in + *mingw*) + # Apply detached codesignatures + WORKDIR=".tmp" + mkdir -p ${WORKDIR} + cp -r --target-directory="${WORKDIR}" "unsigned/${DISTNAME}" + find "${WORKDIR}/${DISTNAME}" -name "*.exe" -type f -exec rm {} \; + find unsigned/ -name "*.exe" -type f | while read -r bin + do + bin_base="$(realpath --relative-to=unsigned/ "${bin}")" + mkdir -p "${WORKDIR}/$(dirname "${bin_base}")" + osslsigncode attach-signature \ + -in "${bin}" \ + -out "${WORKDIR}/${bin_base/-unsigned}" \ + -CAfile "$GUIX_ENVIRONMENT/etc/ssl/certs/ca-certificates.crt" \ + -sigin codesignatures/win/"${bin_base}".pem + done + + # Move installer to outdir + cd "${WORKDIR}" + find . -name "*setup.exe" -print0 \ + | xargs -0r mv --target-directory="${OUTDIR}" + + # Make .zip from binaries + find "${DISTNAME}" -print0 \ + | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" + find "${DISTNAME}" \ + | sort \ + | zip -X@ "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}.zip" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}.zip" && exit 1 ) + ;; + *darwin*) + case "$HOST" in + arm64*) ARCH="arm64" ;; + x86_64*) ARCH="x86_64" ;; + esac + + # Apply detached codesignatures (in-place) + signapple apply dist/Bitcoin-Qt.app codesignatures/osx/"${HOST}"/dist/Bitcoin-Qt.app + find "${DISTNAME}" \( -wholename "*/bin/*" -o -wholename "*/libexec/*" \) -type f | while read -r bin + do + signapple apply "${bin}" "codesignatures/osx/${HOST}/${bin}.${ARCH}sign" + done + + # Make a .zip from dist/ + cd dist/ + find . -print0 \ + | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" + find . | sort \ + | zip -X@ "${OUTDIR}/${DISTNAME}-${HOST}.zip" + cd .. + + # Make a .tar.gz from bins + find "${DISTNAME}" -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" && exit 1 ) + ;; + *) + exit 1 + ;; + esac +) # $DISTSRC + +rm -rf "$ACTUAL_OUTDIR" +mv --no-target-directory "$OUTDIR" "$ACTUAL_OUTDIR" \ + || ( rm -rf "$ACTUAL_OUTDIR" && exit 1 ) + +( + cd /outdir-base + { + echo "$CODESIGNING_TARBALL" + echo "$CODESIGNATURE_GIT_ARCHIVE" + find "$ACTUAL_OUTDIR" -type f + } | xargs realpath --relative-base="$PWD" \ + | xargs sha256sum \ + | sort -k2 \ + | sponge "$ACTUAL_OUTDIR"/SHA256SUMS.part +) diff --git a/bench-ci/guix/libexec/prelude.bash b/bench-ci/guix/libexec/prelude.bash new file mode 100644 index 000000000000..b7c13cc91d8c --- /dev/null +++ b/bench-ci/guix/libexec/prelude.bash @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +export LC_ALL=C +set -e -o pipefail + +# shellcheck source=contrib/shell/realpath.bash +source contrib/shell/realpath.bash + +# shellcheck source=contrib/shell/git-utils.bash +source contrib/shell/git-utils.bash + +################ +# Required non-builtin commands should be invocable +################ + +check_tools() { + for cmd in "$@"; do + if ! command -v "$cmd" > /dev/null 2>&1; then + echo "ERR: This script requires that '$cmd' is installed and available in your \$PATH" + exit 1 + fi + done +} + +################ +# SOURCE_DATE_EPOCH should not unintentionally be set +################ + +check_source_date_epoch() { + if [ -n "$SOURCE_DATE_EPOCH" ] && [ -z "$FORCE_SOURCE_DATE_EPOCH" ]; then + cat << EOF +ERR: Environment variable SOURCE_DATE_EPOCH is set which may break reproducibility. + + Aborting... + +Hint: You may want to: + 1. Unset this variable: \`unset SOURCE_DATE_EPOCH\` before rebuilding + 2. Set the 'FORCE_SOURCE_DATE_EPOCH' environment variable if you insist on + using your own epoch +EOF + exit 1 + fi +} + +check_tools cat env readlink dirname basename git + +################ +# We should be at the top directory of the repository +################ + +same_dir() { + local resolved1 resolved2 + resolved1="$(bash_realpath "${1}")" + resolved2="$(bash_realpath "${2}")" + [ "$resolved1" = "$resolved2" ] +} + +if ! same_dir "${PWD}" "$(git_root)"; then +cat << EOF +ERR: This script must be invoked from the top level of the git repository + +Hint: This may look something like: + env FOO=BAR ./contrib/guix/guix- + +EOF +exit 1 +fi + +################ +# Execute "$@" in a pinned, possibly older version of Guix, for reproducibility +# across time. +time-machine() { + # shellcheck disable=SC2086 + guix time-machine --url=https://codeberg.org/guix/guix.git \ + --commit=5cb84f2013c5b1e48a7d0e617032266f1e6059e2 \ + --cores="$JOBS" \ + --keep-failed \ + --fallback \ + ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ + ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS} \ + -- "$@" +} + + +################ +# Set common variables +################ + +VERSION="${FORCE_VERSION:-$(git_head_version)}" +DISTNAME="${DISTNAME:-bitcoin-${VERSION}}" + +version_base_prefix="${PWD}/guix-build-" +VERSION_BASE="${version_base_prefix}${VERSION}" # TOP + +DISTSRC_BASE="${DISTSRC_BASE:-${VERSION_BASE}}" + +OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}" + +var_base_basename="var" +VAR_BASE="${VAR_BASE:-${VERSION_BASE}/${var_base_basename}}" + +profiles_base_basename="profiles" +PROFILES_BASE="${PROFILES_BASE:-${VAR_BASE}/${profiles_base_basename}}" diff --git a/bench-ci/guix/manifest.scm b/bench-ci/guix/manifest.scm new file mode 100644 index 000000000000..59837e9647e9 --- /dev/null +++ b/bench-ci/guix/manifest.scm @@ -0,0 +1,574 @@ +(use-modules (gnu packages) + ((gnu packages bash) #:select (bash-minimal)) + (gnu packages bison) + ((gnu packages certs) #:select (nss-certs)) + ((gnu packages cmake) #:select (cmake-minimal)) + (gnu packages commencement) + (gnu packages compression) + (gnu packages cross-base) + (gnu packages gawk) + (gnu packages gcc) + ((gnu packages installers) #:select (nsis-x86_64)) + ((gnu packages linux) #:select (linux-libre-headers-6.1)) + (gnu packages llvm) + (gnu packages mingw) + (gnu packages ninja) + (gnu packages pkg-config) + ((gnu packages python) #:select (python-minimal)) + ((gnu packages python-build) #:select (python-poetry-core)) + ((gnu packages python-crypto) #:select (python-asn1crypto)) + ((gnu packages python-science) #:select (python-scikit-build-core)) + ((gnu packages python-xyz) #:select (python-pydantic-2)) + ((gnu packages tls) #:select (openssl)) + ((gnu packages version-control) #:select (git-minimal)) + (guix build-system cmake) + (guix build-system gnu) + (guix build-system python) + (guix build-system pyproject) + (guix build-system trivial) + (guix download) + (guix gexp) + (guix git-download) + ((guix licenses) #:prefix license:) + (guix packages) + ((guix utils) #:select (cc-for-target substitute-keyword-arguments))) + +(define-syntax-rule (search-our-patches file-name ...) + "Return the list of absolute file names corresponding to each +FILE-NAME found in ./patches relative to the current file." + (parameterize + ((%patch-path (list (string-append (dirname (current-filename)) "/patches")))) + (list (search-patch file-name) ...))) + +(define building-on (string-append "--build=" (list-ref (string-split (%current-system) #\-) 0) "-guix-linux-gnu")) + +(define (make-cross-toolchain target + base-gcc-for-libc + base-kernel-headers + base-libc + base-gcc) + "Create a cross-compilation toolchain package for TARGET" + (let* ((xbinutils (cross-binutils target)) + ;; 1. Build a cross-compiling gcc without targeting any libc, derived + ;; from BASE-GCC-FOR-LIBC + (xgcc-sans-libc (cross-gcc target + #:xgcc base-gcc-for-libc + #:xbinutils xbinutils)) + ;; 2. Build cross-compiled kernel headers with XGCC-SANS-LIBC, derived + ;; from BASE-KERNEL-HEADERS + (xkernel (cross-kernel-headers target + #:linux-headers base-kernel-headers + #:xgcc xgcc-sans-libc + #:xbinutils xbinutils)) + ;; 3. Build a cross-compiled libc with XGCC-SANS-LIBC and XKERNEL, + ;; derived from BASE-LIBC + (xlibc (cross-libc target + #:libc base-libc + #:xgcc xgcc-sans-libc + #:xbinutils xbinutils + #:xheaders xkernel)) + ;; 4. Build a cross-compiling gcc targeting XLIBC, derived from + ;; BASE-GCC + (xgcc (cross-gcc target + #:xgcc base-gcc + #:xbinutils xbinutils + #:libc xlibc))) + ;; Define a meta-package that propagates the resulting XBINUTILS, XLIBC, and + ;; XGCC + (package + (name (string-append target "-toolchain")) + (version (package-version xgcc)) + (source #f) + (build-system trivial-build-system) + (arguments '(#:builder (begin (mkdir %output) #t))) + (propagated-inputs + (list xbinutils + xlibc + xgcc + `(,xlibc "static") + `(,xgcc "lib"))) + (synopsis (string-append "Complete GCC tool chain for " target)) + (description (string-append "This package provides a complete GCC tool +chain for " target " development.")) + (home-page (package-home-page xgcc)) + (license (package-license xgcc))))) + +(define base-gcc gcc-13) ;; 13.3.0 + +(define base-linux-kernel-headers linux-libre-headers-6.1) + +(define* (make-bitcoin-cross-toolchain target + #:key + (base-gcc-for-libc linux-base-gcc) + (base-kernel-headers base-linux-kernel-headers) + (base-libc glibc-2.31) + (base-gcc linux-base-gcc)) + "Convenience wrapper around MAKE-CROSS-TOOLCHAIN with default values +desirable for building Bitcoin Core release binaries." + (make-cross-toolchain target + base-gcc-for-libc + base-kernel-headers + base-libc + base-gcc)) + +(define (gcc-mingw-patches gcc) + (package-with-extra-patches gcc + (search-our-patches "gcc-remap-guix-store.patch"))) + +(define (binutils-mingw-patches binutils) + (package-with-extra-patches binutils + (search-our-patches "binutils-unaligned-default.patch"))) + +(define (winpthreads-patches mingw-w64-x86_64-winpthreads) + (package-with-extra-patches mingw-w64-x86_64-winpthreads + (search-our-patches "winpthreads-remap-guix-store.patch"))) + +(define (make-mingw-pthreads-cross-toolchain target) + "Create a cross-compilation toolchain package for TARGET" + (let* ((xbinutils (binutils-mingw-patches (cross-binutils target))) + (machine (substring target 0 (string-index target #\-))) + (pthreads-xlibc (winpthreads-patches (make-mingw-w64 machine + #:xgcc (cross-gcc target #:xgcc (gcc-mingw-patches base-gcc)) + #:with-winpthreads? #t))) + (pthreads-xgcc (cross-gcc target + #:xgcc (gcc-mingw-patches mingw-w64-base-gcc) + #:xbinutils xbinutils + #:libc pthreads-xlibc))) + ;; Define a meta-package that propagates the resulting XBINUTILS, XLIBC, and + ;; XGCC + (package + (name (string-append target "-posix-toolchain")) + (version (package-version pthreads-xgcc)) + (source #f) + (build-system trivial-build-system) + (arguments '(#:builder (begin (mkdir %output) #t))) + (propagated-inputs + (list xbinutils + pthreads-xlibc + pthreads-xgcc + `(,pthreads-xgcc "lib"))) + (synopsis (string-append "Complete GCC tool chain for " target)) + (description (string-append "This package provides a complete GCC tool +chain for " target " development.")) + (home-page (package-home-page pthreads-xgcc)) + (license (package-license pthreads-xgcc))))) + +;; While LIEF is packaged in Guix, we maintain our own package, +;; to simplify building, and more easily apply updates. +;; Moreover, the Guix's package uses cmake, which caused build +;; failure; see https://github.com/bitcoin/bitcoin/pull/27296. +(define-public python-lief + (package + (name "python-lief") + (version "0.16.6") + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/lief-project/LIEF") + (commit version))) + (file-name (git-file-name name version)) + (sha256 + (base32 + "1pq9nagrnkl1x943bqnpiyxmkd9vk99znfxiwqp6vf012b50bz2a")) + (patches (search-our-patches "lief-scikit-0-9.patch")))) + (build-system pyproject-build-system) + (native-inputs (list cmake-minimal + ninja + python-scikit-build-core + python-pydantic-2)) + (arguments + (list + #:tests? #f ;needs network + #:phases #~(modify-phases %standard-phases + (add-before 'build 'set-pythonpath + (lambda _ + (setenv "PYTHONPATH" + (string-append (string-append (getcwd) "/api/python/backend") + ":" (or (getenv "PYTHONPATH") ""))))) + (add-after 'set-pythonpath 'change-directory + (lambda _ + (chdir "api/python")))))) + (home-page "https://github.com/lief-project/LIEF") + (synopsis "Library to instrument executable formats") + (description + "@code{python-lief} is a cross platform library which can parse, modify +and abstract ELF, PE and MachO formats.") + (license license:asl2.0))) + +(define osslsigncode + (package + (name "osslsigncode") + (version "2.5") + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/mtrojnar/osslsigncode") + (commit version))) + (sha256 + (base32 + "1j47vwq4caxfv0xw68kw5yh00qcpbd56d7rq6c483ma3y7s96yyz")))) + (build-system cmake-build-system) + (inputs (list openssl)) + (home-page "https://github.com/mtrojnar/osslsigncode") + (synopsis "Authenticode signing and timestamping tool") + (description "osslsigncode is a small tool that implements part of the +functionality of the Microsoft tool signtool.exe - more exactly the Authenticode +signing and timestamping. But osslsigncode is based on OpenSSL and cURL, and +thus should be able to compile on most platforms where these exist.") + (license license:gpl3+))) ; license is with openssl exception + +(define-public python-elfesteem + (let ((commit "2eb1e5384ff7a220fd1afacd4a0170acff54fe56")) + (package + (name "python-elfesteem") + (version (git-version "0.1" "1" commit)) + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/LRGH/elfesteem") + (commit commit))) + (file-name (git-file-name name commit)) + (sha256 + (base32 + "07x6p8clh11z8s1n2kdxrqwqm2almgc5qpkcr9ckb6y5ivjdr5r6")))) + (build-system python-build-system) + ;; There are no tests, but attempting to run python setup.py test leads to + ;; PYTHONPATH problems, just disable the test + (arguments '(#:tests? #f)) + (home-page "https://github.com/LRGH/elfesteem") + (synopsis "ELF/PE/Mach-O parsing library") + (description "elfesteem parses ELF, PE and Mach-O files.") + (license license:lgpl2.1)))) + +(define-public python-oscrypto + (package + (name "python-oscrypto") + (version "1.3.0") + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/wbond/oscrypto") + (commit version))) + (file-name (git-file-name name version)) + (sha256 + (base32 + "1v5wkmzcyiqy39db8j2dvkdrv2nlsc48556h73x4dzjwd6kg4q0a")) + (patches (search-our-patches "oscrypto-hard-code-openssl.patch")))) + (build-system python-build-system) + (native-search-paths + (list (search-path-specification + (variable "SSL_CERT_FILE") + (file-type 'regular) + (separator #f) ;single entry + (files '("etc/ssl/certs/ca-certificates.crt"))))) + + (propagated-inputs + (list python-asn1crypto openssl)) + (arguments + `(#:phases + (modify-phases %standard-phases + (add-after 'unpack 'hard-code-path-to-libscrypt + (lambda* (#:key inputs #:allow-other-keys) + (let ((openssl (assoc-ref inputs "openssl"))) + (substitute* "oscrypto/__init__.py" + (("@GUIX_OSCRYPTO_USE_OPENSSL@") + (string-append openssl "/lib/libcrypto.so" "," openssl "/lib/libssl.so"))) + #t))) + (add-after 'unpack 'disable-broken-tests + (lambda _ + ;; This test is broken as there is no keyboard interrupt. + (substitute* "tests/test_trust_list.py" + (("^(.*)class TrustListTests" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) + (substitute* "tests/test_tls.py" + (("^(.*)class TLSTests" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) + #t)) + (replace 'check + (lambda _ + (invoke "python" "run.py" "tests") + #t))))) + (home-page "https://github.com/wbond/oscrypto") + (synopsis "Compiler-free Python crypto library backed by the OS") + (description "oscrypto is a compilation-free, always up-to-date encryption library for Python.") + (license license:expat))) + +(define-public python-oscryptotests + (package (inherit python-oscrypto) + (name "python-oscryptotests") + (propagated-inputs + (list python-oscrypto)) + (arguments + `(#:tests? #f + #:phases + (modify-phases %standard-phases + (add-after 'unpack 'hard-code-path-to-libscrypt + (lambda* (#:key inputs #:allow-other-keys) + (chdir "tests") + #t))))))) + +(define-public python-certvalidator + (let ((commit "a145bf25eb75a9f014b3e7678826132efbba6213")) + (package + (name "python-certvalidator") + (version (git-version "0.1" "1" commit)) + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/achow101/certvalidator") + (commit commit))) + (file-name (git-file-name name commit)) + (sha256 + (base32 + "1qw2k7xis53179lpqdqyylbcmp76lj7sagp883wmxg5i7chhc96k")))) + (build-system python-build-system) + (propagated-inputs + (list python-asn1crypto + python-oscrypto + python-oscryptotests)) ;; certvalidator tests import oscryptotests + (arguments + `(#:phases + (modify-phases %standard-phases + (add-after 'unpack 'disable-broken-tests + (lambda _ + (substitute* "tests/test_certificate_validator.py" + (("^(.*)class CertificateValidatorTests" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) + (substitute* "tests/test_crl_client.py" + (("^(.*)def test_fetch_crl" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) + (substitute* "tests/test_ocsp_client.py" + (("^(.*)def test_fetch_ocsp" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) + (substitute* "tests/test_registry.py" + (("^(.*)def test_build_paths" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) + (substitute* "tests/test_validate.py" + (("^(.*)def test_revocation_mode_hard" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) + (substitute* "tests/test_validate.py" + (("^(.*)def test_revocation_mode_soft" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) + #t)) + (replace 'check + (lambda _ + (invoke "python" "run.py" "tests") + #t))))) + (home-page "https://github.com/wbond/certvalidator") + (synopsis "Python library for validating X.509 certificates and paths") + (description "certvalidator is a Python library for validating X.509 +certificates or paths. Supports various options, including: validation at a +specific moment in time, whitelisting and revocation checks.") + (license license:expat)))) + +(define-public python-signapple + (let ((commit "85bfcecc33d2773bc09bc318cec0614af2c8e287")) + (package + (name "python-signapple") + (version (git-version "0.2.0" "1" commit)) + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/achow101/signapple") + (commit commit))) + (file-name (git-file-name name commit)) + (sha256 + (base32 + "17yqjll8nw83q6dhgqhkl7w502z5vy9sln8m6mlx0f1c10isg8yg")))) + (build-system pyproject-build-system) + (propagated-inputs + (list python-asn1crypto + python-oscrypto + python-certvalidator + python-elfesteem)) + (native-inputs (list python-poetry-core)) + ;; There are no tests, but attempting to run python setup.py test leads to + ;; problems, just disable the test + (arguments '(#:tests? #f)) + (home-page "https://github.com/achow101/signapple") + (synopsis "Mach-O binary signature tool") + (description "signapple is a Python tool for creating, verifying, and +inspecting signatures in Mach-O binaries.") + (license license:expat)))) + +(define-public mingw-w64-base-gcc + (package + (inherit base-gcc) + (arguments + (substitute-keyword-arguments (package-arguments base-gcc) + ((#:configure-flags flags) + `(append ,flags + ;; https://gcc.gnu.org/install/configure.html + (list "--enable-threads=posix", + "--enable-default-ssp=yes", + "--disable-gcov", + building-on))))))) + +(define-public linux-base-gcc + (package + (inherit base-gcc) + (arguments + (substitute-keyword-arguments (package-arguments base-gcc) + ((#:configure-flags flags) + `(append ,flags + ;; https://gcc.gnu.org/install/configure.html + (list "--enable-initfini-array=yes", + "--enable-default-ssp=yes", + "--enable-default-pie=yes", + "--enable-standard-branch-protection=yes", + "--enable-cet=yes", + "--disable-gcov", + building-on))) + ((#:phases phases) + `(modify-phases ,phases + ;; Given a XGCC package, return a modified package that replace each instance of + ;; -rpath in the default system spec that's inserted by Guix with -rpath-link + (add-after 'pre-configure 'replace-rpath-with-rpath-link + (lambda _ + (substitute* (cons "gcc/config/rs6000/sysv4.h" + (find-files "gcc/config" + "^gnu-user.*\\.h$")) + (("-rpath=") "-rpath-link=")) + #t)))))))) + +(define-public glibc-2.31 + (let ((commit "7b27c450c34563a28e634cccb399cd415e71ebfe")) + (package + (inherit glibc) ;; 2.39 + (version "2.31") + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://sourceware.org/git/glibc.git") + (commit commit))) + (file-name (git-file-name "glibc" commit)) + (sha256 + (base32 + "017qdpr5id7ddb4lpkzj2li1abvw916m3fc6n7nw28z4h5qbv2n0")) + (patches (search-our-patches "glibc-guix-prefix.patch" + "glibc-riscv-jumptarget.patch")))) + (arguments + (substitute-keyword-arguments (package-arguments glibc) + ((#:configure-flags flags) + `(append ,flags + ;; https://www.gnu.org/software/libc/manual/html_node/Configuring-and-compiling.html + (list "--enable-stack-protector=all", + "--enable-cet", + "--enable-bind-now", + "--disable-werror", + "--disable-timezone-tools", + "--disable-profile", + building-on))) + ((#:phases phases) + `(modify-phases ,phases + (add-before 'configure 'set-etc-rpc-installation-directory + (lambda* (#:key outputs #:allow-other-keys) + ;; Install the rpc data base file under `$out/etc/rpc'. + ;; Otherwise build will fail with "Permission denied." + ;; Can be removed when we are building 2.32 or later. + (let ((out (assoc-ref outputs "out"))) + (substitute* "sunrpc/Makefile" + (("^\\$\\(inst_sysconfdir\\)/rpc(.*)$" _ suffix) + (string-append out "/etc/rpc" suffix "\n")) + (("^install-others =.*$") + (string-append "install-others = " out "/etc/rpc\n"))))))))))))) + +;; The sponge tool from moreutils. +(define-public sponge + (package + (name "sponge") + (version "0.69") + (source (origin + (method url-fetch) + (uri (string-append + "https://git.joeyh.name/index.cgi/moreutils.git/snapshot/ + moreutils-" version ".tar.gz")) + (file-name (string-append "moreutils-" version ".tar.gz")) + (sha256 + (base32 + "1l859qnzccslvxlh5ghn863bkq2vgmqgnik6jr21b9kc6ljmsy8g")))) + (build-system gnu-build-system) + (arguments + (list #:phases + #~(modify-phases %standard-phases + (delete 'configure) + (replace 'install + (lambda* (#:key outputs #:allow-other-keys) + (let ((bin (string-append (assoc-ref outputs "out") "/bin"))) + (install-file "sponge" bin))))) + #:make-flags + #~(list "sponge" (string-append "CC=" #$(cc-for-target))))) + (home-page "https://joeyh.name/code/moreutils/") + (synopsis "Miscellaneous general-purpose command-line tools") + (description "Just sponge") + (license license:gpl2+))) + +(packages->manifest + (append + (list ;; The Basics + bash-minimal + which + coreutils-minimal + ;; File(system) inspection + grep + diffutils + findutils + ;; File transformation + patch + gawk + sed + sponge + ;; Compression and archiving + tar + gzip + xz + ;; Build tools + gcc-toolchain-13 + cmake-minimal + gnu-make + ninja + ;; Scripting + python-minimal ;; (3.10) + ;; Git + git-minimal + ;; Tests + python-lief) + (let ((target (getenv "HOST"))) + (cond ((string-suffix? "-mingw32" target) + (list zip + (make-mingw-pthreads-cross-toolchain "x86_64-w64-mingw32") + nsis-x86_64 + nss-certs + osslsigncode)) + ((string-contains target "-linux-") + (list bison + pkg-config + (list gcc-toolchain-13 "static") + (make-bitcoin-cross-toolchain target))) + ((string-contains target "darwin") + (list clang-toolchain-19 + lld-19 + (make-lld-wrapper lld-19 #:lld-as-ld? #t) + python-signapple + zip)) + (else '()))))) diff --git a/bench-ci/guix/patches/binutils-unaligned-default.patch b/bench-ci/guix/patches/binutils-unaligned-default.patch new file mode 100644 index 000000000000..d1bc71aee142 --- /dev/null +++ b/bench-ci/guix/patches/binutils-unaligned-default.patch @@ -0,0 +1,22 @@ +commit 6537181f59ed186a341db621812a6bc35e22eaf6 +Author: fanquake +Date: Wed Apr 10 12:15:52 2024 +0200 + + build: turn on -muse-unaligned-vector-move by default + + This allows us to avoid (more invasively) patching GCC, to avoid + unaligned instruction use. + +diff --git a/gas/config/tc-i386.c b/gas/config/tc-i386.c +index e0632681477..14a9653abdf 100644 +--- a/gas/config/tc-i386.c ++++ b/gas/config/tc-i386.c +@@ -801,7 +801,7 @@ static unsigned int no_cond_jump_promotion = 0; + static unsigned int sse2avx; + + /* Encode aligned vector move as unaligned vector move. */ +-static unsigned int use_unaligned_vector_move; ++static unsigned int use_unaligned_vector_move = 1; + + /* Encode scalar AVX instructions with specific vector length. */ + static enum diff --git a/bench-ci/guix/patches/gcc-remap-guix-store.patch b/bench-ci/guix/patches/gcc-remap-guix-store.patch new file mode 100644 index 000000000000..a8b41d485b04 --- /dev/null +++ b/bench-ci/guix/patches/gcc-remap-guix-store.patch @@ -0,0 +1,20 @@ +Without ffile-prefix-map, the debug symbols will contain paths for the +guix store which will include the hashes of each package. However, the +hash for the same package will differ when on different architectures. +In order to be reproducible regardless of the architecture used to build +the package, map all guix store prefixes to something fixed, e.g. /usr. + +--- a/libgcc/Makefile.in ++++ b/libgcc/Makefile.in +@@ -854,7 +854,7 @@ endif + # libgcc_eh.a, only LIB2ADDEH matters. If we do, only LIB2ADDEHSTATIC and + # LIB2ADDEHSHARED matter. (Usually all three are identical.) + +-c_flags := -fexceptions ++c_flags := -fexceptions $(shell find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;) + + ifeq ($(enable_shared),yes) + +-- +2.37.0 + diff --git a/bench-ci/guix/patches/glibc-2.42-guix-prefix.patch b/bench-ci/guix/patches/glibc-2.42-guix-prefix.patch new file mode 100644 index 000000000000..f2fc1b90f183 --- /dev/null +++ b/bench-ci/guix/patches/glibc-2.42-guix-prefix.patch @@ -0,0 +1,47 @@ +Without -ffile-prefix-map, the debug symbols will contain paths for the +guix store which will include the hashes of each package. However, the +hash for the same package will differ when on different architectures. +In order to be reproducible regardless of the architecture used to build +the package, map all guix store prefixes to something fixed, e.g. /usr. + +--- a/Makeconfig ++++ b/Makeconfig +@@ -1074,6 +1074,10 @@ CPPFLAGS-.o = $(pic-default) + CFLAGS-.o = $(filter %frame-pointer,$(+cflags)) $(pie-default) + CFLAGS-.o += $(call elide-fortify-source,.o,$(routines_no_fortify)) + CFLAGS-.o += $(call elide-fortify-source,_chk.o,$(routines_no_fortify)) ++ ++# Map Guix store paths to /usr ++CFLAGS-.o += `find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;` ++ + libtype.o := lib%.a + object-suffixes += .o + ifeq (yes,$(build-shared)) +diff --git a/iconv/Makefile b/iconv/Makefile +index afb3fb7bdb..5acee345e0 100644 +--- a/iconv/Makefile ++++ b/iconv/Makefile +@@ -65,6 +65,9 @@ CFLAGS-gconv_cache.c += -DGCONV_DIR='"$(gconvdir)"' + CFLAGS-gconv_conf.c += -DGCONV_PATH='"$(gconvdir)"' + CFLAGS-iconvconfig.c += -DGCONV_PATH='"$(gconvdir)"' -DGCONV_DIR='"$(gconvdir)"' + ++# Map Guix store paths to /usr ++CFLAGS-.c += `find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;` ++ + # Set libof-* for each routine. + cpp-srcs-left := $(iconv_prog-modules) $(iconvconfig-modules) + lib := iconvprogs +diff --git a/posix/Makefile b/posix/Makefile +index 3d368b91f6..d79d8fb648 100644 +--- a/posix/Makefile ++++ b/posix/Makefile +@@ -590,6 +590,9 @@ CFLAGS-execlp.os = -fomit-frame-pointer + CFLAGS-nanosleep.c += -fexceptions -fasynchronous-unwind-tables + CFLAGS-fork.c = $(libio-mtsafe) $(config-cflags-wno-ignored-attributes) + ++# Map Guix store paths to /usr ++CFLAGS-.c += `find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;` ++ + tstgetopt-ARGS = -a -b -cfoobar --required foobar --optional=bazbug \ + --none random --col --color --colour + diff --git a/bench-ci/guix/patches/glibc-guix-prefix.patch b/bench-ci/guix/patches/glibc-guix-prefix.patch new file mode 100644 index 000000000000..60e12ca52546 --- /dev/null +++ b/bench-ci/guix/patches/glibc-guix-prefix.patch @@ -0,0 +1,16 @@ +Without ffile-prefix-map, the debug symbols will contain paths for the +guix store which will include the hashes of each package. However, the +hash for the same package will differ when on different architectures. +In order to be reproducible regardless of the architecture used to build +the package, map all guix store prefixes to something fixed, e.g. /usr. + +--- a/Makeconfig ++++ b/Makeconfig +@@ -1007,6 +1007,7 @@ object-suffixes := + CPPFLAGS-.o = $(pic-default) + # libc.a must be compiled with -fPIE/-fpie for static PIE. + CFLAGS-.o = $(filter %frame-pointer,$(+cflags)) $(pie-default) ++CFLAGS-.o += `find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;` + libtype.o := lib%.a + object-suffixes += .o + ifeq (yes,$(build-shared)) diff --git a/bench-ci/guix/patches/glibc-riscv-jumptarget.patch b/bench-ci/guix/patches/glibc-riscv-jumptarget.patch new file mode 100644 index 000000000000..702959433d86 --- /dev/null +++ b/bench-ci/guix/patches/glibc-riscv-jumptarget.patch @@ -0,0 +1,57 @@ +commit 68389203832ab39dd0dbaabbc4059e7fff51c29b +Author: Fangrui Song +Date: Thu Oct 28 11:39:49 2021 -0700 + + riscv: Fix incorrect jal with HIDDEN_JUMPTARGET + + A non-local STV_DEFAULT defined symbol is by default preemptible in a + shared object. j/jal cannot target a preemptible symbol. On other + architectures, such a jump instruction either causes PLT [BZ #18822], or + if short-ranged, sometimes rejected by the linker (but not by GNU ld's + riscv port [ld PR/28509]). + + Use HIDDEN_JUMPTARGET to target a non-preemptible symbol instead. + + With this patch, ld.so and libc.so can be linked with LLD if source + files are compiled/assembled with -mno-relax/-Wa,-mno-relax. + + Acked-by: Palmer Dabbelt + Reviewed-by: Adhemerval Zanella + +Can be dropped when we are using glibc 2.35 or later. + +diff --git a/sysdeps/riscv/setjmp.S b/sysdeps/riscv/setjmp.S +index 0b92016b31..bec7ff80f4 100644 +--- a/sysdeps/riscv/setjmp.S ++++ b/sysdeps/riscv/setjmp.S +@@ -21,7 +21,7 @@ + + ENTRY (_setjmp) + li a1, 0 +- j __sigsetjmp ++ j HIDDEN_JUMPTARGET (__sigsetjmp) + END (_setjmp) + ENTRY (setjmp) + li a1, 1 +diff --git a/sysdeps/unix/sysv/linux/riscv/setcontext.S b/sysdeps/unix/sysv/linux/riscv/setcontext.S +index 9510518750..e44a68aad4 100644 +--- a/sysdeps/unix/sysv/linux/riscv/setcontext.S ++++ b/sysdeps/unix/sysv/linux/riscv/setcontext.S +@@ -95,6 +95,7 @@ LEAF (__setcontext) + 99: j __syscall_error + + END (__setcontext) ++libc_hidden_def (__setcontext) + weak_alias (__setcontext, setcontext) + + LEAF (__start_context) +@@ -108,7 +109,7 @@ LEAF (__start_context) + /* Invoke subsequent context if present, else exit(0). */ + mv a0, s2 + beqz s2, 1f +- jal __setcontext +-1: j exit ++ jal HIDDEN_JUMPTARGET (__setcontext) ++1: j HIDDEN_JUMPTARGET (exit) + + END (__start_context) diff --git a/bench-ci/guix/patches/lief-scikit-0-9.patch b/bench-ci/guix/patches/lief-scikit-0-9.patch new file mode 100644 index 000000000000..71e617834f07 --- /dev/null +++ b/bench-ci/guix/patches/lief-scikit-0-9.patch @@ -0,0 +1,21 @@ +Partially revert f23ced2f4ffc170d0a6f40ff4a1bee575e3447cf + +Restore compat with python-scikit-build-core 0.9.x +Can be dropped when using python-scikit-build-core >= 0.10.x + +--- a/api/python/backend/setup.py ++++ b/api/python/backend/setup.py +@@ -101,12 +101,12 @@ def _get_hooked_config(is_editable: bool) -> Optional[dict[str, Union[str, List[ + config_settings = { + "logging.level": "DEBUG", + "build-dir": config.build_dir, +- "build.targets": config.build.targets, + "install.strip": config.strip, + "backport.find-python": "0", + "wheel.py-api": config.build.py_api, + "cmake.source-dir": SRC_DIR.as_posix(), + "cmake.build-type": config.build.build_type, ++ "cmake.targets": config.build.targets, + "cmake.args": [ + *config.cmake_generator, + *config.get_cmake_args(is_editable), diff --git a/bench-ci/guix/patches/oscrypto-hard-code-openssl.patch b/bench-ci/guix/patches/oscrypto-hard-code-openssl.patch new file mode 100644 index 000000000000..32027f2d09af --- /dev/null +++ b/bench-ci/guix/patches/oscrypto-hard-code-openssl.patch @@ -0,0 +1,13 @@ +diff --git a/oscrypto/__init__.py b/oscrypto/__init__.py +index eb27313..371ab24 100644 +--- a/oscrypto/__init__.py ++++ b/oscrypto/__init__.py +@@ -302,3 +302,8 @@ def load_order(): + 'oscrypto._win.tls', + 'oscrypto.tls', + ] ++ ++ ++paths = '@GUIX_OSCRYPTO_USE_OPENSSL@'.split(',') ++assert len(paths) == 2, 'Value for OSCRYPTO_USE_OPENSSL env var must be two paths separated by a comma' ++use_openssl(*paths) diff --git a/bench-ci/guix/patches/winpthreads-remap-guix-store.patch b/bench-ci/guix/patches/winpthreads-remap-guix-store.patch new file mode 100644 index 000000000000..e1f1a6eba531 --- /dev/null +++ b/bench-ci/guix/patches/winpthreads-remap-guix-store.patch @@ -0,0 +1,17 @@ +Without ffile-prefix-map, the debug symbols will contain paths for the +guix store which will include the hashes of each package. However, the +hash for the same package will differ when on different architectures. +In order to be reproducible regardless of the architecture used to build +the package, map all guix store prefixes to something fixed, e.g. /usr. + +--- a/mingw-w64-libraries/winpthreads/Makefile.in ++++ b/mingw-w64-libraries/winpthreads/Makefile.in +@@ -478,7 +478,7 @@ top_build_prefix = @top_build_prefix@ + top_builddir = @top_builddir@ + top_srcdir = @top_srcdir@ + SUBDIRS = . tests +-AM_CFLAGS = -Wall -DWIN32_LEAN_AND_MEAN $(am__append_1) ++AM_CFLAGS = -Wall -DWIN32_LEAN_AND_MEAN $(am__append_1) $(shell find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;) + ACLOCAL_AMFLAGS = -I m4 + lib_LTLIBRARIES = libwinpthread.la + include_HEADERS = include/pthread.h include/sched.h include/semaphore.h include/pthread_unistd.h include/pthread_time.h include/pthread_compat.h include/pthread_signal.h diff --git a/bench-ci/guix/security-check.py b/bench-ci/guix/security-check.py new file mode 100755 index 000000000000..be2e0cfbe2af --- /dev/null +++ b/bench-ci/guix/security-check.py @@ -0,0 +1,293 @@ +#!/usr/bin/env python3 +# Copyright (c) 2015-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +''' +Perform basic security checks on a series of executables. +Exit status will be 0 if successful, and the program will be silent. +Otherwise the exit status will be 1 and it will log which executables failed which checks. + +Example usage: + + find ../path/to/guix/binaries -type f -executable | xargs python3 contrib/guix/security-check.py +''' +import re +import sys + +import lief + +def check_ELF_RELRO(binary) -> bool: + ''' + Check for read-only relocations. + GNU_RELRO program header must exist + Dynamic section must have BIND_NOW flag + ''' + have_gnu_relro = False + for segment in binary.segments: + # Note: not checking p_flags == PF_R: here as linkers set the permission differently + # This does not affect security: the permission flags of the GNU_RELRO program + # header are ignored, the PT_LOAD header determines the effective permissions. + # However, the dynamic linker need to write to this area so these are RW. + # Glibc itself takes care of mprotecting this area R after relocations are finished. + # See also https://marc.info/?l=binutils&m=1498883354122353 + if segment.type == lief.ELF.Segment.TYPE.GNU_RELRO: + have_gnu_relro = True + + have_bindnow = False + try: + flags = binary.get(lief.ELF.DynamicEntry.TAG.FLAGS) + if flags.has(lief.ELF.DynamicEntryFlags.FLAG.BIND_NOW): + have_bindnow = True + except Exception: + have_bindnow = False + + return have_gnu_relro and have_bindnow + +def check_ELF_CANARY(binary) -> bool: + ''' + Check for use of stack canary + ''' + return binary.has_symbol('__stack_chk_fail') + +def check_ELF_SEPARATE_CODE(binary): + ''' + Check that sections are appropriately separated in virtual memory, + based on their permissions. This checks for missing -Wl,-z,separate-code + and potentially other problems. + ''' + R = lief.ELF.Segment.FLAGS.R + W = lief.ELF.Segment.FLAGS.W + E = lief.ELF.Segment.FLAGS.X + EXPECTED_FLAGS = { + # Read + execute + '.init': R | E, + '.plt': R | E, + '.plt.got': R | E, + '.plt.sec': R | E, + '.text': R | E, + '.fini': R | E, + # Read-only data + '.interp': R, + '.note.gnu.property': R, + '.note.gnu.build-id': R, + '.note.ABI-tag': R, + '.gnu.hash': R, + '.dynsym': R, + '.dynstr': R, + '.gnu.version': R, + '.gnu.version_r': R, + '.rela.dyn': R, + '.rela.plt': R, + '.rodata': R, + '.eh_frame_hdr': R, + '.eh_frame': R, + '.qtmetadata': R, + '.gcc_except_table': R, + '.stapsdt.base': R, + # Writable data + '.init_array': R | W, + '.fini_array': R | W, + '.dynamic': R | W, + '.got': R | W, + '.data': R | W, + '.bss': R | W, + } + if binary.header.machine_type == lief.ELF.ARCH.PPC64: + # .plt is RW on ppc64 even with separate-code + EXPECTED_FLAGS['.plt'] = R | W + # For all LOAD program headers get mapping to the list of sections, + # and for each section, remember the flags of the associated program header. + flags_per_section = {} + for segment in binary.segments: + if segment.type == lief.ELF.Segment.TYPE.LOAD: + for section in segment.sections: + flags_per_section[section.name] = segment.flags + # Spot-check ELF LOAD program header flags per section + # If these sections exist, check them against the expected R/W/E flags + for (section, flags) in flags_per_section.items(): + if section in EXPECTED_FLAGS: + if int(EXPECTED_FLAGS[section]) != int(flags): + return False + return True + +def check_ELF_CONTROL_FLOW(binary) -> bool: + ''' + Check for control flow instrumentation + ''' + main = binary.get_function_address('main') + content = binary.get_content_from_virtual_address(main, 4, lief.Binary.VA_TYPES.AUTO) + + if content.tolist() == [243, 15, 30, 250]: # endbr64 + return True + return False + +def check_ELF_FORTIFY(binary) -> bool: + + # bitcoin wrapper does not currently contain any fortified functions + if '--monolithic' in binary.strings: + return True + + chk_funcs = set() + + for sym in binary.imported_symbols: + match = re.search(r'__[a-z]*_chk', sym.name) + if match: + chk_funcs.add(match.group(0)) + + # ignore stack-protector + chk_funcs.discard('__stack_chk') + + return len(chk_funcs) >= 1 + +def check_PE_DYNAMIC_BASE(binary) -> bool: + '''PIE: DllCharacteristics bit 0x40 signifies dynamicbase (ASLR)''' + return lief.PE.OptionalHeader.DLL_CHARACTERISTICS.DYNAMIC_BASE in binary.optional_header.dll_characteristics_lists + +# Must support high-entropy 64-bit address space layout randomization +# in addition to DYNAMIC_BASE to have secure ASLR. +def check_PE_HIGH_ENTROPY_VA(binary) -> bool: + '''PIE: DllCharacteristics bit 0x20 signifies high-entropy ASLR''' + return lief.PE.OptionalHeader.DLL_CHARACTERISTICS.HIGH_ENTROPY_VA in binary.optional_header.dll_characteristics_lists + +def check_PE_RELOC_SECTION(binary) -> bool: + '''Check for a reloc section. This is required for functional ASLR.''' + return binary.has_relocations + +def check_PE_CONTROL_FLOW(binary) -> bool: + ''' + Check for control flow instrumentation + ''' + main = binary.get_symbol('main').value + + section_addr = binary.section_from_rva(main).virtual_address + virtual_address = binary.optional_header.imagebase + section_addr + main + + content = binary.get_content_from_virtual_address(virtual_address, 4, lief.Binary.VA_TYPES.VA) + + if content.tolist() == [243, 15, 30, 250]: # endbr64 + return True + return False + +def check_PE_CANARY(binary) -> bool: + ''' + Check for use of stack canary + ''' + return binary.has_symbol('__stack_chk_fail') + +def check_MACHO_NOUNDEFS(binary) -> bool: + ''' + Check for no undefined references. + ''' + return binary.header.has(lief.MachO.Header.FLAGS.NOUNDEFS) + +def check_MACHO_FIXUP_CHAINS(binary) -> bool: + ''' + Check for use of chained fixups. + ''' + return binary.has_dyld_chained_fixups + +def check_MACHO_CANARY(binary) -> bool: + ''' + Check for use of stack canary + ''' + return binary.has_symbol('___stack_chk_fail') + +def check_PIE(binary) -> bool: + ''' + Check for position independent executable (PIE), + allowing for address space randomization. + ''' + return binary.is_pie + +def check_NX(binary) -> bool: + ''' + Check for no stack execution + ''' + + # binary.has_nx checks are only for the stack, but MachO binaries might + # have executable heaps. + if binary.format == lief.Binary.FORMATS.MACHO: + return binary.concrete.has_nx_stack and binary.concrete.has_nx_heap + else: + return binary.has_nx + +def check_MACHO_CONTROL_FLOW(binary) -> bool: + ''' + Check for control flow instrumentation + ''' + content = binary.get_content_from_virtual_address(binary.entrypoint, 4, lief.Binary.VA_TYPES.AUTO) + + if content.tolist() == [243, 15, 30, 250]: # endbr64 + return True + return False + +def check_MACHO_BRANCH_PROTECTION(binary) -> bool: + ''' + Check for branch protection instrumentation + ''' + content = binary.get_content_from_virtual_address(binary.entrypoint, 4, lief.Binary.VA_TYPES.AUTO) + + if content.tolist() == [95, 36, 3, 213]: # bti + return True + return False + +BASE_ELF = [ + ('FORTIFY', check_ELF_FORTIFY), + ('PIE', check_PIE), + ('NX', check_NX), + ('RELRO', check_ELF_RELRO), + ('CANARY', check_ELF_CANARY), + ('SEPARATE_CODE', check_ELF_SEPARATE_CODE), +] + +BASE_PE = [ + ('PIE', check_PIE), + ('DYNAMIC_BASE', check_PE_DYNAMIC_BASE), + ('HIGH_ENTROPY_VA', check_PE_HIGH_ENTROPY_VA), + ('NX', check_NX), + ('RELOC_SECTION', check_PE_RELOC_SECTION), + ('CONTROL_FLOW', check_PE_CONTROL_FLOW), + ('CANARY', check_PE_CANARY), +] + +BASE_MACHO = [ + ('NOUNDEFS', check_MACHO_NOUNDEFS), + ('CANARY', check_MACHO_CANARY), + ('FIXUP_CHAINS', check_MACHO_FIXUP_CHAINS), +] + +CHECKS = { + lief.Binary.FORMATS.ELF: { + lief.Header.ARCHITECTURES.X86_64: BASE_ELF + [('CONTROL_FLOW', check_ELF_CONTROL_FLOW)], + lief.Header.ARCHITECTURES.ARM: BASE_ELF, + lief.Header.ARCHITECTURES.ARM64: BASE_ELF, + lief.Header.ARCHITECTURES.PPC64: BASE_ELF, + lief.Header.ARCHITECTURES.RISCV: BASE_ELF, + }, + lief.Binary.FORMATS.PE: { + lief.Header.ARCHITECTURES.X86_64: BASE_PE, + }, + lief.Binary.FORMATS.MACHO: { + lief.Header.ARCHITECTURES.X86_64: BASE_MACHO + [('PIE', check_PIE), + ('NX', check_NX), + ('CONTROL_FLOW', check_MACHO_CONTROL_FLOW)], + lief.Header.ARCHITECTURES.ARM64: BASE_MACHO + [('BRANCH_PROTECTION', check_MACHO_BRANCH_PROTECTION)], + } +} + +if __name__ == '__main__': + retval: int = 0 + for filename in sys.argv[1:]: + binary = lief.parse(filename) + + etype = binary.format + arch = binary.abstract.header.architecture + + failed: list[str] = [] + for (name, func) in CHECKS[etype][arch]: + if not func(binary): + failed.append(name) + if failed: + print(f'{filename}: failed {" ".join(failed)}') + retval = 1 + sys.exit(retval) diff --git a/bench-ci/guix/symbol-check.py b/bench-ci/guix/symbol-check.py new file mode 100755 index 000000000000..464b33cf66fb --- /dev/null +++ b/bench-ci/guix/symbol-check.py @@ -0,0 +1,335 @@ +#!/usr/bin/env python3 +# Copyright (c) 2014 Wladimir J. van der Laan +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +''' +A script to check that release executables only contain certain symbols +and are only linked against allowed libraries. + +Example usage: + + find ../path/to/guix/binaries -type f -executable | xargs python3 contrib/guix/symbol-check.py +''' +import sys + +import lief + +# Debian 11 (Bullseye) EOL: 2026. https://wiki.debian.org/LTS +# +# - libgcc version 10.2.1 (https://packages.debian.org/bullseye/libgcc-s1) +# - libc version 2.31 (https://packages.debian.org/source/bullseye/glibc) +# +# Ubuntu 20.04 (Focal) EOL: 2030. https://wiki.ubuntu.com/ReleaseTeam +# +# - libgcc version 10.5.0 (https://packages.ubuntu.com/focal/libgcc1) +# - libc version 2.31 (https://packages.ubuntu.com/focal/libc6) +# +# CentOS Stream 9 EOL: 2027. https://www.centos.org/cl-vs-cs/#end-of-life +# +# - libgcc version 12.2.1 (https://mirror.stream.centos.org/9-stream/AppStream/x86_64/os/Packages/) +# - libc version 2.34 (https://mirror.stream.centos.org/9-stream/AppStream/x86_64/os/Packages/) +# +# See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html for more info. + +MAX_VERSIONS = { +'GCC': (7,0,0), +'GLIBC': { + lief.ELF.ARCH.X86_64: (2,31), + lief.ELF.ARCH.ARM: (2,31), + lief.ELF.ARCH.AARCH64:(2,31), + lief.ELF.ARCH.PPC64: (2,31), + lief.ELF.ARCH.RISCV: (2,31), +}, +'LIBATOMIC': (1,0), +'V': (0,5,0), # xkb (bitcoin-qt only) +} + +# Ignore symbols that are exported as part of every executable +IGNORE_EXPORTS = { +'environ', '_environ', '__environ', '_fini', '_init', 'stdin', +'stdout', 'stderr', +} + +# Expected linker-loader names can be found here: +# https://sourceware.org/glibc/wiki/ABIList?action=recall&rev=16 +ELF_INTERPRETER_NAMES: dict[lief.ELF.ARCH, dict[lief.Header.ENDIANNESS, str]] = { + lief.ELF.ARCH.X86_64: { + lief.Header.ENDIANNESS.LITTLE: "/lib64/ld-linux-x86-64.so.2", + }, + lief.ELF.ARCH.ARM: { + lief.Header.ENDIANNESS.LITTLE: "/lib/ld-linux-armhf.so.3", + }, + lief.ELF.ARCH.AARCH64: { + lief.Header.ENDIANNESS.LITTLE: "/lib/ld-linux-aarch64.so.1", + }, + lief.ELF.ARCH.PPC64: { + lief.Header.ENDIANNESS.BIG: "/lib64/ld64.so.1", + lief.Header.ENDIANNESS.LITTLE: "/lib64/ld64.so.2", + }, + lief.ELF.ARCH.RISCV: { + lief.Header.ENDIANNESS.LITTLE: "/lib/ld-linux-riscv64-lp64d.so.1", + }, +} + +ELF_ABIS: dict[lief.ELF.ARCH, dict[lief.Header.ENDIANNESS, list[int]]] = { + lief.ELF.ARCH.X86_64: { + lief.Header.ENDIANNESS.LITTLE: [3,2,0], + }, + lief.ELF.ARCH.ARM: { + lief.Header.ENDIANNESS.LITTLE: [3,2,0], + }, + lief.ELF.ARCH.AARCH64: { + lief.Header.ENDIANNESS.LITTLE: [3,7,0], + }, + lief.ELF.ARCH.PPC64: { + lief.Header.ENDIANNESS.LITTLE: [3,10,0], + lief.Header.ENDIANNESS.BIG: [3,2,0], + }, + lief.ELF.ARCH.RISCV: { + lief.Header.ENDIANNESS.LITTLE: [4,15,0], + }, +} + +# Allowed NEEDED libraries +ELF_ALLOWED_LIBRARIES = { +# bitcoind and bitcoin-qt +'libgcc_s.so.1', # GCC base support +'libc.so.6', # C library +'libpthread.so.0', # threading +'libm.so.6', # math library +'libatomic.so.1', +'ld-linux-x86-64.so.2', # 64-bit dynamic linker +'ld-linux.so.2', # 32-bit dynamic linker +'ld-linux-aarch64.so.1', # 64-bit ARM dynamic linker +'ld-linux-armhf.so.3', # 32-bit ARM dynamic linker +'ld64.so.1', # POWER64 ABIv1 dynamic linker +'ld64.so.2', # POWER64 ABIv2 dynamic linker +'ld-linux-riscv64-lp64d.so.1', # 64-bit RISC-V dynamic linker +# bitcoin-qt only +'libxcb.so.1', # part of X11 +'libxkbcommon.so.0', # keyboard keymapping +'libxkbcommon-x11.so.0', # keyboard keymapping +'libfontconfig.so.1', # font support +'libfreetype.so.6', # font parsing +'libdl.so.2', # programming interface to dynamic linker +'libxcb-icccm.so.4', +'libxcb-image.so.0', +'libxcb-shm.so.0', +'libxcb-keysyms.so.1', +'libxcb-randr.so.0', +'libxcb-render-util.so.0', +'libxcb-render.so.0', +'libxcb-shape.so.0', +'libxcb-sync.so.1', +'libxcb-xfixes.so.0', +'libxcb-xkb.so.1', +} + +MACHO_ALLOWED_LIBRARIES = { +# bitcoind and bitcoin-qt +'libc++.1.dylib', # C++ Standard Library +'libSystem.B.dylib', # libc, libm, libpthread, libinfo +# bitcoin-qt only +'AppKit', # user interface +'ApplicationServices', # common application tasks. +'Carbon', # deprecated c back-compat API +'ColorSync', +'CoreFoundation', # low level func, data types +'CoreGraphics', # 2D rendering +'CoreServices', # operating system services +'CoreText', # interface for laying out text and handling fonts. +'CoreVideo', # video processing +'Foundation', # base layer functionality for apps/frameworks +'ImageIO', # read and write image file formats. +'IOKit', # user-space access to hardware devices and drivers. +'IOSurface', # cross process image/drawing buffers +'libobjc.A.dylib', # Objective-C runtime library +'Metal', # 3D graphics +'QuartzCore', # animation +'Security', # access control and authentication +'UniformTypeIdentifiers', # collection of types that map to MIME and file types +} + +PE_ALLOWED_LIBRARIES = { +'ADVAPI32.dll', # legacy security & registry +'bcrypt.dll', # newer security and identity API +'IPHLPAPI.DLL', # IP helper API +'KERNEL32.dll', # win32 base APIs +'msvcrt.dll', # C standard library for MSVC +'SHELL32.dll', # shell API +'WS2_32.dll', # sockets +# bitcoin-qt only +'api-ms-win-core-synch-l1-2-0.dll', # Synchronization Primitives API +'api-ms-win-core-winrt-l1-1-0.dll', # Windows Runtime API +'api-ms-win-core-winrt-string-l1-1-0.dll', # WinRT String API +'AUTHZ.dll', # Windows Authorization Framework +'comdlg32.dll', # Common Dialog Box Library +'d3d11.dll', # Direct3D 11 API +'d3d12.dll', # Direct3D 12 API +'d3d9.dll', # Direct3D 9 API +'dwmapi.dll', # desktop window manager +'DWrite.dll', # DirectX Typography Services +'dxgi.dll', # DirectX Graphics Infrastructure +'GDI32.dll', # graphics device interface +'IMM32.dll', # input method editor +'NETAPI32.dll', # network management +'ole32.dll', # component object model +'OLEAUT32.dll', # OLE Automation API +'SHLWAPI.dll', # light weight shell API +'USER32.dll', # user interface +'USERENV.dll', # user management +'UxTheme.dll', # visual style +'VERSION.dll', # version checking +'WINMM.dll', # WinMM audio API +'WTSAPI32.dll', # Remote Desktop +'SETUPAPI.dll', # Windows Setup API +'SHCORE.dll', # Stream Handler Core +} + +def check_version(max_versions, version, arch) -> bool: + (lib, _, ver) = version.rpartition('_') + ver = tuple([int(x) for x in ver.split('.')]) + if not lib in max_versions: + return False + if isinstance(max_versions[lib], tuple): + return ver <= max_versions[lib] + else: + return ver <= max_versions[lib][arch] + +def check_imported_symbols(binary) -> bool: + ok: bool = True + + for symbol in binary.imported_symbols: + if not symbol.imported: + continue + + version = symbol.symbol_version if symbol.has_version else None + + if version: + aux_version = version.symbol_version_auxiliary.name if version.has_auxiliary_version else None + if aux_version and not check_version(MAX_VERSIONS, aux_version, binary.header.machine_type): + print(f'{filename}: symbol {symbol.name} from unsupported version {version}') + ok = False + return ok + +def check_exported_symbols(binary) -> bool: + ok: bool = True + + for symbol in binary.dynamic_symbols: + if not symbol.exported: + continue + name = symbol.name + if binary.header.machine_type == lief.ELF.ARCH.RISCV or name in IGNORE_EXPORTS: + continue + print(f'{filename}: export of symbol {name} not allowed!') + ok = False + return ok + +def check_RUNPATH(binary) -> bool: + assert binary.get(lief.ELF.DynamicEntry.TAG.RUNPATH) is None + assert binary.get(lief.ELF.DynamicEntry.TAG.RPATH) is None + return True + +def check_ELF_libraries(binary) -> bool: + ok: bool = True + for library in binary.libraries: + if library not in ELF_ALLOWED_LIBRARIES: + print(f'{filename}: {library} is not in ALLOWED_LIBRARIES!') + ok = False + return ok + +def check_MACHO_libraries(binary) -> bool: + ok: bool = True + for dylib in binary.libraries: + split = dylib.name.split('/') + if split[-1] not in MACHO_ALLOWED_LIBRARIES: + print(f'{split[-1]} is not in ALLOWED_LIBRARIES!') + ok = False + return ok + +def check_MACHO_min_os(binary) -> bool: + if binary.build_version.minos == [14,0,0]: + return True + return False + +def check_MACHO_sdk(binary) -> bool: + if binary.build_version.sdk == [14, 0, 0]: + return True + return False + +def check_MACHO_lld(binary) -> bool: + if binary.build_version.tools[0].version == [19, 1, 4]: + return True + return False + +def check_PE_libraries(binary) -> bool: + ok: bool = True + for dylib in binary.libraries: + if dylib not in PE_ALLOWED_LIBRARIES: + print(f'{dylib} is not in ALLOWED_LIBRARIES!') + ok = False + return ok + +def check_PE_subsystem_version(binary) -> bool: + major: int = binary.optional_header.major_subsystem_version + minor: int = binary.optional_header.minor_subsystem_version + if major == 6 and minor == 2: + return True + return False + +def check_PE_application_manifest(binary) -> bool: + if not binary.has_resources: + # No resources at all. + return False + + rm = binary.resources_manager + return rm.has_manifest + +def check_ELF_interpreter(binary) -> bool: + expected_interpreter = ELF_INTERPRETER_NAMES[binary.header.machine_type][binary.abstract.header.endianness] + + return binary.concrete.interpreter == expected_interpreter + +def check_ELF_ABI(binary) -> bool: + expected_abi = ELF_ABIS[binary.header.machine_type][binary.abstract.header.endianness] + note = binary.concrete.get(lief.ELF.Note.TYPE.GNU_ABI_TAG) + assert note.abi == lief.ELF.NoteAbi.ABI.LINUX + return note.version == expected_abi + +CHECKS = { +lief.Binary.FORMATS.ELF: [ + ('IMPORTED_SYMBOLS', check_imported_symbols), + ('EXPORTED_SYMBOLS', check_exported_symbols), + ('LIBRARY_DEPENDENCIES', check_ELF_libraries), + ('INTERPRETER_NAME', check_ELF_interpreter), + ('ABI', check_ELF_ABI), + ('RUNPATH', check_RUNPATH), +], +lief.Binary.FORMATS.MACHO: [ + ('DYNAMIC_LIBRARIES', check_MACHO_libraries), + ('MIN_OS', check_MACHO_min_os), + ('SDK', check_MACHO_sdk), + ('LLD', check_MACHO_lld), +], +lief.Binary.FORMATS.PE: [ + ('DYNAMIC_LIBRARIES', check_PE_libraries), + ('SUBSYSTEM_VERSION', check_PE_subsystem_version), + ('APPLICATION_MANIFEST', check_PE_application_manifest), +] +} + +if __name__ == '__main__': + retval: int = 0 + for filename in sys.argv[1:]: + binary = lief.parse(filename) + + etype = binary.format + + failed: list[str] = [] + for (name, func) in CHECKS[etype]: + if not func(binary): + failed.append(name) + if failed: + print(f'{filename}: failed {" ".join(failed)}') + retval = 1 + sys.exit(retval) From f0b7eb839745d80505fb8130fbeb05e103b5345b Mon Sep 17 00:00:00 2001 From: willcl-ark Date: Tue, 28 Oct 2025 22:23:37 +0000 Subject: [PATCH 04/48] modify bench-ci/guix params for benchmarking --- bench-ci/guix/guix-build | 4 ++-- bench-ci/guix/libexec/build.sh | 20 ++++++++++++++++---- bench-ci/guix/libexec/prelude.bash | 7 +++++++ 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/bench-ci/guix/guix-build b/bench-ci/guix/guix-build index ee285bf322cf..84d4f201b259 100755 --- a/bench-ci/guix/guix-build +++ b/bench-ci/guix/guix-build @@ -439,7 +439,7 @@ EOF # more information. # # shellcheck disable=SC2086,SC2031 - time-machine shell --manifest="${PWD}/contrib/guix/manifest.scm" \ + time-machine shell --manifest="${PWD}/bench-ci/guix/manifest.scm" \ --container \ --pure \ --no-cwd \ @@ -468,7 +468,7 @@ EOF DISTSRC="$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")" \ OUTDIR="$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST")" \ DIST_ARCHIVE_BASE=/outdir-base/dist-archive \ - bash -c "cd /bitcoin && bash contrib/guix/libexec/build.sh" + bash -c "cd /bitcoin && bash bench-ci/guix/libexec/build.sh" ) done diff --git a/bench-ci/guix/libexec/build.sh b/bench-ci/guix/libexec/build.sh index 16e12d563816..d12c795beaa4 100755 --- a/bench-ci/guix/libexec/build.sh +++ b/bench-ci/guix/libexec/build.sh @@ -72,6 +72,8 @@ unset OBJCPLUS_INCLUDE_PATH # Set native toolchain build_CC="${NATIVE_GCC}/bin/gcc -isystem ${NATIVE_GCC}/include" build_CXX="${NATIVE_GCC}/bin/g++ -isystem ${NATIVE_GCC}/include/c++ -isystem ${NATIVE_GCC}/include" +export C_INCLUDE_PATH="${NATIVE_GCC}/include" +export CPLUS_INCLUDE_PATH="${NATIVE_GCC}/include/c++:${NATIVE_GCC}/include" case "$HOST" in *darwin*) export LIBRARY_PATH="${NATIVE_GCC}/lib" ;; # Required for native packages @@ -178,7 +180,13 @@ make -C depends --jobs="$JOBS" HOST="$HOST" \ x86_64_linux_AR=x86_64-linux-gnu-gcc-ar \ x86_64_linux_RANLIB=x86_64-linux-gnu-gcc-ranlib \ x86_64_linux_NM=x86_64-linux-gnu-gcc-nm \ - x86_64_linux_STRIP=x86_64-linux-gnu-strip + x86_64_linux_STRIP=x86_64-linux-gnu-strip \ + NO_QT=1 \ + NO_QR=1 \ + NO_ZMQ=1 \ + NO_WALLET=1 \ + NO_BDB=1 \ + NO_USDT=1 case "$HOST" in *darwin*) @@ -208,6 +216,9 @@ mkdir -p "$OUTDIR" # CONFIGFLAGS CONFIGFLAGS="-DREDUCE_EXPORTS=ON -DBUILD_BENCH=OFF -DBUILD_GUI_TESTS=OFF -DBUILD_FUZZ_BINARY=OFF -DCMAKE_SKIP_RPATH=TRUE" +# BENCHCOINFLAGS +BENCHCOINFLAGS="-DBUILD_CLI=OFF -DBUILD_TESTS=OFF -DCMAKE_CXX_FLAGS=-fno-omit-frame-pointer" + # CFLAGS HOST_CFLAGS="-O2 -g" HOST_CFLAGS+=$(find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;) @@ -243,15 +254,16 @@ mkdir -p "$DISTSRC" --toolchain "${BASEPREFIX}/${HOST}/toolchain.cmake" \ -DWITH_CCACHE=OFF \ -Werror=dev \ - ${CONFIGFLAGS} + ${CONFIGFLAGS} \ + ${BENCHCOINFLAGS} # Build Bitcoin Core cmake --build build -j "$JOBS" ${V:+--verbose} # Perform basic security checks on a series of executables. - cmake --build build -j 1 --target check-security ${V:+--verbose} + # cmake --build build -j 1 --target check-security ${V:+--verbose} # Check that executables only contain allowed version symbols. - cmake --build build -j 1 --target check-symbols ${V:+--verbose} + # cmake --build build -j 1 --target check-symbols ${V:+--verbose} mkdir -p "$OUTDIR" diff --git a/bench-ci/guix/libexec/prelude.bash b/bench-ci/guix/libexec/prelude.bash index b7c13cc91d8c..b4e828013351 100644 --- a/bench-ci/guix/libexec/prelude.bash +++ b/bench-ci/guix/libexec/prelude.bash @@ -8,6 +8,13 @@ source contrib/shell/realpath.bash # shellcheck source=contrib/shell/git-utils.bash source contrib/shell/git-utils.bash +# Source guix profile from the runner home directory +GUIX_PROFILE=/home/github-runner/.config/guix/current +. "$GUIX_PROFILE/etc/profile" +echo "Using the following guix command:" +command -v guix +guix describe + ################ # Required non-builtin commands should be invocable ################ From 8078cb82d3538819e416827dbf2a21298a67bcdd Mon Sep 17 00:00:00 2001 From: willcl-ark Date: Tue, 28 Oct 2025 22:24:58 +0000 Subject: [PATCH 05/48] bench: add benchmark ci workflows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: David Gumberg Co-authored-by: Lőrinc --- .github/workflows/benchmark.yml | 113 ++++++++++ .github/workflows/publish-results.yml | 298 ++++++++++++++++++++++++++ .github/workflows/sync_upstream.yml | 32 +++ bench-ci/build_binaries.sh | 52 +++++ bench-ci/parse_and_plot.py | 167 +++++++++++++++ bench-ci/run-assumeutxo-bench.sh | 154 +++++++++++++ bench-ci/run-benchmark.sh | 163 ++++++++++++++ 7 files changed, 979 insertions(+) create mode 100644 .github/workflows/benchmark.yml create mode 100644 .github/workflows/publish-results.yml create mode 100644 .github/workflows/sync_upstream.yml create mode 100755 bench-ci/build_binaries.sh create mode 100755 bench-ci/parse_and_plot.py create mode 100755 bench-ci/run-assumeutxo-bench.sh create mode 100755 bench-ci/run-benchmark.sh diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 000000000000..c8c1a992fa25 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,113 @@ +name: Benchmark +on: + pull_request: + branches: + - master +jobs: + build-binaries: + runs-on: [self-hosted, linux, x64] + env: + NIX_PATH: nixpkgs=channel:nixos-unstable + BASE_SHA: ${{ github.event.pull_request.base.sha }} + steps: + - name: Checkout repo + uses: actions/checkout@v4 + with: + fetch-depth: 1 + - name: Fetch base commit + run: | + echo "CHECKOUT_COMMIT=$(git rev-parse HEAD)" >> "$GITHUB_ENV" + git fetch --depth=1 origin ${{ github.event.pull_request.base.sha }} + - name: Setup ccache + run: | + mkdir -p /data/ccache + export CCACHE_DIR=/data/ccache + export CCACHE_MAXSIZE=50G + ccache -M 50G + ccache -s + - name: Build both binaries + env: + CCACHE_DIR: /data/ccache + run: | + mkdir -p ${{ runner.temp }}/binaries/base + mkdir -p ${{ runner.temp }}/binaries/head + nix-shell --command "just build-assumeutxo-binaries-guix $BASE_SHA $CHECKOUT_COMMIT" + cp binaries/base/bitcoind ${{ runner.temp }}/binaries/base/bitcoind + cp binaries/head/bitcoind ${{ runner.temp }}/binaries/head/bitcoind + - name: Upload binaries + uses: actions/upload-artifact@v4 + with: + name: bitcoind-binaries + path: ${{ runner.temp }}/binaries/ + assumeutxo: + needs: build-binaries + strategy: + matrix: + include: + - network: mainnet + name: mainnet-default + timeout: 600 + datadir_path: /data/pruned-840k + dbcache: 450 + - network: mainnet + name: mainnet-large + timeout: 600 + datadir_path: /data/pruned-840k + dbcache: 32000 + runs-on: [self-hosted, linux, x64] + timeout-minutes: ${{ matrix.timeout }} + env: + NIX_PATH: nixpkgs=channel:nixos-unstable + ORIGINAL_DATADIR: ${{ matrix.datadir_path }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + steps: + - name: Checkout repo + uses: actions/checkout@v4 + with: + fetch-depth: 1 + - name: Download binaries + uses: actions/download-artifact@v4 + with: + name: bitcoind-binaries + path: ${{ runner.temp }}/binaries + - name: Set binary permissions + run: | + chmod +x ${{ runner.temp }}/binaries/base/bitcoind + chmod +x ${{ runner.temp }}/binaries/head/bitcoind + - name: Fetch base commit + run: | + echo "CHECKOUT_COMMIT=$(git rev-parse HEAD)" >> "$GITHUB_ENV" + git fetch --depth=1 origin ${{ github.event.pull_request.base.sha }} + - name: Run AssumeUTXO ${{ matrix.network }} + env: + TMP_DATADIR: "${{ runner.temp }}/base_datadir" + BINARIES_DIR: "${{ runner.temp }}/binaries" + run: | + env + mkdir -p "$TMP_DATADIR" + nix-shell --command "just run-${{ matrix.network }}-ci $BASE_SHA $CHECKOUT_COMMIT $TMP_DATADIR $ORIGINAL_DATADIR ${{ runner.temp }}/results.json ${{ matrix.dbcache }} ${{ runner.temp }}/pngs $BINARIES_DIR" + - uses: actions/upload-artifact@v4 + with: + name: result-${{ matrix.name }} + path: "${{ runner.temp }}/results.json" + - uses: actions/upload-artifact@v4 + with: + name: pngs-${{ matrix.name }} + path: "${{ runner.temp }}/pngs/*.png" + - uses: actions/upload-artifact@v4 + with: + name: flamegraph-${{ matrix.name }} + path: "**/*-flamegraph.svg" + - name: Write GitHub and runner context files + env: + GITHUB_CONTEXT: ${{ toJSON(github) }} + RUNNER_CONTEXT: ${{ toJSON(runner) }} + run: | + mkdir contexts + echo "$GITHUB_CONTEXT" | nix-shell -p jq --command "jq 'del(.token)' > contexts/github.json" + echo "$RUNNER_CONTEXT" > contexts/runner.json + - name: Upload context metadata as artifact + uses: actions/upload-artifact@v4 + with: + name: run-metadata-${{ matrix.name }} + path: ./contexts/ diff --git a/.github/workflows/publish-results.yml b/.github/workflows/publish-results.yml new file mode 100644 index 000000000000..62076d85b5eb --- /dev/null +++ b/.github/workflows/publish-results.yml @@ -0,0 +1,298 @@ +name: Publish Results +on: + workflow_run: + workflows: ["Benchmark"] + types: [completed] +jobs: + build: + runs-on: ubuntu-latest + if: ${{ github.event.workflow_run.conclusion == 'success' }} + permissions: + actions: read + contents: write + checks: read + env: + NETWORKS: "mainnet-default,mainnet-large" + outputs: + speedups: ${{ steps.organize.outputs.speedups }} + pr-number: ${{ steps.organize.outputs.pr-number }} + steps: + - uses: actions/checkout@v4 + with: + ref: gh-pages + - name: Download artifacts + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh run download ${{ github.event.workflow_run.id }} --repo ${{ github.repository }} + + - name: Extract artifacts + run: | + for network in ${NETWORKS//,/ }; do + if [ -d "result-${network}" ]; then + mkdir -p "${network}-results" + mv "result-${network}/results.json" "${network}-results/" + fi + + if [ -d "flamegraph-${network}" ]; then + mkdir -p "${network}-flamegraph" + mv "flamegraph-${network}"/* "${network}-flamegraph/" + fi + + if [ -d "run-metadata-${network}" ]; then + mkdir -p "${network}-metadata" + mv "run-metadata-${network}"/* "${network}-metadata/" + fi + + if [ -d "pngs-${network}" ]; then + mkdir -p "${network}-plots" + mv "pngs-${network}"/*.png "${network}-plots/" + fi + done + - name: Organize results + id: organize + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const path = require('path'); + const networks = process.env.NETWORKS.split(','); + let prNumber = 'main'; + let runId; + + // First, extract metadata and get PR number + for (const network of networks) { + if (fs.existsSync(`${network}-metadata/github.json`)) { + const metadata = JSON.parse(fs.readFileSync(`${network}-metadata/github.json`, 'utf8')); + prNumber = metadata.event.pull_request?.number || prNumber; + runId = metadata.run_id; + } + } + + if (!runId) { + console.error('No valid metadata found for any network'); + process.exit(1); + } + + // Create directory structure + const resultDir = `results/pr-${prNumber}/${runId}`; + fs.mkdirSync(resultDir, { recursive: true }); + + // Now copy metadata files + for (const network of networks) { + if (fs.existsSync(`${network}-metadata/github.json`)) { + const metadataDir = `${resultDir}/${network}-metadata`; + fs.mkdirSync(metadataDir, { recursive: true }); + fs.copyFileSync(`${network}-metadata/github.json`, `${metadataDir}/github.json`); + } + } + + // Process each network's results + const combinedResults = { + results: [], + speedups: {} + }; + + for (const network of networks) { + if (fs.existsSync(`${network}-results`)) { + const networkResults = JSON.parse(fs.readFileSync(`${network}-results/results.json`, 'utf8')); + let baseMean, headMean; + + // Add network name to each result and collect means + networkResults.results.forEach(result => { + result.network = network; + combinedResults.results.push(result); + if (result.command.includes('base')) { + baseMean = result.mean; + } else if (result.command.includes('head')) { + headMean = result.mean; + } + }); + + // Calculate speedup if we have both measurements + if (baseMean && headMean) { + const speedup = baseMean > 0 ? ((baseMean - headMean) / baseMean * 100).toFixed(1) : 'N/A'; + combinedResults.speedups[network] = speedup; + } + + // Move flamegraphs + if (fs.existsSync(`${network}-flamegraph`)) { + fs.readdirSync(`${network}-flamegraph`).forEach(file => { + const sourceFile = `${network}-flamegraph/${file}`; + const targetFile = `${resultDir}/${network}-${file}`; + fs.copyFileSync(sourceFile, targetFile); + }); + } + + // Move plots + if (fs.existsSync(`${network}-plots`)) { + const targetPlotsDir = `${resultDir}/${network}-plots`; + fs.mkdirSync(targetPlotsDir, { recursive: true }); + fs.readdirSync(`${network}-plots`).forEach(plot => { + const sourcePlot = `${network}-plots/${plot}`; + const targetPlot = `${targetPlotsDir}/${plot}`; + fs.copyFileSync(sourcePlot, targetPlot); + }); + } + } + } + + // Write combined results + fs.writeFileSync(`${resultDir}/results.json`, JSON.stringify(combinedResults, null, 2)); + + // Create index.html for this run + const indexHtml = ` + + + Benchmark Results + + + +
+

Benchmark Results

+
+

PR #${prNumber} - Run ${runId}

+ ${networks.map(network => ` +
+

+ ${network} Results + ${combinedResults.speedups[network] ? + `(${combinedResults.speedups[network]}% speedup)` + : ''} +

+
+ ${combinedResults.results + .filter(result => result.network === network) + .map(result => { + const commitShortId = result.parameters.commit.slice(0, 8); + const flameGraphFile = `${network}-${result.parameters.commit}-flamegraph.svg`; + const flameGraphPath = `${resultDir}/${network}-${result.parameters.commit}-flamegraph.svg`; + + // Query PNG files dynamically + const plotDir = `${resultDir}/${network}-plots`; + const plots = fs.existsSync(plotDir) + ? fs.readdirSync(plotDir) + .map(plot => ` + + ${plot} + + `) + .join('') + : ''; + + return ` + + + + + + + + + + + + + + + + + + + +
CommandMean (s)Std DevUser (s)System (s)
+ ${result.command.replace( + /\((\w+)\)/, + (_, commit) => `(${commit.slice(0, 8)})` + )} + ${result.mean.toFixed(3)}${result.stddev?.toFixed(3) || 'N/A'}${result.user.toFixed(3)}${result.system.toFixed(3)}
+ ${fs.existsSync(flameGraphPath) ? ` + + ` : ''} + ${plots} + `; + }).join('')} +
+
+ `).join('')} +
+
+ + `; + + fs.writeFileSync(`${resultDir}/index.html`, indexHtml); + + // Update main index.html + const prs = fs.readdirSync('results') + .filter(dir => dir.startsWith('pr-')) + .map(dir => ({ + pr: dir.replace('pr-', ''), + runs: fs.readdirSync(`results/${dir}`) + })); + + const mainIndexHtml = ` + + + Bitcoin Benchmark Results + + + +
+

Bitcoin Benchmark Results

+
+

Available Results

+
    + ${prs.map(({pr, runs}) => ` +
  • PR #${pr} +
      + ${runs.map(run => ` +
    • Run ${run}
      • + `).join('')} +
    +
    • + `).join('')} +
+
+
+ + `; + + fs.writeFileSync('index.html', mainIndexHtml); + + // Set outputs for use in PR comment + const resultUrl = `https://${context.repo.owner}.github.io/${context.repo.name}/results/pr-${prNumber}/${runId}/index.html`; + const speedupString = Object.entries(combinedResults.speedups) + .map(([network, speedup]) => `${network}: ${speedup}%`) + .join(', '); + + core.setOutput('result-url', resultUrl); + core.setOutput('speedups', speedupString); + core.setOutput('pr-number', prNumber); + return { url: resultUrl, speedups: speedupString }; + - name: Upload Pages artifact + uses: actions/upload-pages-artifact@v3 + with: + path: results + - name: Commit and push to gh-pages + run: | + git config --global user.name "github-actions[bot]" + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git add results/ index.html + git commit -m "Update benchmark results from run ${{ github.event.workflow_run.id }}" + git push origin gh-pages + comment-pr: + needs: build + runs-on: ubuntu-latest + permissions: + pull-requests: write + actions: read + steps: + - name: Comment on PR + if: ${{ needs.build.outputs.pr-number != 'main' }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh pr comment ${{ needs.build.outputs.pr-number }} \ + --repo ${{ github.repository }} \ + --body "📊 Benchmark results for this run (${{ github.event.workflow_run.id }}) will be available at: https://${{ github.repository_owner }}.github.io/${{ github.event.repository.name }}/results/pr-${{ needs.build.outputs.pr-number }}/${{ github.event.workflow_run.id }}/index.html after the github pages \"build and deployment\" action has completed. + 🚀 Speedups: ${{ needs.build.outputs.speedups }}" diff --git a/.github/workflows/sync_upstream.yml b/.github/workflows/sync_upstream.yml new file mode 100644 index 000000000000..11d9cfafc2e4 --- /dev/null +++ b/.github/workflows/sync_upstream.yml @@ -0,0 +1,32 @@ +name: Sync with Upstream +on: + schedule: + - cron: '0 3 * * *' # 03:00 UTC daily + workflow_dispatch: +permissions: + contents: write # Required for pushing to master +jobs: + sync: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} + - name: Add upstream remote + run: | + git remote add upstream https://github.com/bitcoin/bitcoin.git + git remote -v + - name: Fetch upstream + run: git fetch upstream + - name: Configure Git + run: | + git config user.name github-actions + git config user.email github-actions@github.com + - name: Rebase onto upstream + run: | + git checkout master + git rebase upstream/master + - name: Push changes + run: git push --force-with-lease origin master diff --git a/bench-ci/build_binaries.sh b/bench-ci/build_binaries.sh new file mode 100755 index 000000000000..9a396a00659f --- /dev/null +++ b/bench-ci/build_binaries.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +set -euxo pipefail + +if [ $# -ne 2 ]; then + echo "Usage: $0 " + exit 1 +fi + +# Save current state of git +initial_ref=$(git symbolic-ref -q HEAD || git rev-parse HEAD) +if git symbolic-ref -q HEAD >/dev/null; then + initial_state="branch" + initial_branch=${initial_ref#refs/heads/} +else + initial_state="detached" +fi + +base_commit="$1" +head_commit="$2" + +mkdir -p binaries/base +mkdir -p binaries/head + +for build in "base:${base_commit}" "head:${head_commit}"; do + name="${build%%:*}" + commit="${build#*:}" + git checkout "$commit" + # Use environment variables if set, otherwise use defaults + HOSTS="${HOSTS:-x86_64-linux-gnu}" \ + SOURCES_PATH="${SOURCES_PATH:-/data/SOURCES_PATH}" \ + BASE_CACHE="${BASE_CACHE:-/data/BASE_CACHE}" \ + taskset -c 2-15 chrt -f 1 bench-ci/guix/guix-build + + # Truncate commit hash to 12 characters + short_commit=$(echo "$commit" | cut -c 1-12) + + # Extract the Guix output + tar -xzf "guix-build-${short_commit}/output/x86_64-linux-gnu/bitcoin-${short_commit}-x86_64-linux-gnu.tar.gz" + + # Copy the binary to our binaries directory + cp "bitcoin-${short_commit}/bin/bitcoind" "binaries/${name}/bitcoind" + + # Cleanup extracted files + rm -rf "bitcoin-${short_commit}" +done + +# Restore initial git state +if [ "$initial_state" = "branch" ]; then + git checkout "$initial_branch" +else + git checkout "$initial_ref" +fi diff --git a/bench-ci/parse_and_plot.py b/bench-ci/parse_and_plot.py new file mode 100755 index 000000000000..db577417b2ff --- /dev/null +++ b/bench-ci/parse_and_plot.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +import sys +import os +import re +import datetime +import matplotlib.pyplot as plt + + +def parse_updatetip_line(line): + match = re.match( + r'^([\d\-:TZ]+) UpdateTip: new best.+height=(\d+).+tx=(\d+).+cache=([\d.]+)MiB\((\d+)txo\)', + line + ) + if not match: + return None + iso_str, height_str, tx_str, cache_size_mb_str, cache_coins_count_str = match.groups() + parsed_datetime = datetime.datetime.strptime(iso_str, "%Y-%m-%dT%H:%M:%SZ") + return parsed_datetime, int(height_str), int(tx_str), float(cache_size_mb_str), int(cache_coins_count_str) + + +def parse_leveldb_compact_line(line): + match = re.match(r'^([\d\-:TZ]+) \[leveldb] Compacting.*files', line) + if not match: + return None + iso_str = match.groups()[0] + parsed_datetime = datetime.datetime.strptime(iso_str, "%Y-%m-%dT%H:%M:%SZ") + return parsed_datetime + + +def parse_leveldb_generated_table_line(line): + match = re.match(r'^([\d\-:TZ]+) \[leveldb] Generated table.*: (\d+) keys, (\d+) bytes', line) + if not match: + return None + iso_str, keys_count_str, bytes_count_str = match.groups() + parsed_datetime = datetime.datetime.strptime(iso_str, "%Y-%m-%dT%H:%M:%SZ") + return parsed_datetime, int(keys_count_str), int(bytes_count_str) + +def parse_validation_txadd_line(line): + match = re.match(r'^([\d\-:TZ]+) \[validation] TransactionAddedToMempool: txid=.+wtxid=.+', line) + if not match: + return None + iso_str = match.groups()[0] + parsed_datetime = datetime.datetime.strptime(iso_str, "%Y-%m-%dT%H:%M:%SZ") + return parsed_datetime + + +def parse_coindb_write_batch_line(line): + match = re.match(r'^([\d\-:TZ]+) \[coindb] Writing (partial|final) batch of ([\d.]+) MiB', line) + if not match: + return None + iso_str, is_partial_str, size_mb_str = match.groups() + parsed_datetime = datetime.datetime.strptime(iso_str, "%Y-%m-%dT%H:%M:%SZ") + return parsed_datetime, is_partial_str, float(size_mb_str) + + +def parse_coindb_commit_line(line): + match = re.match(r'^([\d\-:TZ]+) \[coindb] Committed (\d+) changed transaction outputs', line) + if not match: + return None + iso_str, txout_count_str = match.groups() + parsed_datetime = datetime.datetime.strptime(iso_str, "%Y-%m-%dT%H:%M:%SZ") + return parsed_datetime, int(txout_count_str) + +def parse_log_file(log_file): + with open(log_file, 'r', encoding='utf-8') as f: + update_tip_data = [] + leveldb_compact_data = [] + leveldb_gen_table_data = [] + validation_txadd_data = [] + coindb_write_batch_data = [] + coindb_commit_data = [] + + for line in f: + if result := parse_updatetip_line(line): + update_tip_data.append(result) + elif result := parse_leveldb_compact_line(line): + leveldb_compact_data.append(result) + elif result := parse_leveldb_generated_table_line(line): + leveldb_gen_table_data.append(result) + elif result := parse_validation_txadd_line(line): + validation_txadd_data.append(result) + elif result := parse_coindb_write_batch_line(line): + coindb_write_batch_data.append(result) + elif result := parse_coindb_commit_line(line): + coindb_commit_data.append(result) + + if not update_tip_data: + print("No UpdateTip entries found.") + sys.exit(0) + + assert all(update_tip_data[i][0] <= update_tip_data[i + 1][0] for i in + range(len(update_tip_data) - 1)), "UpdateTip entries are not sorted by time" + + return update_tip_data, leveldb_compact_data, leveldb_gen_table_data, validation_txadd_data, coindb_write_batch_data, coindb_commit_data + + +def generate_plot(x, y, x_label, y_label, title, output_file): + if not x or not y: + print(f"Skipping plot '{title}' as there is no data.") + return + + plt.figure(figsize=(30, 10)) + plt.plot(x, y) + plt.title(title, fontsize=20) + plt.xlabel(x_label, fontsize=16) + plt.ylabel(y_label, fontsize=16) + plt.grid(True) + plt.xticks(rotation=90, fontsize=12) + plt.yticks(fontsize=12) + plt.tight_layout() + plt.savefig(output_file) + plt.close() + print(f"Saved plot to {output_file}") + + +if __name__ == "__main__": + if len(sys.argv) != 3: + print(f"Usage: {sys.argv[0]} ") + sys.exit(1) + + log_file = sys.argv[1] + if not os.path.isfile(log_file): + print(f"File not found: {log_file}") + sys.exit(1) + + png_dir = sys.argv[2] + os.makedirs(png_dir, exist_ok=True) + + update_tip_data, leveldb_compact_data, leveldb_gen_table_data, validation_txadd_data, coindb_write_batch_data, coindb_commit_data = parse_log_file(log_file) + times, heights, tx_counts, cache_size, cache_count = zip(*update_tip_data) + float_minutes = [(t - times[0]).total_seconds() / 60 for t in times] + + generate_plot(float_minutes, heights, "Elapsed minutes", "Block Height", "Block Height vs Time", os.path.join(png_dir, "height_vs_time.png")) + generate_plot(heights, cache_size, "Block Height", "Cache Size (MiB)", "Cache Size vs Block Height", os.path.join(png_dir, "cache_vs_height.png")) + generate_plot(float_minutes, cache_size, "Elapsed minutes", "Cache Size (MiB)", "Cache Size vs Time", os.path.join(png_dir, "cache_vs_time.png")) + generate_plot(heights, tx_counts, "Block Height", "Transaction Count", "Transactions vs Block Height", os.path.join(png_dir, "tx_vs_height.png")) + generate_plot(times, cache_count, "Block Height", "Coins Cache Size", "Coins Cache Size vs Time", os.path.join(png_dir, "coins_cache_vs_time.png")) + + # LevelDB Compaction and Generated Tables + if leveldb_compact_data: + leveldb_compact_times = [(t - times[0]).total_seconds() / 60 for t in leveldb_compact_data] + leveldb_compact_y = [1 for _ in leveldb_compact_times] # dummy y axis to mark compactions + generate_plot(leveldb_compact_times, leveldb_compact_y, "Elapsed minutes", "LevelDB Compaction", "LevelDB Compaction Events vs Time", os.path.join(png_dir, "leveldb_compact_vs_time.png")) + if leveldb_gen_table_data: + leveldb_gen_table_times, leveldb_gen_table_keys, leveldb_gen_table_bytes = zip(*leveldb_gen_table_data) + leveldb_gen_table_float_minutes = [(t - times[0]).total_seconds() / 60 for t in leveldb_gen_table_times] + generate_plot(leveldb_gen_table_float_minutes, leveldb_gen_table_keys, "Elapsed minutes", "Number of keys", "LevelDB Keys Generated vs Time", os.path.join(png_dir, "leveldb_gen_keys_vs_time.png")) + generate_plot(leveldb_gen_table_float_minutes, leveldb_gen_table_bytes, "Elapsed minutes", "Number of bytes", "LevelDB Bytes Generated vs Time", os.path.join(png_dir, "leveldb_gen_bytes_vs_time.png")) + + # validation mempool add transaction lines + if validation_txadd_data: + validation_txadd_times = [(t - times[0]).total_seconds() / 60 for t in validation_txadd_data] + validation_txadd_y = [1 for _ in validation_txadd_times] # dummy y axis to mark transaction additions + generate_plot(validation_txadd_times, validation_txadd_y, "Elapsed minutes", "Transaction Additions", "Transaction Additions to Mempool vs Time", os.path.join(png_dir, "validation_txadd_vs_time.png")) + + # coindb write batch lines + if coindb_write_batch_data: + coindb_write_batch_times, is_partial_strs, sizes_mb = zip(*coindb_write_batch_data) + coindb_write_batch_float_minutes = [(t - times[0]).total_seconds() / 60 for t in coindb_write_batch_times] + generate_plot(coindb_write_batch_float_minutes, sizes_mb, "Elapsed minutes", "Batch Size MiB", "Coin Database Partial/Final Write Batch Size vs Time", os.path.join(png_dir, "coindb_write_batch_size_vs_time.png")) + if coindb_commit_data: + coindb_commit_times, txout_counts = zip(*coindb_commit_data) + coindb_commit_float_minutes = [(t - times[0]).total_seconds() / 60 for t in coindb_commit_times] + generate_plot(coindb_commit_float_minutes, txout_counts, "Elapsed minutes", "Transaction Output Count", "Coin Database Transaction Output Committed vs Time", os.path.join(png_dir, "coindb_commit_txout_vs_time.png")) + + + print("Plots saved!") \ No newline at end of file diff --git a/bench-ci/run-assumeutxo-bench.sh b/bench-ci/run-assumeutxo-bench.sh new file mode 100755 index 000000000000..a1ee910ed428 --- /dev/null +++ b/bench-ci/run-assumeutxo-bench.sh @@ -0,0 +1,154 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +# Helper function to check and clean datadir +clean_datadir() { + set -euxo pipefail + + local TMP_DATADIR="$1" + + # Create the directory if it doesn't exist + mkdir -p "${TMP_DATADIR}" + + # If we're in CI, clean without confirmation + if [ -n "${CI:-}" ]; then + rm -Rf "${TMP_DATADIR:?}"/* + else + read -rp "Are you sure you want to delete everything in ${TMP_DATADIR}? [y/N] " response + if [[ "$response" =~ ^[Yy]$ ]]; then + rm -Rf "${TMP_DATADIR:?}"/* + else + echo "Aborting..." + exit 1 + fi + fi +} + +# Helper function to clear logs +clean_logs() { + set -euxo pipefail + + local TMP_DATADIR="$1" + local logfile="${TMP_DATADIR}/debug.log" + + echo "Checking for ${logfile}" + if [ -e "${logfile}" ]; then + echo "Removing ${logfile}" + rm "${logfile}" + fi +} + +# Execute CMD before each set of timing runs. +setup_assumeutxo_snapshot_run() { + set -euxo pipefail + + local TMP_DATADIR="$1" + local commit="$2" + clean_datadir "${TMP_DATADIR}" +} + +# Execute CMD before each timing run. +prepare_assumeutxo_snapshot_run() { + set -euxo pipefail + + local TMP_DATADIR="$1" + local UTXO_PATH="$2" + local CONNECT_ADDRESS="$3" + local CHAIN="$4" + local DBCACHE="$5" + local commit="$6" + local BINARIES_DIR="$7" + + # Run the actual preparation steps + clean_datadir "${TMP_DATADIR}" + # Use the pre-built binaries from BINARIES_DIR + "${BINARIES_DIR}/${commit}/bitcoind" --help + taskset -c 0-15 "${BINARIES_DIR}/${commit}/bitcoind" -datadir="${TMP_DATADIR}" -connect="${CONNECT_ADDRESS}" -daemon=0 -chain="${CHAIN}" -stopatheight=1 -printtoconsole=0 + taskset -c 0-15 "${BINARIES_DIR}/${commit}/bitcoind" -datadir="${TMP_DATADIR}" -connect="${CONNECT_ADDRESS}" -daemon=0 -chain="${CHAIN}" -dbcache="${DBCACHE}" -pausebackgroundsync=1 -loadutxosnapshot="${UTXO_PATH}" -printtoconsole=0 || true + clean_logs "${TMP_DATADIR}" +} + +# Executed after each timing run +conclude_assumeutxo_snapshot_run() { + set -euxo pipefail + + local commit="$1" + local TMP_DATADIR="$2" + local PNG_DIR="$3" + + # Search in subdirs e.g. $datadir/signet + debug_log=$(find "${TMP_DATADIR}" -name debug.log -print -quit) + if [ -n "${debug_log}" ]; then + echo "Generating plots from ${debug_log}" + if [ -x "bench-ci/parse_and_plot.py" ]; then + bench-ci/parse_and_plot.py "${debug_log}" "${PNG_DIR}" + else + ls -al "bench-ci/" + echo "parse_and_plot.py not found or not executable, skipping plot generation" + fi + else + ls -al "${TMP_DATADIR}/" + echo "debug.log not found, skipping plot generation" + fi + + # Move flamegraph if exists + if [ -e flamegraph.svg ]; then + mv flamegraph.svg "${commit}"-flamegraph.svg + fi +} + +# Execute CMD after the completion of all benchmarking runs for each individual +# command to be benchmarked. +cleanup_assumeutxo_snapshot_run() { + set -euxo pipefail + + local TMP_DATADIR="$1" + + # Clean up the datadir + clean_datadir "${TMP_DATADIR}" +} + +run_benchmark() { + local base_commit="$1" + local head_commit="$2" + local TMP_DATADIR="$3" + local UTXO_PATH="$4" + local results_file="$5" + local png_dir="$6" + local chain="$7" + local stop_at_height="$8" + local connect_address="$9" + local dbcache="${10}" + local BINARIES_DIR="${11}" + + # Export functions so they can be used by hyperfine + export -f setup_assumeutxo_snapshot_run + export -f prepare_assumeutxo_snapshot_run + export -f conclude_assumeutxo_snapshot_run + export -f cleanup_assumeutxo_snapshot_run + export -f clean_datadir + export -f clean_logs + + # Run hyperfine + hyperfine \ + --shell=bash \ + --setup "setup_assumeutxo_snapshot_run ${TMP_DATADIR} {commit}" \ + --prepare "prepare_assumeutxo_snapshot_run ${TMP_DATADIR} ${UTXO_PATH} ${connect_address} ${chain} ${dbcache} {commit} ${BINARIES_DIR}" \ + --conclude "conclude_assumeutxo_snapshot_run {commit} ${TMP_DATADIR} ${png_dir}" \ + --cleanup "cleanup_assumeutxo_snapshot_run ${TMP_DATADIR}" \ + --runs 1 \ + --export-json "${results_file}" \ + --command-name "base (${base_commit})" \ + --command-name "head (${head_commit})" \ + "taskset -c 1 flamegraph --palette bitcoin --title 'bitcoind assumeutxo IBD@{commit}' -c 'record -F 101 --call-graph fp' -- taskset -c 2-15 ${BINARIES_DIR}/{commit}/bitcoind -datadir=${TMP_DATADIR} -connect=${connect_address} -daemon=0 -chain=${chain} -stopatheight=${stop_at_height} -dbcache=${dbcache} -printtoconsole=0 -debug=coindb -debug=leveldb -debug=bench -debug=validation" \ + -L commit "base,head" +} + +# Main execution +if [ "$#" -ne 11 ]; then + echo "Usage: $0 base_commit head_commit TMP_DATADIR UTXO_PATH results_dir png_dir chain stop_at_height connect_address dbcache BINARIES_DIR" + exit 1 +fi + +run_benchmark "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" "$9" "${10}" "${11}" diff --git a/bench-ci/run-benchmark.sh b/bench-ci/run-benchmark.sh new file mode 100755 index 000000000000..dc190a5fa207 --- /dev/null +++ b/bench-ci/run-benchmark.sh @@ -0,0 +1,163 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +# Helper function to check and clean datadir +clean_datadir() { + set -euxo pipefail + + local TMP_DATADIR="$1" + + # Create the directory if it doesn't exist + mkdir -p "${TMP_DATADIR}" + + # If we're in CI, clean without confirmation + if [ -n "${CI:-}" ]; then + rm -Rf "${TMP_DATADIR:?}"/* + else + read -rp "Are you sure you want to delete everything in ${TMP_DATADIR}? [y/N] " response + if [[ "$response" =~ ^[Yy]$ ]]; then + rm -Rf "${TMP_DATADIR:?}"/* + else + echo "Aborting..." + exit 1 + fi + fi +} + +# Helper function to clear logs +clean_logs() { + set -euxo pipefail + + local TMP_DATADIR="$1" + local logfile="${TMP_DATADIR}/debug.log" + + echo "Checking for ${logfile}" + if [ -e "${logfile}" ]; then + echo "Removing ${logfile}" + rm "${logfile}" + fi +} + +# Execute CMD before each set of timing runs. +setup_run() { + set -euxo pipefail + + local TMP_DATADIR="$1" + local commit="$2" + clean_datadir "${TMP_DATADIR}" +} + +# Execute CMD before each timing run. +prepare_run() { + set -euxo pipefail + + local TMP_DATADIR="$1" + local ORIGINAL_DATADIR="$2" + + # Run the actual preparation steps + clean_datadir "${TMP_DATADIR}" + # Don't copy hidden files so use * + taskset -c 0-15 cp -r "$ORIGINAL_DATADIR"/* "$TMP_DATADIR" + clean_logs "${TMP_DATADIR}" +} + +# Executed after each timing run +conclude_run() { + set -euxo pipefail + + local commit="$1" + local TMP_DATADIR="$2" + local PNG_DIR="$3" + + # Search in subdirs e.g. $datadir/signet + debug_log=$(find "${TMP_DATADIR}" -name debug.log -print -quit) + if [ -n "${debug_log}" ]; then + echo "Generating plots from ${debug_log}" + if [ -x "bench-ci/parse_and_plot.py" ]; then + bench-ci/parse_and_plot.py "${debug_log}" "${PNG_DIR}" + else + ls -al "bench-ci/" + echo "parse_and_plot.py not found or not executable, skipping plot generation" + fi + else + ls -al "${TMP_DATADIR}/" + echo "debug.log not found, skipping plot generation" + fi + + # Move flamegraph if exists + if [ -e flamegraph.svg ]; then + mv flamegraph.svg "${commit}"-flamegraph.svg + fi +} + +# Execute CMD after the completion of all benchmarking runs for each individual +# command to be benchmarked. +cleanup_run() { + set -euxo pipefail + + local TMP_DATADIR="$1" + + # Clean up the datadir + clean_datadir "${TMP_DATADIR}" +} + +run_benchmark() { + local base_commit="$1" + local head_commit="$2" + local TMP_DATADIR="$3" + local ORIGINAL_DATADIR="$4" + local results_file="$5" + local png_dir="$6" + local chain="$7" + local stop_at_height="$8" + local connect_address="$9" + local dbcache="${10}" + local BINARIES_DIR="${11}" + + # Export functions so they can be used by hyperfine + export -f setup_run + export -f prepare_run + export -f conclude_run + export -f cleanup_run + export -f clean_datadir + export -f clean_logs + + # Debug: Print all variables being used + echo "=== Debug Information ===" + echo "TMP_DATADIR: ${TMP_DATADIR}" + echo "ORIGINAL_DATADIR: ${ORIGINAL_DATADIR}" + echo "BINARIES_DIR: ${BINARIES_DIR}" + echo "base_commit: ${base_commit}" + echo "head_commit: ${head_commit}" + echo "results_file: ${results_file}" + echo "png_dir: ${png_dir}" + echo "chain: ${chain}" + echo "stop_at_height: ${stop_at_height}" + echo "connect_address: ${connect_address}" + echo "dbcache: ${dbcache}" + echo "\n" + + # Run hyperfine + hyperfine \ + --shell=bash \ + --setup "setup_run ${TMP_DATADIR} {commit}" \ + --prepare "prepare_run ${TMP_DATADIR} ${ORIGINAL_DATADIR}" \ + --conclude "conclude_run {commit} ${TMP_DATADIR} ${png_dir}" \ + --cleanup "cleanup_run ${TMP_DATADIR}" \ + --runs 2 \ + --export-json "${results_file}" \ + --show-output \ + --command-name "base (${base_commit})" \ + --command-name "head (${head_commit})" \ + "taskset -c 1 flamegraph --palette bitcoin --title 'bitcoind IBD@{commit}' -c 'record -F 101 --call-graph fp' -- taskset -c 2-15 chrt -r 1 ${BINARIES_DIR}/{commit}/bitcoind -datadir=${TMP_DATADIR} -connect=${connect_address} -daemon=0 -prune=10000 -chain=${chain} -stopatheight=${stop_at_height} -dbcache=${dbcache} -printtoconsole=0 -debug=coindb -debug=leveldb -debug=bench -debug=validation" \ + -L commit "base,head" +} + +# Main execution +if [ "$#" -ne 11 ]; then + echo "Usage: $0 base_commit head_commit TMP_DATADIR ORIGINAL_DATADIR results_dir png_dir chain stop_at_height connect_address dbcache BINARIES_DIR" + exit 1 +fi + +run_benchmark "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" "$9" "${10}" "${11}" From e730c8ad8e77f0ff246581d216bc2f956cb2e616 Mon Sep 17 00:00:00 2001 From: willcl-ark Date: Tue, 28 Oct 2025 22:37:54 +0000 Subject: [PATCH 06/48] add justfile --- justfile | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 justfile diff --git a/justfile b/justfile new file mode 100644 index 000000000000..0ecd3e990f77 --- /dev/null +++ b/justfile @@ -0,0 +1,36 @@ +set shell := ["bash", "-uc"] + +os := os() + +default: + just --list + +# Build base and head binaries for CI +[group('ci')] +build-assumeutxo-binaries-guix base_commit head_commit: + #!/usr/bin/env bash + set -euxo pipefail + unset SOURCE_DATE_EPOCH # needed to run on NixOS + ./bench-ci/build_binaries.sh {{ base_commit }} {{ head_commit }} + +# Run mainnet benchmark workflow for large cache +[group('ci')] +run-mainnet-ci base_commit head_commit TMP_DATADIR ORIGINAL_DATADIR results_file dbcache png_dir binaries_dir: + #!/usr/bin/env bash + set -euxo pipefail + unset SOURCE_DATE_EPOCH # needed to run on NixOS + ./bench-ci/run-benchmark.sh {{ base_commit }} {{ head_commit }} {{ TMP_DATADIR }} {{ ORIGINAL_DATADIR }} {{ results_file }} {{ png_dir }} main 855000 "148.251.128.115:33333" {{ dbcache }} {{ binaries_dir }} + +# Cherry-pick commits from a bitcoin core PR onto this branch +[group('git')] +pick-pr pr_number: + #!/usr/bin/env bash + set -euxo pipefail + + if ! git remote get-url upstream 2>/dev/null | grep -q "bitcoin/bitcoin"; then + echo "Error: 'upstream' remote not found or doesn't point to bitcoin/bitcoin" + echo "Please add it with: `git remote add upstream https://github.com/bitcoin/bitcoin.git`" + exit 1 + fi + + git fetch upstream pull/{{ pr_number }}/head:bench-{{ pr_number }} && git cherry-pick $(git rev-list --reverse bench-{{ pr_number }} --not upstream/master) From 5b6f06485ce4ea634c1536749472fc5b77f3f90b Mon Sep 17 00:00:00 2001 From: willcl-ark Date: Wed, 29 Oct 2025 00:04:54 +0000 Subject: [PATCH 07/48] guix: build static Essentially picked (into bench-ci/guix) from https://github.com/bitcoin/bitcoin/pull/25573 Co-authored-by: fanquake --- bench-ci/guix/libexec/build.sh | 3 ++- bench-ci/guix/libexec/prelude.bash | 5 +++++ bench-ci/guix/manifest.scm | 36 ++++++++++++++++++++++++++++++ bench-ci/guix/security-check.py | 4 ++++ bench-ci/guix/symbol-check.py | 11 +++++---- 5 files changed, 54 insertions(+), 5 deletions(-) diff --git a/bench-ci/guix/libexec/build.sh b/bench-ci/guix/libexec/build.sh index d12c795beaa4..87ed1996cf38 100755 --- a/bench-ci/guix/libexec/build.sh +++ b/bench-ci/guix/libexec/build.sh @@ -144,10 +144,10 @@ export GUIX_LD_WRAPPER_DISABLE_RPATH=yes # Determine the correct value for -Wl,--dynamic-linker for the current $HOST case "$HOST" in + x86_64-linux-gnu) ;; *linux*) glibc_dynamic_linker=$( case "$HOST" in - x86_64-linux-gnu) echo /lib64/ld-linux-x86-64.so.2 ;; arm-linux-gnueabihf) echo /lib/ld-linux-armhf.so.3 ;; aarch64-linux-gnu) echo /lib/ld-linux-aarch64.so.1 ;; riscv64-linux-gnu) echo /lib/ld-linux-riscv64-lp64d.so.1 ;; @@ -236,6 +236,7 @@ esac # LDFLAGS case "$HOST" in + x86_64-linux-gnu) HOST_LDFLAGS=" -static-pie -static-libgcc -Wl,-O2" ;; *linux*) HOST_LDFLAGS="-Wl,--as-needed -Wl,--dynamic-linker=$glibc_dynamic_linker -static-libstdc++ -Wl,-O2" ;; *mingw*) HOST_LDFLAGS="-Wl,--no-insert-timestamp" ;; esac diff --git a/bench-ci/guix/libexec/prelude.bash b/bench-ci/guix/libexec/prelude.bash index b4e828013351..5756e856b240 100644 --- a/bench-ci/guix/libexec/prelude.bash +++ b/bench-ci/guix/libexec/prelude.bash @@ -13,6 +13,11 @@ GUIX_PROFILE=/home/github-runner/.config/guix/current . "$GUIX_PROFILE/etc/profile" echo "Using the following guix command:" command -v guix +echo "Guix command symlink points to:" +readlink -f "$(command -v guix)" +echo "Current Guix profile:" +echo "$GUIX_PROFILE" +echo "Profile generation info:" guix describe ################ diff --git a/bench-ci/guix/manifest.scm b/bench-ci/guix/manifest.scm index 59837e9647e9..f89eccc85360 100644 --- a/bench-ci/guix/manifest.scm +++ b/bench-ci/guix/manifest.scm @@ -438,6 +438,7 @@ inspecting signatures in Mach-O binaries.") "--enable-standard-branch-protection=yes", "--enable-cet=yes", "--disable-gcov", + "--disable-libsanitizer", building-on))) ((#:phases phases) `(modify-phases ,phases @@ -493,6 +494,37 @@ inspecting signatures in Mach-O binaries.") (("^install-others =.*$") (string-append "install-others = " out "/etc/rpc\n"))))))))))))) +(define-public glibc-2.42 + (let ((commit "71874f167aa5bb1538ff7e394beaacee28ebe65f")) + (package + (inherit glibc) ;; 2.39 + (version "2.42") + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://sourceware.org/git/glibc.git") + (commit commit))) + (file-name (git-file-name "glibc" commit)) + (sha256 + (base32 + "1pfbk907fkbavg7grbvb5zlhd3y47f8jj3d2v1s5w7xjnn0ypigq")) + (patches (search-our-patches "glibc-2.42-guix-prefix.patch")))) + (arguments + (substitute-keyword-arguments (package-arguments glibc) + ((#:configure-flags flags) + `(append ,flags + ;; https://www.gnu.org/software/libc/manual/html_node/Configuring-and-compiling.html + (list "--enable-stack-protector=all", + "--enable-bind-now", + "--enable-fortify-source", + "--enable-cet=yes", + "--enable-nscd=no", + "--enable-static-nss=yes", + "--disable-timezone-tools", + "--disable-profile", + "--disable-werror", + building-on)))))))) + ;; The sponge tool from moreutils. (define-public sponge (package @@ -560,6 +592,10 @@ inspecting signatures in Mach-O binaries.") nsis-x86_64 nss-certs osslsigncode)) + ((string-contains target "x86_64-linux-") + (list (list gcc-toolchain-13 "static") + (make-bitcoin-cross-toolchain target + #:base-libc glibc-2.42))) ((string-contains target "-linux-") (list bison pkg-config diff --git a/bench-ci/guix/security-check.py b/bench-ci/guix/security-check.py index be2e0cfbe2af..ac943e33aabd 100755 --- a/bench-ci/guix/security-check.py +++ b/bench-ci/guix/security-check.py @@ -122,6 +122,10 @@ def check_ELF_CONTROL_FLOW(binary) -> bool: return False def check_ELF_FORTIFY(binary) -> bool: + # no imported fortified funcs if we are fully static + # check could be changed to include all symbols + if binary.header.machine_type == lief.ELF.ARCH.X86_64: + return True # bitcoin wrapper does not currently contain any fortified functions if '--monolithic' in binary.strings: diff --git a/bench-ci/guix/symbol-check.py b/bench-ci/guix/symbol-check.py index 464b33cf66fb..3d7a654c8589 100755 --- a/bench-ci/guix/symbol-check.py +++ b/bench-ci/guix/symbol-check.py @@ -34,7 +34,7 @@ MAX_VERSIONS = { 'GCC': (7,0,0), 'GLIBC': { - lief.ELF.ARCH.X86_64: (2,31), + lief.ELF.ARCH.X86_64: (0,0), lief.ELF.ARCH.ARM: (2,31), lief.ELF.ARCH.AARCH64:(2,31), lief.ELF.ARCH.PPC64: (2,31), @@ -47,14 +47,14 @@ # Ignore symbols that are exported as part of every executable IGNORE_EXPORTS = { 'environ', '_environ', '__environ', '_fini', '_init', 'stdin', -'stdout', 'stderr', +'stdout', 'stderr', '__libc_single_threaded', } # Expected linker-loader names can be found here: # https://sourceware.org/glibc/wiki/ABIList?action=recall&rev=16 ELF_INTERPRETER_NAMES: dict[lief.ELF.ARCH, dict[lief.Header.ENDIANNESS, str]] = { lief.ELF.ARCH.X86_64: { - lief.Header.ENDIANNESS.LITTLE: "/lib64/ld-linux-x86-64.so.2", + lief.Header.ENDIANNESS.LITTLE: "", }, lief.ELF.ARCH.ARM: { lief.Header.ENDIANNESS.LITTLE: "/lib/ld-linux-armhf.so.3", @@ -98,7 +98,6 @@ 'libpthread.so.0', # threading 'libm.so.6', # math library 'libatomic.so.1', -'ld-linux-x86-64.so.2', # 64-bit dynamic linker 'ld-linux.so.2', # 32-bit dynamic linker 'ld-linux-aarch64.so.1', # 64-bit ARM dynamic linker 'ld-linux-armhf.so.3', # 32-bit ARM dynamic linker @@ -232,6 +231,10 @@ def check_RUNPATH(binary) -> bool: def check_ELF_libraries(binary) -> bool: ok: bool = True + + if binary.header.machine_type == lief.ELF.ARCH.X86_64: + return len(binary.libraries) == 0 + for library in binary.libraries: if library not in ELF_ALLOWED_LIBRARIES: print(f'{filename}: {library} is not in ALLOWED_LIBRARIES!') From 964fc243f16e5aa3f7246e3f286abbdffc65a05c Mon Sep 17 00:00:00 2001 From: willcl-ark Date: Mon, 10 Feb 2025 12:21:14 +0000 Subject: [PATCH 08/48] doc: add benchcoin docs --- .github/README.md | 1 + doc/benchcoin.md | 127 ++++++++++++ doc/flamegraph.svg | 491 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 619 insertions(+) create mode 120000 .github/README.md create mode 100644 doc/benchcoin.md create mode 100644 doc/flamegraph.svg diff --git a/.github/README.md b/.github/README.md new file mode 120000 index 000000000000..e5c578ba74b5 --- /dev/null +++ b/.github/README.md @@ -0,0 +1 @@ +../doc/benchcoin.md \ No newline at end of file diff --git a/doc/benchcoin.md b/doc/benchcoin.md new file mode 100644 index 000000000000..0b4159256c95 --- /dev/null +++ b/doc/benchcoin.md @@ -0,0 +1,127 @@ +# benchcoin + +A Bitcoin Core benchmarking fork + +This repository is a fork of Bitcoin Core that performs automated IBD benchmarking. +It allows you to measure and compare the performance impact of certain types of changes to Bitcoin Core's codebase on a longer-running IBD benchmark, in a (pretty) reproducible fashion. + +## Features + +- Automated IBD benchmarking on pull requests +- Multiple configurations: + - Mainnet with default cache + - Mainnet with large cache +- Performance visualizations including: + - Flamegraphs for CPU profiling + - Time series plots of various metrics + - Compare `base` (bitcoin/bitcoin:master) and `head` (PR) + +## Example Flamegraph + +Below is an example flamegraph showing CPU utilization during IBD: + +![Example Flamegraph](../doc/flamegraph.svg) + +## How to use it + +1. Open a Pull Request against **this repo** +2. Wait for the bot to comment on your PR after it's finished. + +See the [Contributing](#contributing) section for more details. + +## How it works + +When you open a pull request against this repository: + +1. The CI workflow automatically builds both the base and PR versions of bitcoind +2. Runs IBD benchmarks +3. Records performance metrics and creates various visualizations +4. Posts results as a comment on your PR + +The benchmarks test three configurations: +- Mainnet-default: with default (450 MB) dbcache + - From a pruned datadir @ height 840,000 to height 855,000 +- Mainnet-large: with 32000 MB dbcache + - From a pruned datadir @ height 840,000 to height 855,000 + +## Benchmark Outputs + +For each benchmark run, you'll get a github pages page with: + +- Timing comparisons between base and PR versions +- CPU flamegraphs showing where time is spent +- Time series plots showing: + - Block height vs time + - Cache size vs block height + - Cache size vs time + - Transaction count vs block height + - Coins cache size vs time + - LevelDB metrics + - Memory pool metrics + +## Local Development (WIP) + +To run benchmarks locally (WIP, and Linux-only due to [shell.nix](../shell.nix) limitations): + +1. Make sure you have [Nix package manager](https://nixos.org/download/) installed + +2. Setup the Nix development environment: +```bash +nix-shell +``` + +3. Run a local benchmark: +```bash +just run-signet +``` + +This will: +- Create a temporary directory for testing +- Build both base and PR versions +- Download the required UTXO snapshot if needed +- Run the benchmark +- Generate performance visualizations + +## Technical Details + +The benchmarking system uses: +- [Hyperfine](https://github.com/sharkdp/hyperfine) for benchmark timing +- [Flamegraph](https://github.com/willcl-ark/flamegraph) for CPU profiling +- [matplotlib](https://matplotlib.org/) for metric visualization +- [GitHub Actions](https://github.com/features/actions) for CI automation + +The system copies over a pruned datadir to speed up IBD to a more interesting height (840k). + +### Runner & seed + +The CI runner is self-hosted on a Hetzner AX52 running at the bitcoin-dev-tools organsation level. +It is running NixOS using configuration found in this repo: [nix-github-runner](https://github.com/bitcoin-dev-tools/nix-github-runner) for easier deployment and reproducibility. + +The runner host has 16 cores, with one used for system, one for `flamegraph` (i.e. `perf record`) and 14 dedicated to the Bitcoin Core node under test. + +The benchmarking peer on the runner is served blocks over the (real) "internet" (it may be LAN as it's within a single Hetzner region) via a single peer to exercise full IBD codepaths. This naturally may introduce some variance, but it was deemed preferable to running another bitcoin core on the same machine. + +This seed peer is another Hetzner VPS in the same region, and its configuration can be found here: [nix-seed-node](https://github.com/bitcoin-dev-tools/nix-seed-node) + +## Contributing + +### Benchmark an existing bitcoin/bitcoin PR + +This requires `just` be installed. If you don't have `just` installed you can run the commands in the [justfile](../justfile) manually. + +1. Fork this repository (or bitcoin/bitcoin and add this as a remote) +2. Create a new branch from benchcoin/master +3. Run: `just pick-pr ` to cherry-pick commits from the PR +4. Push the branch +5. Open a pull request **against this repo. NOT bitcoin/bitcoin** + +### Benchmark standalone/new changes + +1. Fork this repository (or bitcoin/bitcoin and add this as a remote) +2. Make your changes to Bitcoin Core +3. Open a pull request **against this repo. NOT bitcoin/bitcoin** +4. Wait for benchmark results to be posted on your PR here + +## License + +This project is licensed under the same terms as Bitcoin Core - see the [COPYING](../COPYING) file for details. diff --git a/doc/flamegraph.svg b/doc/flamegraph.svg new file mode 100644 index 000000000000..77f05068edd1 --- /dev/null +++ b/doc/flamegraph.svg @@ -0,0 +1,491 @@ +bitcoind assumeutxo IBD@head Reset ZoomSearch [unknown] (930,216,305 samples, 0.03%)libc.so.6::__GI___libc_open (1,277,437,934 samples, 0.04%)[unknown] (1,277,437,934 samples, 0.04%)[unknown] (1,121,698,471 samples, 0.03%)[unknown] (1,121,698,471 samples, 0.03%)[unknown] (1,121,698,471 samples, 0.03%)[unknown] (808,723,138 samples, 0.02%)[unknown] (705,370,773 samples, 0.02%)[unknown] (654,247,113 samples, 0.02%)[unknown] (601,840,190 samples, 0.02%)[unknown] (412,286,776 samples, 0.01%)libc.so.6::__lll_lock_wait_private (3,169,140,832 samples, 0.09%)[unknown] (3,068,852,192 samples, 0.09%)[unknown] (2,912,247,498 samples, 0.08%)[unknown] (2,859,869,350 samples, 0.08%)[unknown] (2,547,374,665 samples, 0.07%)[unknown] (2,442,338,234 samples, 0.07%)[unknown] (2,018,530,007 samples, 0.06%)[unknown] (1,768,059,272 samples, 0.05%)[unknown] (1,360,516,543 samples, 0.04%)[unknown] (941,780,033 samples, 0.03%)[unknown] (732,126,125 samples, 0.02%)[unknown] (367,091,733 samples, 0.01%)libc.so.6::__lll_lock_wake_private (53,149,822,463 samples, 1.49%)l..[unknown] (52,891,684,033 samples, 1.49%)[..[unknown] (51,489,363,011 samples, 1.45%)[..[unknown] (51,020,482,662 samples, 1.43%)[..[unknown] (46,915,115,303 samples, 1.32%)[unknown] (45,255,852,290 samples, 1.27%)[unknown] (38,150,418,340 samples, 1.07%)[unknown] (35,292,486,865 samples, 0.99%)[unknown] (7,892,404,247 samples, 0.22%)[unknown] (3,327,749,547 samples, 0.09%)[unknown] (1,188,855,625 samples, 0.03%)[unknown] (566,758,595 samples, 0.02%)libc.so.6::_int_free_create_chunk (628,326,946 samples, 0.02%)libc.so.6::_int_free_merge_chunk (358,656,602 samples, 0.01%)libc.so.6::_int_malloc (74,559,659,927 samples, 2.10%)li..[unknown] (721,620,417 samples, 0.02%)[unknown] (610,988,583 samples, 0.02%)[unknown] (610,988,583 samples, 0.02%)[unknown] (610,988,583 samples, 0.02%)[unknown] (559,250,914 samples, 0.02%)[unknown] (559,250,914 samples, 0.02%)libc.so.6::alloc_perturb (425,154,213 samples, 0.01%)libc.so.6::malloc (24,700,554,078 samples, 0.69%)libc.so.6::malloc_consolidate (735,996,757 samples, 0.02%)libc.so.6::unlink_chunk.isra.0 (6,120,352,373 samples, 0.17%)[unknown] (167,607,884,597 samples, 4.71%)[unknown]libstdc++.so.6.0.32::virtual thunk to std::__cxx11::basic_ostringstream<char, std::char_traits<char>, std::allocator<char> >::~basic_ostringstream (417,178,495 samples, 0.01%)[unknown] (417,178,495 samples, 0.01%)libc.so.6::_IO_default_xsputn (371,898,668 samples, 0.01%)libc.so.6::_IO_do_write@@GLIBC_2.2.5 (415,186,042 samples, 0.01%)libc.so.6::_IO_file_xsputn@@GLIBC_2.2.5 (52,841,892,362 samples, 1.49%)l..libc.so.6::_IO_fwrite (157,971,658,633 samples, 4.44%)libc.so...[[ext4]] (1,657,432,113 samples, 0.05%)[unknown] (573,069,492 samples, 0.02%)[[ext4]] (2,536,153,731 samples, 0.07%)[[ext4]] (10,537,322,599 samples, 0.30%)[unknown] (7,422,408,080 samples, 0.21%)[unknown] (6,329,696,449 samples, 0.18%)[unknown] (5,353,636,150 samples, 0.15%)[unknown] (5,041,980,997 samples, 0.14%)[unknown] (3,383,888,214 samples, 0.10%)[unknown] (1,348,486,405 samples, 0.04%)[unknown] (477,579,410 samples, 0.01%)[unknown] (424,961,857 samples, 0.01%)[[ext4]] (48,707,811,335 samples, 1.37%)[..[unknown] (37,296,429,178 samples, 1.05%)[unknown] (35,118,068,672 samples, 0.99%)[unknown] (29,610,843,695 samples, 0.83%)[unknown] (24,208,827,110 samples, 0.68%)[unknown] (17,096,181,771 samples, 0.48%)[unknown] (6,112,761,166 samples, 0.17%)[unknown] (1,344,893,459 samples, 0.04%)[unknown] (458,831,632 samples, 0.01%)[[ext4]] (365,017,200 samples, 0.01%)[[ext4]] (518,180,627 samples, 0.01%)[[ext4]] (466,259,788 samples, 0.01%)[[ext4]] (673,383,386 samples, 0.02%)[[ext4]] (59,764,846,104 samples, 1.68%)[..[unknown] (58,060,722,922 samples, 1.63%)[..[unknown] (7,950,480,723 samples, 0.22%)[unknown] (5,540,377,500 samples, 0.16%)[unknown] (865,590,582 samples, 0.02%)[unknown] (813,212,612 samples, 0.02%)[unknown] (813,212,612 samples, 0.02%)[unknown] (813,212,612 samples, 0.02%)[unknown] (711,368,524 samples, 0.02%)libc.so.6::__GI___libc_write (70,786,161,691 samples, 1.99%)li..[unknown] (70,568,950,557 samples, 1.98%)[u..[unknown] (69,379,113,892 samples, 1.95%)[u..[unknown] (68,772,280,665 samples, 1.93%)[u..[unknown] (66,697,097,059 samples, 1.88%)[u..[unknown] (3,800,961,354 samples, 0.11%)[unknown] (780,895,718 samples, 0.02%)libc.so.6::__memmove_avx512_unaligned_erms (15,769,232,267 samples, 0.44%)libc.so.6::__mempcpy@plt (4,938,637,189 samples, 0.14%)libc.so.6::__send (1,149,037,952 samples, 0.03%)[unknown] (1,149,037,952 samples, 0.03%)[unknown] (1,149,037,952 samples, 0.03%)[unknown] (1,149,037,952 samples, 0.03%)[unknown] (1,096,533,096 samples, 0.03%)[unknown] (1,096,533,096 samples, 0.03%)[unknown] (1,096,533,096 samples, 0.03%)[unknown] (1,094,640,456 samples, 0.03%)[unknown] (943,771,904 samples, 0.03%)[unknown] (626,496,659 samples, 0.02%)[unknown] (522,399,654 samples, 0.01%)[unknown] (469,549,544 samples, 0.01%)[unknown] (469,549,544 samples, 0.01%)[unknown] (366,321,373 samples, 0.01%)libc.so.6::_int_free (16,918,597,179 samples, 0.48%)libc.so.6::_int_free_merge_chunk (716,678,677 samples, 0.02%)libc.so.6::_int_malloc (1,269,524,481 samples, 0.04%)libc.so.6::cfree@GLIBC_2.2.5 (4,352,992,616 samples, 0.12%)libc.so.6::malloc (8,032,159,513 samples, 0.23%)libc.so.6::malloc_consolidate (39,479,511,598 samples, 1.11%)[unknown] (401,333,554 samples, 0.01%)[unknown] (401,333,554 samples, 0.01%)[unknown] (401,333,554 samples, 0.01%)[unknown] (401,333,554 samples, 0.01%)[unknown] (401,333,554 samples, 0.01%)[unknown] (401,333,554 samples, 0.01%)libc.so.6::new_do_write (469,906,341 samples, 0.01%)libc.so.6::read (459,442,054 samples, 0.01%)[unknown] (459,442,054 samples, 0.01%)[unknown] (360,200,514 samples, 0.01%)[unknown] (360,200,514 samples, 0.01%)[unknown] (360,200,514 samples, 0.01%)[unknown] (360,200,514 samples, 0.01%)libc.so.6::sysmalloc (469,717,952 samples, 0.01%)[unknown] (469,717,952 samples, 0.01%)[unknown] (415,893,983 samples, 0.01%)[unknown] (366,135,265 samples, 0.01%)[unknown] (366,135,265 samples, 0.01%)libc.so.6::unlink_chunk.isra.0 (2,862,604,776 samples, 0.08%)bitcoind::CBlockIndex::GetAncestor (412,360,660 samples, 0.01%)bitcoind::CCoinsViewCache::AccessCoin (421,783,849 samples, 0.01%)bitcoind::SipHashUint256Extra (6,150,872,313 samples, 0.17%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_erase (100,736,697,557 samples, 2.83%)bitc..bitcoind::SipHashUint256Extra (1,991,693,392 samples, 0.06%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_find_before_node (90,084,545,787 samples, 2.53%)bit..bitcoind::SipHashUint256Extra (71,251,854,599 samples, 2.00%)bi..bitcoind::SipHashUint256Extra (26,794,756,611 samples, 0.75%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_insert_unique_node (46,369,997,648 samples, 1.30%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_rehash (18,471,505,609 samples, 0.52%)libc.so.6::__memset_avx512_unaligned_erms (632,105,655 samples, 0.02%)[unknown] (579,371,219 samples, 0.02%)[unknown] (474,387,191 samples, 0.01%)[unknown] (421,585,797 samples, 0.01%)[unknown] (421,585,797 samples, 0.01%)[unknown] (368,759,434 samples, 0.01%)[unknown] (368,759,434 samples, 0.01%)[unknown] (368,759,434 samples, 0.01%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::erase (1,518,987,687 samples, 0.04%)bitcoind::SipHashUint256Extra (625,645,482 samples, 0.02%)bitcoind::SipHashUint256Extra (6,692,957,315 samples, 0.19%)[unknown] (1,036,177,296 samples, 0.03%)[unknown] (928,879,608 samples, 0.03%)[unknown] (877,183,919 samples, 0.02%)[unknown] (719,026,447 samples, 0.02%)[unknown] (666,701,067 samples, 0.02%)[unknown] (626,005,752 samples, 0.02%)[unknown] (364,282,815 samples, 0.01%)[unknown] (364,282,815 samples, 0.01%)[unknown] (364,282,815 samples, 0.01%)[unknown] (364,282,815 samples, 0.01%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::find (133,163,328,034 samples, 3.75%)bitcoi..bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_find_before_node (119,438,100,972 samples, 3.36%)bitco..bitcoind::SipHashUint256Extra (986,497,657 samples, 0.03%)bitcoind::std::__detail::_Hash_node<std::pair<COutPoint const, CCoinsCacheEntry>, false>* std::__detail::_Hashtable_alloc<PoolAllocator<std::__detail::_Hash_node<std::pair<COutPoint const, CCoinsCacheEntry>, false>, 144ul, 8ul> >::_M_allocate_node<std::piecewise_construct_t const&, std::tuple<COutPoint const&>, std::tuple<> > (5,414,052,109 samples, 0.15%)libc.so.6::cfree@GLIBC_2.2.5 (4,527,272,747 samples, 0.13%)bitcoind::CCoinsViewCache::BatchWrite (408,297,908,928 samples, 11.48%)bitcoind::CCoinsViewCac..bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::clear (4,431,167,402 samples, 0.12%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::find (676,133,697 samples, 0.02%)bitcoind::CCoinsViewCache::Flush (414,604,793,420 samples, 11.66%)bitcoind::CCoinsViewCach..bitcoind::CTxMemPool::removeConflicts (1,307,189,422 samples, 0.04%)bitcoind::std::_Rb_tree<COutPoint const*, std::pair<COutPoint const* const, CTransaction const*>, std::_Select1st<std::pair<COutPoint const* const, CTransaction const*> >, DereferencingComparator<COutPoint const*>, std::allocator<std::pair<COutPoint const* const, CTransaction const*> > >::find (940,298,479 samples, 0.03%)bitcoind::SipHashUint256 (1,301,282,993 samples, 0.04%)bitcoind::std::_Rb_tree<uint256, std::pair<uint256 const, long>, std::_Select1st<std::pair<uint256 const, long> >, std::less<uint256>, std::allocator<std::pair<uint256 const, long> > >::_M_erase (1,201,625,005 samples, 0.03%)bitcoind::CTxMemPool::removeForBlock (17,028,655,239 samples, 0.48%)bitcoind::std::_Rb_tree<uint256, std::pair<uint256 const, long>, std::_Select1st<std::pair<uint256 const, long> >, std::less<uint256>, std::allocator<std::pair<uint256 const, long> > >::erase (12,855,923,134 samples, 0.36%)bitcoind::std::_Rb_tree<uint256, std::pair<uint256 const, long>, std::_Select1st<std::pair<uint256 const, long> >, std::less<uint256>, std::allocator<std::pair<uint256 const, long> > >::equal_range (2,508,971,022 samples, 0.07%)[unknown] (3,441,479,431 samples, 0.10%)[unknown] (3,089,709,936 samples, 0.09%)[unknown] (2,820,174,820 samples, 0.08%)[unknown] (2,720,356,939 samples, 0.08%)[unknown] (2,720,356,939 samples, 0.08%)[unknown] (2,557,087,196 samples, 0.07%)[unknown] (2,356,775,337 samples, 0.07%)[unknown] (1,672,816,080 samples, 0.05%)[unknown] (1,100,674,926 samples, 0.03%)[unknown] (787,217,059 samples, 0.02%)[unknown] (574,492,426 samples, 0.02%)bitcoind::SipHashUint256Extra (359,543,734 samples, 0.01%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_find_before_node (10,977,516,042 samples, 0.31%)bitcoind::SipHashUint256Extra (3,562,058,963 samples, 0.10%)bitcoind::SipHashUint256Extra (1,836,963,585 samples, 0.05%)bitcoind::SipHashUint256Extra (6,867,820,925 samples, 0.19%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_insert_unique_node (16,890,522,357 samples, 0.48%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_rehash (12,768,158,119 samples, 0.36%)bitcoind::std::__detail::_Hash_node<std::pair<COutPoint const, CCoinsCacheEntry>, false>* std::__detail::_Hashtable_alloc<PoolAllocator<std::__detail::_Hash_node<std::pair<COutPoint const, CCoinsCacheEntry>, false>, 144ul, 8ul> >::_M_allocate_node<std::piecewise_construct_t const&, std::tuple<COutPoint const&>, std::tuple<> > (6,083,575,685 samples, 0.17%)[unknown] (2,667,289,880 samples, 0.08%)[unknown] (2,453,773,220 samples, 0.07%)[unknown] (2,293,236,868 samples, 0.06%)[unknown] (2,189,852,142 samples, 0.06%)[unknown] (1,978,814,058 samples, 0.06%)[unknown] (1,713,021,112 samples, 0.05%)[unknown] (1,360,558,892 samples, 0.04%)[unknown] (1,099,770,850 samples, 0.03%)[unknown] (785,095,967 samples, 0.02%)[unknown] (468,560,942 samples, 0.01%)[unknown] (366,515,283 samples, 0.01%)bitcoind::CCoinsViewCache::AddCoin (67,517,205,631 samples, 1.90%)bi..bitcoind::AddCoins (83,151,504,659 samples, 2.34%)bit..bitcoind::std::__detail::_Hash_node<std::pair<COutPoint const, CCoinsCacheEntry>, false>* std::__detail::_Hashtable_alloc<PoolAllocator<std::__detail::_Hash_node<std::pair<COutPoint const, CCoinsCacheEntry>, false>, 144ul, 8ul> >::_M_allocate_node<std::piecewise_construct_t const&, std::tuple<COutPoint const&>, std::tuple<> > (368,308,911 samples, 0.01%)bitcoind::CBlockIndex::GetAncestor (780,828,411 samples, 0.02%)bitcoind::SipHashUint256Extra (6,967,127,022 samples, 0.20%)bitcoind::CCoinsViewCache::FetchCoin (11,631,656,359 samples, 0.33%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_find_before_node (2,762,447,333 samples, 0.08%)bitcoind::CCoinsViewCache::AccessCoin (13,718,933,582 samples, 0.39%)bitcoind::CCoinsViewCache::AddCoin (935,848,977 samples, 0.03%)bitcoind::CCoinsViewCache::HaveInputs (363,967,847 samples, 0.01%)bitcoind::CCoinsViewCache::SpendCoin (775,446,488 samples, 0.02%)bitcoind::CTransaction::GetValueOut (571,129,594 samples, 0.02%)bitcoind::SipHashUint256Extra (6,132,196,838 samples, 0.17%)bitcoind::CCoinsViewCache::FetchCoin (22,771,955,106 samples, 0.64%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_find_before_node (8,387,260,917 samples, 0.24%)bitcoind::SipHashUint256Extra (672,360,582 samples, 0.02%)bitcoind::CCoinsViewCache::AccessCoin (27,541,380,041 samples, 0.77%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_find_before_node (840,128,595 samples, 0.02%)bitcoind::CCoinsViewCache::FetchCoin (9,862,576,991 samples, 0.28%)bitcoind::CCoinsViewCache::FetchCoin (723,258,358 samples, 0.02%)bitcoind::CCoinsViewBacked::GetCoin (1,001,559,892 samples, 0.03%)bitcoind::leveldb::LookupKey::LookupKey (468,932,422 samples, 0.01%)bitcoind::leveldb::(anonymous namespace)::BytewiseComparatorImpl::Compare (477,889,771 samples, 0.01%)bitcoind::leveldb::(anonymous namespace)::BytewiseComparatorImpl::Compare (2,464,437,114 samples, 0.07%)bitcoind::leveldb::FindFile (12,889,348,897 samples, 0.36%)bitcoind::leveldb::InternalKeyComparator::Compare (8,952,657,039 samples, 0.25%)libc.so.6::__memcmp_evex_movbe (3,658,168,717 samples, 0.10%)bitcoind::leveldb::InternalKeyComparator::Compare (468,603,758 samples, 0.01%)bitcoind::leveldb::(anonymous namespace)::ShardedLRUCache::Lookup (2,481,353,143 samples, 0.07%)[unknown] (470,703,247 samples, 0.01%)[unknown] (419,110,322 samples, 0.01%)[unknown] (367,081,554 samples, 0.01%)[unknown] (367,081,554 samples, 0.01%)bitcoind::std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_mutate (723,558,367 samples, 0.02%)libc.so.6::__memmove_avx512_unaligned_erms (682,634,544 samples, 0.02%)bitcoind::leveldb::Block::Iter::ParseNextKey (6,607,693,428 samples, 0.19%)libc.so.6::malloc (468,621,157 samples, 0.01%)bitcoind::leveldb::(anonymous namespace)::BytewiseComparatorImpl::Compare (3,736,764,421 samples, 0.11%)bitcoind::leveldb::InternalKeyComparator::Compare (22,752,758,306 samples, 0.64%)libc.so.6::__memcmp_evex_movbe (16,502,022,326 samples, 0.46%)bitcoind::leveldb::Block::Iter::Seek (81,753,854,146 samples, 2.30%)bit..libc.so.6::__memmove_avx512_unaligned_erms (624,754,079 samples, 0.02%)bitcoind::leveldb::Block::Iter::~Iter (1,202,042,453 samples, 0.03%)bitcoind::leveldb::Iterator::~Iterator (886,809,043 samples, 0.02%)bitcoind::leveldb::DeleteBlock (418,661,180 samples, 0.01%)bitcoind::leveldb::Block::NewIterator (1,830,741,267 samples, 0.05%)bitcoind::leveldb::BlockHandle::DecodeFrom (1,350,133,609 samples, 0.04%)bitcoind::leveldb::FilterBlockReader::KeyMayMatch (3,241,956,535 samples, 0.09%)bitcoind::leveldb::InternalFilterPolicy::KeyMayMatch (470,469,134 samples, 0.01%)bitcoind::leveldb::(anonymous namespace)::BloomFilterPolicy::KeyMayMatch (470,469,134 samples, 0.01%)bitcoind::leveldb::InternalKeyComparator::Compare (2,930,394,374 samples, 0.08%)bitcoind::leveldb::SaveValue (885,107,264 samples, 0.02%)bitcoind::leveldb::(anonymous namespace)::ShardedLRUCache::Lookup (1,152,034,360 samples, 0.03%)bitcoind::leveldb::Hash (363,890,191 samples, 0.01%)bitcoind::leveldb::Block::NewIterator (1,259,229,813 samples, 0.04%)bitcoind::leveldb::BlockHandle::DecodeFrom (1,156,612,863 samples, 0.03%)bitcoind::leveldb::GetVarint64 (416,693,035 samples, 0.01%)bitcoind::leveldb::Iterator::RegisterCleanup (363,166,691 samples, 0.01%)[unknown] (2,314,123,053 samples, 0.07%)[unknown] (2,156,687,800 samples, 0.06%)[unknown] (2,051,108,413 samples, 0.06%)[unknown] (1,945,393,833 samples, 0.05%)[unknown] (1,894,650,811 samples, 0.05%)[unknown] (1,894,650,811 samples, 0.05%)[unknown] (1,794,842,453 samples, 0.05%)[unknown] (1,315,291,384 samples, 0.04%)[unknown] (733,842,157 samples, 0.02%)[unknown] (421,059,647 samples, 0.01%)[unknown] (367,252,654 samples, 0.01%)bitcoind::crc32c::ExtendSse42 (56,521,776,403 samples, 1.59%)b..bitcoind::leveldb::ReadBlock (62,722,682,079 samples, 1.76%)b..libc.so.6::__GI___pthread_mutex_unlock_usercnt (978,769,336 samples, 0.03%)libc.so.6::cfree@GLIBC_2.2.5 (571,745,263 samples, 0.02%)bitcoind::leveldb::Table::BlockReader (93,027,689,265 samples, 2.62%)bit..libc.so.6::__memmove_avx512_unaligned_erms (525,280,305 samples, 0.01%)bitcoind::leveldb::Table::InternalGet (191,009,481,478 samples, 5.37%)bitcoind::..bitcoind::leveldb::(anonymous namespace)::ShardedLRUCache::Lookup (2,456,558,609 samples, 0.07%)bitcoind::leveldb::Hash (674,476,478 samples, 0.02%)libc.so.6::__GI___pthread_mutex_unlock_usercnt (949,827,762 samples, 0.03%)libc.so.6::__memcmp_evex_movbe (672,469,665 samples, 0.02%)libc.so.6::pthread_mutex_lock@@GLIBC_2.2.5 (770,697,666 samples, 0.02%)bitcoind::leveldb::TableCache::FindTable (5,889,647,371 samples, 0.17%)bitcoind::leveldb::TableCache::Get (199,229,141,358 samples, 5.60%)bitcoind::..bitcoind::leveldb::Version::Get (200,226,855,069 samples, 5.63%)bitcoind::..libc.so.6::__GI___pthread_mutex_unlock_usercnt (733,288,816 samples, 0.02%)bitcoind::leveldb::Version::ForEachOverlapping (215,208,197,899 samples, 6.05%)bitcoind::l..libc.so.6::__memcmp_evex_movbe (359,285,284 samples, 0.01%)bitcoind::leveldb::Version::Get (216,049,507,027 samples, 6.08%)bitcoind::l..bitcoind::leveldb::DBImpl::Get (217,672,929,621 samples, 6.12%)bitcoind::l..libc.so.6::__GI___pthread_mutex_unlock_usercnt (1,861,877,233 samples, 0.05%)bitcoind::CDBWrapper::ReadImpl[abi:cxx11] (221,752,252,623 samples, 6.24%)bitcoind::CD..libc.so.6::pthread_mutex_lock@@GLIBC_2.2.5 (1,748,433,964 samples, 0.05%)bitcoind::DecompressAmount (1,005,313,570 samples, 0.03%)bitcoind::void ScriptCompression::Unser<DataStream> (2,769,444,330 samples, 0.08%)bitcoind::void std::vector<std::byte, zero_after_free_allocator<std::byte> >::_M_range_insert<std::byte const*> (7,911,029,894 samples, 0.22%)libc.so.6::__memmove_avx512_unaligned_erms (416,410,569 samples, 0.01%)bitcoind::CCoinsViewDB::GetCoin (247,131,705,346 samples, 6.95%)bitcoind::CCo..bitcoind::CCoinsViewBacked::GetCoin (251,714,610,750 samples, 7.08%)bitcoind::CCo..bitcoind::CCoinsViewErrorCatcher::GetCoin (257,960,090,912 samples, 7.26%)bitcoind::CCoi..bitcoind::CCoinsViewDB::GetCoin (5,789,812,101 samples, 0.16%)bitcoind::SipHashUint256Extra (686,778,601 samples, 0.02%)[unknown] (1,028,820,936 samples, 0.03%)[unknown] (974,950,139 samples, 0.03%)[unknown] (867,196,862 samples, 0.02%)[unknown] (710,030,298 samples, 0.02%)[unknown] (710,030,298 samples, 0.02%)[unknown] (600,430,034 samples, 0.02%)[unknown] (489,234,171 samples, 0.01%)[unknown] (434,975,120 samples, 0.01%)[unknown] (434,975,120 samples, 0.01%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_find_before_node (29,304,700,539 samples, 0.82%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_insert_unique_node (21,307,639,964 samples, 0.60%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_rehash (20,780,827,998 samples, 0.58%)libc.so.6::__memset_avx512_unaligned_erms (579,451,231 samples, 0.02%)[unknown] (579,451,231 samples, 0.02%)[unknown] (526,649,228 samples, 0.01%)[unknown] (526,649,228 samples, 0.01%)[unknown] (526,649,228 samples, 0.01%)[unknown] (473,772,435 samples, 0.01%)[unknown] (420,996,348 samples, 0.01%)[unknown] (368,735,591 samples, 0.01%)[unknown] (368,735,591 samples, 0.01%)bitcoind::std::__detail::_Hash_node<std::pair<COutPoint const, CCoinsCacheEntry>, false>* std::__detail::_Hashtable_alloc<PoolAllocator<std::__detail::_Hash_node<std::pair<COutPoint const, CCoinsCacheEntry>, false>, 144ul, 8ul> >::_M_allocate_node<std::piecewise_construct_t const&, std::tuple<COutPoint const&>, std::tuple<> > (4,934,629,385 samples, 0.14%)[unknown] (421,130,280 samples, 0.01%)[unknown] (368,737,467 samples, 0.01%)[unknown] (368,737,467 samples, 0.01%)bitcoind::CCoinsViewCache::FetchCoin (327,425,895,563 samples, 9.21%)bitcoind::CCoinsVi..bitcoind::CCoinsViewErrorCatcher::GetCoin (601,145,923 samples, 0.02%)bitcoind::CCoinsViewCache::GetCoin (349,247,006,292 samples, 9.82%)bitcoind::CCoinsView..bitcoind::SipHashUint256Extra (17,454,209,723 samples, 0.49%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_find_before_node (22,697,810,020 samples, 0.64%)bitcoind::SipHashUint256Extra (4,124,049,750 samples, 0.12%)bitcoind::SipHashUint256Extra (4,306,133,540 samples, 0.12%)bitcoind::SipHashUint256Extra (7,085,914,542 samples, 0.20%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_insert_unique_node (19,180,887,889 samples, 0.54%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_rehash (12,199,005,039 samples, 0.34%)libc.so.6::__memset_avx512_unaligned_erms (574,777,734 samples, 0.02%)bitcoind::std::__detail::_Hash_node<std::pair<COutPoint const, CCoinsCacheEntry>, false>* std::__detail::_Hashtable_alloc<PoolAllocator<std::__detail::_Hash_node<std::pair<COutPoint const, CCoinsCacheEntry>, false>, 144ul, 8ul> >::_M_allocate_node<std::piecewise_construct_t const&, std::tuple<COutPoint const&>, std::tuple<> > (7,865,255,678 samples, 0.22%)[unknown] (1,969,736,150 samples, 0.06%)[unknown] (1,916,111,977 samples, 0.05%)[unknown] (1,812,200,695 samples, 0.05%)[unknown] (1,812,200,695 samples, 0.05%)[unknown] (1,812,200,695 samples, 0.05%)[unknown] (1,496,076,465 samples, 0.04%)[unknown] (1,234,917,855 samples, 0.03%)[unknown] (921,179,131 samples, 0.03%)[unknown] (658,036,512 samples, 0.02%)[unknown] (507,636,670 samples, 0.01%)bitcoind::CCoinsViewCache::FetchCoin (439,862,693,437 samples, 12.37%)bitcoind::CCoinsViewCache..bitcoind::CCoinsViewCache::GetCoin (567,408,453 samples, 0.02%)bitcoind::SipHashUint256Extra (11,079,411,759 samples, 0.31%)bitcoind::CCoinsViewCache::HaveInputs (468,021,622,384 samples, 13.16%)bitcoind::CCoinsViewCache::..bitcoind::Consensus::CheckTxInputs (525,550,058,887 samples, 14.78%)bitcoind::Consensus::CheckTxInp..bitcoind::CTransaction::GetValueOut (8,116,827,965 samples, 0.23%)bitcoind::EvaluateSequenceLocks (13,084,419,728 samples, 0.37%)bitcoind::CBlockIndex::GetMedianTimePast (12,762,378,539 samples, 0.36%)bitcoind::void std::__introsort_loop<long*, long, __gnu_cxx::__ops::_Iter_less_iter> (1,776,177,595 samples, 0.05%)bitcoind::SipHashUint256Extra (3,528,590,848 samples, 0.10%)bitcoind::CCoinsViewCache::FetchCoin (9,099,104,563 samples, 0.26%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_find_before_node (3,448,133,251 samples, 0.10%)bitcoind::SipHashUint256Extra (373,550,141 samples, 0.01%)bitcoind::CCoinsViewCache::AccessCoin (10,147,664,939 samples, 0.29%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_find_before_node (368,697,772 samples, 0.01%)bitcoind::CScript::GetSigOpCount (1,181,105,155 samples, 0.03%)bitcoind::CScript::IsPayToScriptHash (361,942,649 samples, 0.01%)bitcoind::CScript::IsPushOnly (1,550,137,517 samples, 0.04%)bitcoind::CScript::IsWitnessProgram (14,154,912,421 samples, 0.40%)bitcoind::GetScriptOp (1,727,592,712 samples, 0.05%)bitcoind::CScript::GetSigOpCount (1,617,517,251 samples, 0.05%)bitcoind::GetScriptOp (834,793,526 samples, 0.02%)bitcoind::WitnessSigOps (3,120,635,596 samples, 0.09%)bitcoind::CountWitnessSigOps (25,211,941,345 samples, 0.71%)bitcoind::CScript::GetSigOpCount (21,895,087,837 samples, 0.62%)bitcoind::GetScriptOp (11,871,223,047 samples, 0.33%)bitcoind::GetLegacySigOpCount (26,548,006,408 samples, 0.75%)bitcoind::GetScriptOp (1,822,747,918 samples, 0.05%)bitcoind::SipHashUint256Extra (1,613,835,917 samples, 0.05%)bitcoind::CCoinsViewCache::FetchCoin (6,631,397,326 samples, 0.19%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_find_before_node (2,817,298,340 samples, 0.08%)bitcoind::CCoinsViewCache::AccessCoin (7,316,792,317 samples, 0.21%)bitcoind::CCoinsViewCache::FetchCoin (363,943,746 samples, 0.01%)bitcoind::CScript::GetSigOpCount (1,160,904,417 samples, 0.03%)bitcoind::GetScriptOp (688,273,084 samples, 0.02%)bitcoind::GetScriptOp (2,964,048,193 samples, 0.08%)bitcoind::CScript::GetSigOpCount (5,643,658,755 samples, 0.16%)bitcoind::CScript::IsPayToScriptHash (581,631,871 samples, 0.02%)bitcoind::GetP2SHSigOpCount (15,633,133,461 samples, 0.44%)bitcoind::GetTransactionSigOpCost (84,183,784,739 samples, 2.37%)bit..libstdc++.so.6.0.32::operator delete (405,410,027 samples, 0.01%)bitcoind::SequenceLocks (1,661,951,664 samples, 0.05%)bitcoind::CalculateSequenceLocks (1,453,270,225 samples, 0.04%)bitcoind::SipHashUint256Extra (937,441,713 samples, 0.03%)bitcoind::CCoinsViewCache::FetchCoin (2,049,216,208 samples, 0.06%)bitcoind::SipHashUint256Extra (1,345,870,966 samples, 0.04%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_erase (363,086,362 samples, 0.01%)bitcoind::CCoinsViewCache::SpendCoin (20,676,663,595 samples, 0.58%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::erase (2,777,265,349 samples, 0.08%)bitcoind::SipHashUint256Extra (1,428,091,877 samples, 0.04%)bitcoind::UpdateCoins (24,385,621,354 samples, 0.69%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::erase (473,710,256 samples, 0.01%)bitcoind::AutoFile::write (1,453,030,200 samples, 0.04%)bitcoind::CSHA256::Write (1,519,729,645 samples, 0.04%)bitcoind::CompressAmount (618,711,609 samples, 0.02%)bitcoind::CompressScript (985,913,050 samples, 0.03%)[[ext4]] (404,866,263 samples, 0.01%)bitcoind::node::BlockManager::FindUndoPos (561,722,604 samples, 0.02%)bitcoind::FlatFileSeq::Allocate (509,374,850 samples, 0.01%)libc.so.6::posix_fallocate (509,374,850 samples, 0.01%)[unknown] (509,374,850 samples, 0.01%)[unknown] (509,374,850 samples, 0.01%)[unknown] (509,374,850 samples, 0.01%)[unknown] (457,299,763 samples, 0.01%)bitcoind::AutoFile::write (10,042,610,399 samples, 0.28%)bitcoind::CSHA256::Write (19,844,383,315 samples, 0.56%)bitcoind::sha256_x86_shani::Transform (3,151,148,807 samples, 0.09%)bitcoind::CompressAmount (1,773,668,392 samples, 0.05%)bitcoind::CompressScript (4,638,408,540 samples, 0.13%)bitcoind::prevector<33u, unsigned char, unsigned int, int>::resize (3,040,914,869 samples, 0.09%)bitcoind::CompressAmount (831,251,028 samples, 0.02%)bitcoind::prevector<33u, unsigned char, unsigned int, int>::resize (1,914,945,145 samples, 0.05%)bitcoind::void VectorFormatter<DefaultFormatter>::Ser<SizeComputer, std::vector<CTxUndo, std::allocator<CTxUndo> > > (7,020,871,233 samples, 0.20%)bitcoind::CompressScript (2,957,454,406 samples, 0.08%)bitcoind::AutoFile::write (4,887,544,250 samples, 0.14%)bitcoind::void WriteVarInt<AutoFile, (VarIntMode)0, unsigned int> (5,868,765,238 samples, 0.17%)bitcoind::CSHA256::Write (8,012,816,481 samples, 0.23%)bitcoind::sha256_x86_shani::Transform (938,301,513 samples, 0.03%)bitcoind::void WriteVarInt<HashWriter, (VarIntMode)0, unsigned int> (12,386,753,309 samples, 0.35%)libc.so.6::__memmove_avx512_unaligned_erms (941,007,723 samples, 0.03%)libc.so.6::_IO_fwrite (1,409,554,078 samples, 0.04%)bitcoind::node::BlockManager::UndoWriteToDisk (74,178,487,109 samples, 2.09%)bi..libc.so.6::__memmove_avx512_unaligned_erms (3,806,477,393 samples, 0.11%)bitcoind::CompressAmount (730,340,863 samples, 0.02%)bitcoind::void VectorFormatter<DefaultFormatter>::Ser<SizeComputer, std::vector<CTxUndo, std::allocator<CTxUndo> > > (9,108,229,147 samples, 0.26%)bitcoind::CompressScript (3,027,453,269 samples, 0.09%)bitcoind::prevector<33u, unsigned char, unsigned int, int>::resize (2,034,465,890 samples, 0.06%)bitcoind::void WriteVarInt<AutoFile, (VarIntMode)0, unsigned int> (367,022,852 samples, 0.01%)bitcoind::void WriteVarInt<HashWriter, (VarIntMode)0, unsigned int> (521,478,522 samples, 0.01%)bitcoind::node::BlockManager::WriteUndoDataForBlock (89,569,504,650 samples, 2.52%)bit..bitcoind::std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_dispose (583,464,405 samples, 0.02%)libc.so.6::malloc (1,716,514,762 samples, 0.05%)bitcoind::Chainstate::ConnectBlock (855,466,273,851 samples, 24.06%)bitcoind::Chainstate::ConnectBlockbitcoind::std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_dispose (418,091,278 samples, 0.01%)libc.so.6::cfree@GLIBC_2.2.5 (1,144,767,280 samples, 0.03%)bitcoind::Chainstate::ConnectTip (1,291,793,481,748 samples, 36.33%)bitcoind::Chainstate::ConnectTiplibstdc++.so.6.0.32::operator delete (627,918,999 samples, 0.02%)bitcoind::Chainstate::ActivateBestChainStep (1,291,995,942,063 samples, 36.34%)bitcoind::Chainstate::ActivateBestChainStepbitcoind::Chainstate::ActivateBestChain (1,292,515,820,515 samples, 36.35%)bitcoind::Chainstate::ActivateBestChainbitcoind::IsFinalTx (467,637,167 samples, 0.01%)bitcoind::void SerializeTransaction<ParamsStream<SizeComputer&, TransactionSerParams>, CTransaction> (25,890,452,766 samples, 0.73%)bitcoind::void WriteCompactSize<ParamsStream<SizeComputer&, TransactionSerParams> > (6,846,762,723 samples, 0.19%)bitcoind::ContextualCheckBlock (27,706,291,261 samples, 0.78%)bitcoind::void WriteCompactSize<ParamsStream<SizeComputer&, TransactionSerParams> > (1,089,597,648 samples, 0.03%)[[ext4]] (5,576,796,020 samples, 0.16%)[unknown] (3,815,201,507 samples, 0.11%)[unknown] (2,345,433,446 samples, 0.07%)[unknown] (520,783,293 samples, 0.01%)[[ext4]] (7,954,320,588 samples, 0.22%)[unknown] (1,662,013,865 samples, 0.05%)[unknown] (1,269,240,468 samples, 0.04%)[unknown] (705,348,263 samples, 0.02%)[unknown] (455,918,938 samples, 0.01%)[[nvme]] (807,919,787 samples, 0.02%)[[nvme]] (807,919,787 samples, 0.02%)[unknown] (807,919,787 samples, 0.02%)[unknown] (807,919,787 samples, 0.02%)[unknown] (807,919,787 samples, 0.02%)[unknown] (547,049,759 samples, 0.02%)[unknown] (496,243,932 samples, 0.01%)[unknown] (448,114,949 samples, 0.01%)[[ext4]] (12,810,206,632 samples, 0.36%)[unknown] (3,316,731,307 samples, 0.09%)[unknown] (2,036,481,321 samples, 0.06%)[unknown] (1,478,602,939 samples, 0.04%)[unknown] (1,322,893,322 samples, 0.04%)[unknown] (1,227,580,922 samples, 0.03%)[[ext4]] (13,121,603,080 samples, 0.37%)[[ext4]] (13,121,603,080 samples, 0.37%)bitcoind::FlatFileSeq::Flush (13,525,287,477 samples, 0.38%)libc.so.6::fdatasync (13,525,287,477 samples, 0.38%)[unknown] (13,525,287,477 samples, 0.38%)[unknown] (13,525,287,477 samples, 0.38%)[unknown] (13,525,287,477 samples, 0.38%)[[ext4]] (13,525,287,477 samples, 0.38%)[unknown] (13,525,287,477 samples, 0.38%)[unknown] (13,525,287,477 samples, 0.38%)[unknown] (13,525,287,477 samples, 0.38%)[unknown] (13,525,287,477 samples, 0.38%)[unknown] (403,684,397 samples, 0.01%)[unknown] (403,684,397 samples, 0.01%)[unknown] (403,684,397 samples, 0.01%)[[ext4]] (619,895,319 samples, 0.02%)[unknown] (483,644,425 samples, 0.01%)[[ext4]] (981,510,072 samples, 0.03%)[[ext4]] (1,916,459,846 samples, 0.05%)[unknown] (398,097,615 samples, 0.01%)[[ext4]] (1,967,105,500 samples, 0.06%)[[ext4]] (1,967,105,500 samples, 0.06%)bitcoind::node::BlockManager::FindNextBlockPos (16,065,033,072 samples, 0.45%)bitcoind::node::BlockManager::FlushBlockFile (15,700,018,553 samples, 0.44%)bitcoind::node::BlockManager::FlushUndoFile (2,174,731,076 samples, 0.06%)bitcoind::FlatFileSeq::Flush (2,174,731,076 samples, 0.06%)libc.so.6::fdatasync (2,174,731,076 samples, 0.06%)[unknown] (2,174,731,076 samples, 0.06%)[unknown] (2,174,731,076 samples, 0.06%)[unknown] (2,174,731,076 samples, 0.06%)[[ext4]] (2,174,731,076 samples, 0.06%)[unknown] (2,174,731,076 samples, 0.06%)[unknown] (2,174,731,076 samples, 0.06%)[unknown] (2,174,731,076 samples, 0.06%)[unknown] (2,119,891,081 samples, 0.06%)bitcoind::AutoFile::write (5,548,941,818 samples, 0.16%)libc.so.6::__GI___fstatat64 (365,833,677 samples, 0.01%)bitcoind::node::BlockManager::OpenBlockFile (470,569,767 samples, 0.01%)bitcoind::AutoFile::write (61,167,375,809 samples, 1.72%)b..[unknown] (598,511,547 samples, 0.02%)[unknown] (457,806,853 samples, 0.01%)[unknown] (457,806,853 samples, 0.01%)[unknown] (457,806,853 samples, 0.01%)[unknown] (409,659,414 samples, 0.01%)[unknown] (357,939,661 samples, 0.01%)bitcoind::AutoFile::write (4,647,493,060 samples, 0.13%)bitcoind::void WriteCompactSize<ParamsStream<AutoFile&, TransactionSerParams> > (6,592,272,733 samples, 0.19%)libc.so.6::_IO_fwrite (910,505,012 samples, 0.03%)bitcoind::void SerializeMany<ParamsStream<AutoFile&, TransactionSerParams>, CBlockHeader, std::vector<std::shared_ptr<CTransaction const>, std::allocator<std::shared_ptr<CTransaction const> > > > (82,131,751,453 samples, 2.31%)bit..libc.so.6::_IO_fwrite (8,527,040,897 samples, 0.24%)bitcoind::void SerializeMany<ParamsStream<SizeComputer&, TransactionSerParams>, CBlockHeader, std::vector<std::shared_ptr<CTransaction const>, std::allocator<std::shared_ptr<CTransaction const> > > > (15,937,770,258 samples, 0.45%)bitcoind::void WriteCompactSize<ParamsStream<SizeComputer&, TransactionSerParams> > (3,905,423,573 samples, 0.11%)bitcoind::void WriteCompactSize<ParamsStream<AutoFile&, TransactionSerParams> > (571,858,007 samples, 0.02%)bitcoind::node::BlockManager::WriteBlockToDisk (106,357,642,754 samples, 2.99%)bitc..bitcoind::void WriteCompactSize<ParamsStream<SizeComputer&, TransactionSerParams> > (1,696,751,451 samples, 0.05%)bitcoind::void SerializeMany<ParamsStream<SizeComputer&, TransactionSerParams>, CBlockHeader, std::vector<std::shared_ptr<CTransaction const>, std::allocator<std::shared_ptr<CTransaction const> > > > (17,168,001,989 samples, 0.48%)bitcoind::void WriteCompactSize<ParamsStream<SizeComputer&, TransactionSerParams> > (4,825,716,114 samples, 0.14%)bitcoind::node::BlockManager::SaveBlockToDisk (141,154,624,112 samples, 3.97%)bitcoi..bitcoind::void WriteCompactSize<ParamsStream<SizeComputer&, TransactionSerParams> > (1,511,914,109 samples, 0.04%)bitcoind::ChainstateManager::AcceptBlock (169,805,644,100 samples, 4.78%)bitcoind..bitcoind::void SerializeTransaction<ParamsStream<SizeComputer&, TransactionSerParams>, CTransaction> (419,732,705 samples, 0.01%)bitcoind::CScript::GetSigOpCount (1,244,733,942 samples, 0.04%)bitcoind::memcmp@plt (416,583,431 samples, 0.01%)bitcoind::std::_Rb_tree<COutPoint, COutPoint, std::_Identity<COutPoint>, std::less<COutPoint>, std::allocator<COutPoint> >::_M_erase (1,490,186,398 samples, 0.04%)bitcoind::std::pair<std::_Rb_tree_iterator<COutPoint>, bool> std::_Rb_tree<COutPoint, COutPoint, std::_Identity<COutPoint>, std::less<COutPoint>, std::allocator<COutPoint> >::_M_insert_unique<COutPoint const&> (4,247,810,353 samples, 0.12%)bitcoind::void WriteCompactSize<ParamsStream<SizeComputer&, TransactionSerParams> > (781,801,877 samples, 0.02%)libc.so.6::__memcmp_evex_movbe (6,070,441,149 samples, 0.17%)libc.so.6::cfree@GLIBC_2.2.5 (421,482,290 samples, 0.01%)libstdc++.so.6.0.32::operator delete (614,232,991 samples, 0.02%)bitcoind::CheckTransaction (25,650,523,240 samples, 0.72%)libstdc++.so.6.0.32::std::_Rb_tree_insert_and_rebalance (2,281,327,330 samples, 0.06%)bitcoind::CScript::GetSigOpCount (19,161,186,078 samples, 0.54%)bitcoind::GetScriptOp (8,992,060,021 samples, 0.25%)bitcoind::GetLegacySigOpCount (22,614,517,690 samples, 0.64%)bitcoind::GetScriptOp (1,176,069,512 samples, 0.03%)bitcoind::std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_dispose (1,349,955,285 samples, 0.04%)bitcoind::void SerializeTransaction<ParamsStream<SizeComputer&, TransactionSerParams>, CTransaction> (6,676,130,736 samples, 0.19%)bitcoind::void WriteCompactSize<ParamsStream<SizeComputer&, TransactionSerParams> > (1,329,967,416 samples, 0.04%)bitcoind::CheckBlock (60,550,319,748 samples, 1.70%)b..bitcoind::void WriteCompactSize<ParamsStream<SizeComputer&, TransactionSerParams> > (572,803,846 samples, 0.02%)bitcoind::ChainstateManager::ProcessNewBlock (1,523,688,403,640 samples, 42.85%)bitcoind::ChainstateManager::ProcessNewBlockbitcoind::sha256d64_x86_shani::Transform_2way (15,194,464,935 samples, 0.43%)bitcoind::BlockMerkleRoot (16,172,687,252 samples, 0.45%)bitcoind::ComputeMerkleRoot (15,499,928,925 samples, 0.44%)bitcoind::SHA256D64 (15,246,405,066 samples, 0.43%)bitcoind::CheckMerkleRoot (16,532,547,442 samples, 0.46%)libc.so.6::__memset_avx512_unaligned_erms (359,860,190 samples, 0.01%)bitcoind::sha256d64_x86_shani::Transform_2way (12,972,294,835 samples, 0.36%)bitcoind::SHA256D64 (13,025,009,373 samples, 0.37%)bitcoind::IsBlockMutated (30,129,022,002 samples, 0.85%)bitcoind::CheckWitnessMalleation (13,596,474,560 samples, 0.38%)bitcoind::BlockWitnessMerkleRoot (13,596,474,560 samples, 0.38%)bitcoind::ComputeMerkleRoot (13,077,728,889 samples, 0.37%)bitcoind::void (anonymous namespace)::PeerManagerImpl::MakeAndPushMessage<std::vector<CInv, std::allocator<CInv> >&> (406,479,193 samples, 0.01%)bitcoind::CConnman::PushMessage (406,479,193 samples, 0.01%)bitcoind::std::vector<unsigned char, std::allocator<unsigned char> >::_M_default_append (367,056,757 samples, 0.01%)bitcoind::unsigned long ReadCompactSize<ParamsStream<DataStream&, TransactionSerParams> > (622,762,372 samples, 0.02%)bitcoind::CTransaction::ComputeHasWitness (1,387,667,716 samples, 0.04%)bitcoind::CSHA256::Write (17,955,645,390 samples, 0.51%)bitcoind::sha256_x86_shani::Transform (11,932,913,194 samples, 0.34%)bitcoind::memcpy@plt (418,918,061 samples, 0.01%)bitcoind::sha256_x86_shani::Transform (3,306,980,273 samples, 0.09%)bitcoind::CSHA256::Finalize (22,917,960,073 samples, 0.64%)libc.so.6::__memmove_avx512_unaligned_erms (668,127,949 samples, 0.02%)bitcoind::CSHA256::Write (3,163,584,691 samples, 0.09%)bitcoind::CSHA256::Write (33,313,763,000 samples, 0.94%)bitcoind::sha256_x86_shani::Transform (14,194,928,537 samples, 0.40%)bitcoind::sha256_x86_shani::Transform (767,994,599 samples, 0.02%)bitcoind::CSHA256::Write (5,341,265,376 samples, 0.15%)bitcoind::void WriteCompactSize<ParamsStream<HashWriter&, TransactionSerParams> > (7,984,745,468 samples, 0.22%)bitcoind::void SerializeTransaction<ParamsStream<HashWriter&, TransactionSerParams>, CTransaction> (50,933,406,220 samples, 1.43%)b..libc.so.6::__memmove_avx512_unaligned_erms (5,183,727,187 samples, 0.15%)bitcoind::void WriteCompactSize<ParamsStream<HashWriter&, TransactionSerParams> > (1,613,593,834 samples, 0.05%)bitcoind::CTransaction::ComputeHash (80,845,793,271 samples, 2.27%)bit..bitcoind::CSHA256::Write (23,348,148,278 samples, 0.66%)bitcoind::sha256_x86_shani::Transform (11,595,812,714 samples, 0.33%)bitcoind::CSHA256::Finalize (24,335,325,870 samples, 0.68%)bitcoind::CSHA256::Write (2,288,432,816 samples, 0.06%)bitcoind::CSHA256::Write (64,681,112,465 samples, 1.82%)bi..bitcoind::sha256_x86_shani::Transform (33,677,349,718 samples, 0.95%)bitcoind::sha256_x86_shani::Transform (622,627,277 samples, 0.02%)bitcoind::CSHA256::Write (11,395,509,513 samples, 0.32%)bitcoind::sha256_x86_shani::Transform (523,186,685 samples, 0.01%)bitcoind::void WriteCompactSize<ParamsStream<HashWriter&, TransactionSerParams> > (17,046,149,334 samples, 0.48%)libc.so.6::__memmove_avx512_unaligned_erms (2,169,704,353 samples, 0.06%)bitcoind::void SerializeTransaction<ParamsStream<HashWriter&, TransactionSerParams>, CTransaction> (92,366,151,212 samples, 2.60%)bit..libc.so.6::__memmove_avx512_unaligned_erms (5,813,350,330 samples, 0.16%)bitcoind::void WriteCompactSize<ParamsStream<HashWriter&, TransactionSerParams> > (2,181,533,875 samples, 0.06%)bitcoind::CTransaction::ComputeWitnessHash (122,098,239,092 samples, 3.43%)bitco..bitcoind::CTransaction::CTransaction (213,407,475,563 samples, 6.00%)bitcoind::C..bitcoind::CTransaction::ComputeHasWitness (420,121,661 samples, 0.01%)bitcoind::CTransaction::ComputeHash (409,129,353 samples, 0.01%)bitcoind::DataStream::read (3,530,026,319 samples, 0.10%)bitcoind::operator new (628,502,415 samples, 0.02%)bitcoind::std::vector<unsigned char, std::allocator<unsigned char> >::_M_default_append (6,309,702,925 samples, 0.18%)bitcoind::unsigned long ReadCompactSize<ParamsStream<DataStream&, TransactionSerParams> > (4,193,618,734 samples, 0.12%)bitcoind::void Unserialize<ParamsStream<DataStream&, TransactionSerParams>, 28u, unsigned char> (727,834,750 samples, 0.02%)bitcoind::unsigned long ReadCompactSize<ParamsStream<DataStream&, TransactionSerParams> > (3,066,020,716 samples, 0.09%)bitcoind::void Unserialize<ParamsStream<DataStream&, TransactionSerParams>, 28u, unsigned char> (3,105,560,893 samples, 0.09%)bitcoind::unsigned long ReadCompactSize<ParamsStream<DataStream&, TransactionSerParams> > (1,455,846,726 samples, 0.04%)libc.so.6::__memmove_avx512_unaligned_erms (465,834,593 samples, 0.01%)bitcoind::void VectorFormatter<DefaultFormatter>::Unser<ParamsStream<DataStream&, TransactionSerParams>, std::vector<CTxIn, std::allocator<CTxIn> > > (15,677,650,112 samples, 0.44%)bitcoind::unsigned long ReadCompactSize<ParamsStream<DataStream&, TransactionSerParams> > (1,078,029,303 samples, 0.03%)bitcoind::void Unserialize<ParamsStream<DataStream&, TransactionSerParams>, 28u, unsigned char> (6,171,925,860 samples, 0.17%)bitcoind::unsigned long ReadCompactSize<ParamsStream<DataStream&, TransactionSerParams> > (982,486,879 samples, 0.03%)libc.so.6::__memmove_avx512_unaligned_erms (1,458,516,290 samples, 0.04%)bitcoind::void VectorFormatter<DefaultFormatter>::Unser<ParamsStream<DataStream&, TransactionSerParams>, std::vector<CTxOut, std::allocator<CTxOut> > > (13,963,877,725 samples, 0.39%)libc.so.6::__memmove_avx512_unaligned_erms (1,048,169,614 samples, 0.03%)libc.so.6::__memset_avx512_unaligned (1,046,482,105 samples, 0.03%)libc.so.6::__memset_avx512_unaligned_erms (1,963,080,141 samples, 0.06%)libc.so.6::malloc (3,025,102,825 samples, 0.09%)libstdc++.so.6.0.32::malloc@plt (1,462,651,744 samples, 0.04%)bitcoind::void Unserialize<ParamsStream<DataStream&, TransactionSerParams>, CTransaction> (283,898,001,379 samples, 7.98%)bitcoind::void U..libstdc++.so.6.0.32::operator new (2,257,486,798 samples, 0.06%)bitcoind::void VectorFormatter<DefaultFormatter>::Unser<ParamsStream<DataStream&, TransactionSerParams>, std::vector<CTxIn, std::allocator<CTxIn> > > (1,090,260,916 samples, 0.03%)libc.so.6::__memmove_avx512_unaligned_erms (2,753,503,546 samples, 0.08%)libc.so.6::malloc (1,149,716,024 samples, 0.03%)bitcoind::void ParamsWrapper<TransactionSerParams, CBlock>::Unserialize<DataStream> (291,189,121,636 samples, 8.19%)bitcoind::void P..bitcoind::void VectorFormatter<DefaultFormatter>::Unser<ParamsStream<DataStream&, TransactionSerParams>, std::vector<std::shared_ptr<CTransaction const>, std::allocator<std::shared_ptr<CTransaction const> > > > (291,134,772,004 samples, 8.19%)bitcoind::void V..libstdc++.so.6.0.32::operator new (518,854,210 samples, 0.01%)libc.so.6::__memset_avx512_unaligned_erms (3,588,859,593 samples, 0.10%)bitcoind::CConnman::ThreadMessageHandler (1,852,055,734,561 samples, 52.09%)bitcoind::CConnman::ThreadMessageHandlerlibstdc++.so.6.0.32::execute_native_thread_routine (1,852,107,180,016 samples, 52.09%)libstdc++.so.6.0.32::execute_native_thread_routinebitcoind::std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (*)(std::basic_string_view<char, std::char_traits<char> >, std::function<void ()>), char const*, CConnman::Start(CScheduler&, CConnman::Options const&)::{lambda()#5}> > >::_M_run (1,852,107,180,016 samples, 52.09%)bitcoind::std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (*)(std::basic_string_view<char, std::char..bitcoind::util::TraceThread (1,852,107,180,016 samples, 52.09%)bitcoind::util::TraceThreadlibstdc++.so.6.0.32::std::__cxx11::basic_stringbuf<char, std::char_traits<char>, std::allocator<char> >::overflow (397,900,679 samples, 0.01%)b-msghand (2,401,934,372,954 samples, 67.55%)b-msghand[[igc]] (638,737,826 samples, 0.02%)[unknown] (492,740,386 samples, 0.01%)[unknown] (492,740,386 samples, 0.01%)[unknown] (492,740,386 samples, 0.01%)libc.so.6::__libc_recv (23,769,090,268 samples, 0.67%)[unknown] (23,681,676,959 samples, 0.67%)[unknown] (23,585,908,630 samples, 0.66%)[unknown] (23,544,049,599 samples, 0.66%)[unknown] (23,499,819,825 samples, 0.66%)[unknown] (23,453,162,931 samples, 0.66%)[unknown] (23,205,326,716 samples, 0.65%)[unknown] (23,046,242,743 samples, 0.65%)[unknown] (23,000,657,790 samples, 0.65%)[unknown] (22,592,454,604 samples, 0.64%)[unknown] (21,715,983,496 samples, 0.61%)[unknown] (20,537,782,242 samples, 0.58%)[unknown] (19,311,079,312 samples, 0.54%)[unknown] (6,108,735,942 samples, 0.17%)[unknown] (1,360,583,546 samples, 0.04%)bitcoind::std::vector<std::byte, zero_after_free_allocator<std::byte> >::_M_fill_insert (16,619,401,507 samples, 0.47%)bitcoind::V2Transport::GetReceivedMessage (16,718,730,797 samples, 0.47%)[[igc]] (507,437,414 samples, 0.01%)[unknown] (412,781,498 samples, 0.01%)bitcoind::ChaCha20::Crypt (134,944,431,601 samples, 3.80%)bitcoi..bitcoind::ChaCha20Aligned::Crypt (134,944,431,601 samples, 3.80%)bitcoi..[unknown] (955,536,462 samples, 0.03%)[unknown] (955,536,462 samples, 0.03%)[unknown] (906,440,192 samples, 0.03%)[unknown] (861,463,927 samples, 0.02%)[unknown] (760,654,093 samples, 0.02%)[unknown] (658,510,836 samples, 0.02%)bitcoind::BIP324Cipher::Decrypt (196,638,059,936 samples, 5.53%)bitcoind::..bitcoind::FSChaCha20Poly1305::Decrypt (196,638,059,936 samples, 5.53%)bitcoind::..bitcoind::AEADChaCha20Poly1305::Decrypt (196,638,059,936 samples, 5.53%)bitcoind::..bitcoind::poly1305_donna::poly1305_update (61,693,628,335 samples, 1.74%)b..bitcoind::poly1305_donna::poly1305_blocks (61,693,628,335 samples, 1.74%)b..[unknown] (655,063,915 samples, 0.02%)[unknown] (607,270,235 samples, 0.02%)[unknown] (525,964,847 samples, 0.01%)[unknown] (525,964,847 samples, 0.01%)[unknown] (470,111,416 samples, 0.01%)[unknown] (470,107,658 samples, 0.01%)bitcoind::V2Transport::ProcessReceivedPacketBytes (198,460,164,481 samples, 5.58%)bitcoind::..libc.so.6::__memset_avx512_unaligned_erms (1,781,860,401 samples, 0.05%)bitcoind::V2Transport::ReceivedBytes (203,432,631,557 samples, 5.72%)bitcoind::..libc.so.6::__memmove_avx512_unaligned_erms (4,655,332,308 samples, 0.13%)libc.so.6::__memmove_avx512_unaligned_erms (10,715,799,436 samples, 0.30%)bitcoind::CNode::ReceiveMsgBytes (231,225,287,054 samples, 6.50%)bitcoind::CN..bitcoind::CConnman::SocketHandlerConnected (231,463,366,433 samples, 6.51%)bitcoind::CC..libc.so.6::__poll (3,830,838,327 samples, 0.11%)[unknown] (3,830,838,327 samples, 0.11%)[unknown] (3,782,920,191 samples, 0.11%)[unknown] (3,725,807,764 samples, 0.10%)[unknown] (3,522,157,004 samples, 0.10%)[unknown] (3,150,768,515 samples, 0.09%)[unknown] (2,627,277,437 samples, 0.07%)[unknown] (2,338,467,135 samples, 0.07%)[unknown] (2,037,878,870 samples, 0.06%)[unknown] (1,480,962,324 samples, 0.04%)[unknown] (688,242,613 samples, 0.02%)bitcoind::CConnman::SocketHandler (236,436,484,949 samples, 6.65%)bitcoind::CCo..b-net (260,905,688,952 samples, 7.34%)b-netlibstdc++.so.6.0.32::execute_native_thread_routine (236,875,778,634 samples, 6.66%)libstdc++.so...bitcoind::std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (*)(std::basic_string_view<char, std::char_traits<char> >, std::function<void ()>), char const*, CConnman::Start(CScheduler&, CConnman::Options const&)::{lambda()#1}> > >::_M_run (236,875,778,634 samples, 6.66%)bitcoind::std..bitcoind::util::TraceThread (236,875,778,634 samples, 6.66%)bitcoind::uti..bitcoind::CConnman::ThreadSocketHandler (236,875,778,634 samples, 6.66%)bitcoind::CCo..libc.so.6::_int_free_create_chunk (982,572,444 samples, 0.03%)libc.so.6::_int_free_merge_chunk (797,147,451 samples, 0.02%)[unknown] (2,170,942,655 samples, 0.06%)libc.so.6::__futex_abstimed_wait_common (459,293,920 samples, 0.01%)[unknown] (459,293,920 samples, 0.01%)[unknown] (405,487,988 samples, 0.01%)[unknown] (405,482,438 samples, 0.01%)[unknown] (405,482,438 samples, 0.01%)[unknown] (356,784,451 samples, 0.01%)libc.so.6::__lll_lock_wait_private (57,276,007,979 samples, 1.61%)l..[unknown] (54,932,210,267 samples, 1.54%)[..[unknown] (52,306,124,993 samples, 1.47%)[..[unknown] (51,843,804,338 samples, 1.46%)[..[unknown] (49,115,074,635 samples, 1.38%)[..[unknown] (47,020,328,627 samples, 1.32%)[unknown] (41,124,744,672 samples, 1.16%)[unknown] (38,571,784,780 samples, 1.08%)[unknown] (36,085,617,902 samples, 1.01%)[unknown] (32,172,048,607 samples, 0.90%)[unknown] (24,296,172,973 samples, 0.68%)[unknown] (14,033,556,774 samples, 0.39%)[unknown] (7,508,395,799 samples, 0.21%)[unknown] (3,295,574,070 samples, 0.09%)[unknown] (1,590,496,727 samples, 0.04%)[unknown] (1,002,849,637 samples, 0.03%)[unknown] (414,545,859 samples, 0.01%)libc.so.6::__lll_lock_wake_private (11,041,124,764 samples, 0.31%)[unknown] (10,991,162,572 samples, 0.31%)[unknown] (9,603,504,474 samples, 0.27%)[unknown] (9,459,439,012 samples, 0.27%)[unknown] (7,207,430,735 samples, 0.20%)[unknown] (5,830,933,319 samples, 0.16%)[unknown] (1,889,493,619 samples, 0.05%)[unknown] (394,342,984 samples, 0.01%)libc.so.6::_int_free (67,830,842,133 samples, 1.91%)li..libc.so.6::_int_free_merge_chunk (832,998,780 samples, 0.02%)libc.so.6::cfree@GLIBC_2.2.5 (2,087,601,863 samples, 0.06%)libc.so.6::malloc_consolidate (3,954,686,383 samples, 0.11%)libc.so.6::unlink_chunk.isra.0 (497,585,449 samples, 0.01%)bitcoind::CRollingBloomFilter::insert (356,229,732 samples, 0.01%)[unknown] (444,029,098 samples, 0.01%)[unknown] (397,328,353 samples, 0.01%)[unknown] (397,328,353 samples, 0.01%)[unknown] (397,328,353 samples, 0.01%)[unknown] (397,328,353 samples, 0.01%)bitcoind::CRollingBloomFilter::insert (165,056,371,702 samples, 4.64%)bitcoind..bitcoind::MurmurHash3 (79,485,956,130 samples, 2.24%)bit..[unknown] (508,285,343 samples, 0.01%)[unknown] (450,228,615 samples, 0.01%)[unknown] (404,433,625 samples, 0.01%)[unknown] (404,433,625 samples, 0.01%)[unknown] (404,433,625 samples, 0.01%)[unknown] (404,433,625 samples, 0.01%)bitcoind::MurmurHash3 (5,783,718,949 samples, 0.16%)bitcoind::TxOrphanage::EraseForBlock (4,219,830,042 samples, 0.12%)bitcoind::std::_Rb_tree<COutPoint, std::pair<COutPoint const, std::set<std::_Rb_tree_iterator<std::pair<transaction_identifier<true> const, TxOrphanage::OrphanTx> >, TxOrphanage::IteratorComparator, std::allocator<std::_Rb_tree_iterator<std::pair<transaction_identifier<true> const, TxOrphanage::OrphanTx> > > > >, std::_Select1st<std::pair<COutPoint const, std::set<std::_Rb_tree_iterator<std::pair<transaction_identifier<true> const, TxOrphanage::OrphanTx> >, TxOrphanage::IteratorComparator, std::allocator<std::_Rb_tree_iterator<std::pair<transaction_identifier<true> const, TxOrphanage::OrphanTx> > > > > >, std::less<COutPoint>, std::allocator<std::pair<COutPoint const, std::set<std::_Rb_tree_iterator<std::pair<transaction_identifier<true> const, TxOrphanage::OrphanTx> >, TxOrphanage::IteratorComparator, std::allocator<std::_Rb_tree_iterator<std::pair<transaction_identifier<true> const, TxOrphanage::OrphanTx> > > > > > >::find (834,275,777 samples, 0.02%)bitcoind::node::TxDownloadManagerImpl::BlockConnected (176,131,189,628 samples, 4.95%)bitcoind:..bitcoind::TxRequestTracker::ForgetTxHash (789,439,865 samples, 0.02%)bitcoind::std::_Function_handler<void (), ValidationSignals::BlockConnected(ChainstateRole, std::shared_ptr<CBlock const> const&, CBlockIndex const*)::{lambda()#2}>::_M_invoke (177,028,683,872 samples, 4.98%)bitcoind:..bitcoind::std::_Sp_counted_ptr_inplace<CTransaction const, std::allocator<void>, (__gnu_cxx::_Lock_policy)2>::_M_dispose (18,094,676,466 samples, 0.51%)libc.so.6::cfree@GLIBC_2.2.5 (20,756,908,966 samples, 0.58%)bitcoind::std::_Sp_counted_ptr_inplace<CBlock, std::allocator<void>, (__gnu_cxx::_Lock_policy)2>::_M_dispose (48,020,349,476 samples, 1.35%)b..libstdc++.so.6.0.32::operator delete (7,010,168,745 samples, 0.20%)bitcoind::std::_Sp_counted_ptr_inplace<CTransaction const, std::allocator<void>, (__gnu_cxx::_Lock_policy)2>::_M_dispose (595,474,492 samples, 0.02%)libc.so.6::cfree@GLIBC_2.2.5 (1,199,917,863 samples, 0.03%)bitcoind::std::_Function_handler<void (), ValidationSignals::BlockConnected(ChainstateRole, std::shared_ptr<CBlock const> const&, CBlockIndex const*)::{lambda()#2}>::_M_manager (50,738,017,178 samples, 1.43%)b..bitcoind::std::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_release (50,738,017,178 samples, 1.43%)b..libstdc++.so.6.0.32::operator delete (678,866,047 samples, 0.02%)bitcoind::CBlockPolicyEstimator::processBlock (2,721,087,031 samples, 0.08%)bitcoind::TxConfirmStats::UpdateMovingAverages (2,530,304,686 samples, 0.07%)bitcoind::std::_Function_handler<void (), ValidationSignals::MempoolTransactionsRemovedForBlock(std::vector<RemovedMempoolTransactionInfo, std::allocator<RemovedMempoolTransactionInfo> > const&, unsigned int)::{lambda()#2}>::_M_invoke (2,804,941,944 samples, 0.08%)bitcoind::SerialTaskRunner::ProcessQueue (230,828,220,555 samples, 6.49%)bitcoind::Se..bitcoind::CScheduler::serviceQueue (231,341,597,555 samples, 6.51%)bitcoind::CS..bitcoind::std::_Function_handler<void (), Repeat(CScheduler&, std::function<void ()>, std::chrono::duration<long, std::ratio<1l, 1000l> >)::{lambda()#1}>::_M_invoke (386,989,959 samples, 0.01%)bitcoind::Repeat (386,989,959 samples, 0.01%)bitcoind::CSHA512::Finalize (386,989,959 samples, 0.01%)b-scheduler (378,036,629,725 samples, 10.63%)b-schedulerlibstdc++.so.6.0.32::execute_native_thread_routine (231,550,611,141 samples, 6.51%)libstdc++.so..bitcoind::std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (*)(std::basic_string_view<char, std::char_traits<char> >, std::function<void ()>), char const*, AppInitMain(node::NodeContext&, interfaces::BlockAndHeaderTipInfo*)::{lambda()#1}> > >::_M_run (231,550,611,141 samples, 6.51%)bitcoind::st..bitcoind::util::TraceThread (231,550,611,141 samples, 6.51%)bitcoind::ut..[unknown] (1,418,945,425 samples, 0.04%)[unknown] (1,418,945,425 samples, 0.04%)[unknown] (1,418,945,425 samples, 0.04%)[unknown] (1,418,945,425 samples, 0.04%)[unknown] (1,418,945,425 samples, 0.04%)[unknown] (1,418,945,425 samples, 0.04%)[unknown] (1,418,945,425 samples, 0.04%)[unknown] (1,368,549,335 samples, 0.04%)[unknown] (1,263,446,697 samples, 0.04%)[unknown] (1,105,228,005 samples, 0.03%)[unknown] (684,110,353 samples, 0.02%)[unknown] (1,463,102,999 samples, 0.04%)libc.so.6::_int_malloc (1,478,820,457 samples, 0.04%)[unknown] (1,323,329,878 samples, 0.04%)[unknown] (1,219,148,488 samples, 0.03%)[unknown] (1,167,736,581 samples, 0.03%)[unknown] (1,167,736,581 samples, 0.03%)[unknown] (1,115,451,061 samples, 0.03%)[unknown] (1,014,330,812 samples, 0.03%)[unknown] (911,337,057 samples, 0.03%)[unknown] (714,835,817 samples, 0.02%)[unknown] (456,457,319 samples, 0.01%)[unknown] (3,458,133,839 samples, 0.10%)bitcoind::CDBWrapper::~CDBWrapper (1,160,687,762 samples, 0.03%)bitcoind::leveldb::DBImpl::~DBImpl (1,160,687,762 samples, 0.03%)bitcoind::leveldb::DBImpl::~DBImpl (1,160,687,762 samples, 0.03%)bitcoind::leveldb::TableCache::~TableCache (1,160,687,762 samples, 0.03%)bitcoind::leveldb::(anonymous namespace)::ShardedLRUCache::~ShardedLRUCache (1,160,687,762 samples, 0.03%)bitcoind::leveldb::(anonymous namespace)::LRUCache::~LRUCache (1,160,687,762 samples, 0.03%)bitcoind::leveldb::DeleteEntry (1,160,687,762 samples, 0.03%)libc.so.6::__munmap (1,160,687,762 samples, 0.03%)[unknown] (1,160,687,762 samples, 0.03%)[unknown] (1,160,687,762 samples, 0.03%)[unknown] (1,160,687,762 samples, 0.03%)[unknown] (1,160,687,762 samples, 0.03%)[unknown] (1,160,687,762 samples, 0.03%)[unknown] (1,160,687,762 samples, 0.03%)[unknown] (1,160,687,762 samples, 0.03%)[unknown] (1,160,687,762 samples, 0.03%)[unknown] (1,160,687,762 samples, 0.03%)[unknown] (580,697,270 samples, 0.02%)bitcoind::leveldb::PutVarint32 (363,737,260 samples, 0.01%)bitcoind::leveldb::PutLengthPrefixedSlice (571,217,019 samples, 0.02%)bitcoind::leveldb::WriteBatch::Delete (2,702,574,018 samples, 0.08%)bitcoind::leveldb::WriteBatchInternal::SetCount (1,715,286,573 samples, 0.05%)bitcoind::leveldb::WriteBatchInternal::SetCount (1,453,616,163 samples, 0.04%)bitcoind::CDBBatch::EraseImpl (5,090,452,967 samples, 0.14%)bitcoind::leveldb::PutVarint32 (1,872,876,736 samples, 0.05%)bitcoind::leveldb::PutLengthPrefixedSlice (2,343,591,543 samples, 0.07%)bitcoind::leveldb::PutVarint32 (572,117,605 samples, 0.02%)bitcoind::leveldb::PutVarint32 (567,491,257 samples, 0.02%)bitcoind::leveldb::PutLengthPrefixedSlice (938,977,738 samples, 0.03%)bitcoind::leveldb::WriteBatchInternal::Count (619,405,896 samples, 0.02%)bitcoind::leveldb::WriteBatch::Put (2,689,024,451 samples, 0.08%)bitcoind::CDBBatch::WriteImpl (10,634,135,335 samples, 0.30%)bitcoind::leveldb::GetLengthPrefixedSlice (463,225,027 samples, 0.01%)bitcoind::leveldb::GetLengthPrefixedSlice (6,489,010,398 samples, 0.18%)bitcoind::leveldb::GetVarint32 (3,004,905,545 samples, 0.08%)bitcoind::leveldb::GetVarint32 (1,160,323,181 samples, 0.03%)bitcoind::leveldb::Arena::AllocateAligned (406,996,319 samples, 0.01%)bitcoind::leveldb::(anonymous namespace)::BytewiseComparatorImpl::Compare (717,398,174 samples, 0.02%)bitcoind::leveldb::MemTable::KeyComparator::operator (5,108,835,410 samples, 0.14%)bitcoind::leveldb::InternalKeyComparator::Compare (3,324,232,989 samples, 0.09%)bitcoind::leveldb::InternalKeyComparator::Compare (4,244,823,969 samples, 0.12%)bitcoind::leveldb::(anonymous namespace)::BytewiseComparatorImpl::Compare (3,179,677,931 samples, 0.09%)bitcoind::leveldb::(anonymous namespace)::BytewiseComparatorImpl::Compare (33,796,395,298 samples, 0.95%)bitcoind::memcmp@plt (943,665,852 samples, 0.03%)bitcoind::leveldb::SkipList<char const*, leveldb::MemTable::KeyComparator>::Insert (185,524,871,422 samples, 5.22%)bitcoind:..bitcoind::leveldb::SkipList<char const*, leveldb::MemTable::KeyComparator>::FindGreaterOrEqual (178,286,921,652 samples, 5.01%)bitcoind:..bitcoind::leveldb::MemTable::KeyComparator::operator (98,574,957,808 samples, 2.77%)bitc..bitcoind::leveldb::InternalKeyComparator::Compare (75,114,665,063 samples, 2.11%)bi..libc.so.6::__memcmp_evex_movbe (8,323,863,446 samples, 0.23%)bitcoind::leveldb::MemTable::Add (188,893,844,275 samples, 5.31%)bitcoind::..bitcoind::leveldb::VarintLength (766,638,876 samples, 0.02%)bitcoind::leveldb::WriteBatchInternal::InsertInto (199,306,778,687 samples, 5.61%)bitcoind::..bitcoind::leveldb::WriteBatch::Iterate (198,740,714,232 samples, 5.59%)bitcoind::..bitcoind::crc32c::ExtendSse42 (471,197,509 samples, 0.01%)[[ext4]] (679,093,773 samples, 0.02%)[unknown] (522,409,669 samples, 0.01%)[[ext4]] (1,096,838,426 samples, 0.03%)[[ext4]] (1,722,362,275 samples, 0.05%)[unknown] (625,523,849 samples, 0.02%)[unknown] (574,147,567 samples, 0.02%)[unknown] (469,028,477 samples, 0.01%)[unknown] (469,028,477 samples, 0.01%)[unknown] (365,648,781 samples, 0.01%)[[ext4]] (4,389,086,262 samples, 0.12%)[unknown] (2,561,710,219 samples, 0.07%)[unknown] (2,561,710,219 samples, 0.07%)[unknown] (2,352,117,097 samples, 0.07%)[unknown] (1,880,182,821 samples, 0.05%)[unknown] (1,308,734,829 samples, 0.04%)[unknown] (523,736,031 samples, 0.01%)[[ext4]] (5,069,490,473 samples, 0.14%)[unknown] (5,069,490,473 samples, 0.14%)[unknown] (575,311,800 samples, 0.02%)[unknown] (470,084,210 samples, 0.01%)libc.so.6::__GI___libc_write (5,174,401,795 samples, 0.15%)[unknown] (5,174,401,795 samples, 0.15%)[unknown] (5,174,401,795 samples, 0.15%)[unknown] (5,174,401,795 samples, 0.15%)[unknown] (5,174,401,795 samples, 0.15%)bitcoind::CDBWrapper::WriteBatch (205,215,727,495 samples, 5.77%)bitcoind::C..bitcoind::leveldb::DBImpl::Write (205,215,727,495 samples, 5.77%)bitcoind::l..bitcoind::leveldb::log::Writer::AddRecord (5,908,948,808 samples, 0.17%)bitcoind::leveldb::log::Writer::EmitPhysicalRecord (5,908,948,808 samples, 0.17%)bitcoind::CompressScript (1,030,024,630 samples, 0.03%)bitcoind::prevector<33u, unsigned char, unsigned int, int>::resize (459,767,226 samples, 0.01%)bitcoind::void WriteVarInt<DataStream, (VarIntMode)0, unsigned int> (11,377,276,951 samples, 0.32%)bitcoind::void std::vector<std::byte, zero_after_free_allocator<std::byte> >::_M_range_insert<std::byte const*> (8,938,854,890 samples, 0.25%)bitcoind::CCoinsViewDB::BatchWrite (244,230,597,449 samples, 6.87%)bitcoind::CCo..bitcoind::void std::vector<std::byte, zero_after_free_allocator<std::byte> >::_M_range_insert<std::byte const*> (6,482,431,215 samples, 0.18%)bitcoind::std::_Hashtable<COutPoint, std::pair<COutPoint const, CCoinsCacheEntry>, PoolAllocator<std::pair<COutPoint const, CCoinsCacheEntry>, 144ul, 8ul>, std::__detail::_Select1st, std::equal_to<COutPoint>, SaltedOutpointHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::clear (13,761,064,935 samples, 0.39%)bitcoind::void std::vector<std::byte, zero_after_free_allocator<std::byte> >::_M_range_insert<std::byte const*> (364,557,178 samples, 0.01%)bitcoind::CCoinsViewCache::Flush (264,031,161,045 samples, 7.43%)bitcoind::CCoi..libc.so.6::cfree@GLIBC_2.2.5 (5,262,867,110 samples, 0.15%)bitcoind::Chainstate::ForceFlushStateToDisk (264,186,830,154 samples, 7.43%)bitcoind::Chai..bitcoind::Chainstate::FlushStateToDisk (264,186,830,154 samples, 7.43%)bitcoind::Chai..libc.so.6::__libc_start_call_main (265,453,083,455 samples, 7.47%)libc.so.6::__l..bitcoind::main (265,453,083,455 samples, 7.47%)bitcoind::mainbitcoind::Shutdown (265,453,083,455 samples, 7.47%)bitcoind::Shut..libc.so.6::_int_free (2,825,988,487 samples, 0.08%)libc.so.6::malloc_consolidate (2,950,349,980 samples, 0.08%)b-shutoff (278,389,331,208 samples, 7.83%)b-shutofflibc.so.6::unlink_chunk.isra.0 (3,181,018,445 samples, 0.09%)libc.so.6::_int_malloc (620,560,935 samples, 0.02%)[unknown] (518,649,070 samples, 0.01%)[unknown] (466,591,536 samples, 0.01%)[unknown] (466,591,536 samples, 0.01%)[unknown] (466,591,536 samples, 0.01%)[unknown] (415,625,450 samples, 0.01%)[unknown] (415,625,450 samples, 0.01%)[unknown] (363,215,208 samples, 0.01%)[unknown] (1,501,827,638 samples, 0.04%)bitcoind::leveldb::BlockBuilder::Add (581,064,351 samples, 0.02%)bitcoind::leveldb::TableBuilder::Add (1,003,488,869 samples, 0.03%)bitcoind::leveldb::DBImpl::WriteLevel0Table (1,214,913,728 samples, 0.03%)bitcoind::leveldb::BuildTable (1,214,913,728 samples, 0.03%)bitcoind::leveldb::WriteBatchInternal::InsertInto (2,528,384,688 samples, 0.07%)bitcoind::leveldb::WriteBatch::Iterate (2,528,384,688 samples, 0.07%)bitcoind::leveldb::MemTable::Add (2,422,985,691 samples, 0.07%)bitcoind::leveldb::SkipList<char const*, leveldb::MemTable::KeyComparator>::Insert (2,422,985,691 samples, 0.07%)bitcoind::leveldb::SkipList<char const*, leveldb::MemTable::KeyComparator>::FindGreaterOrEqual (2,318,036,540 samples, 0.07%)bitcoind::leveldb::MemTable::KeyComparator::operator (1,429,299,251 samples, 0.04%)bitcoind::leveldb::InternalKeyComparator::Compare (910,982,229 samples, 0.03%)bitcoind::CDBWrapper::CDBWrapper (5,007,147,537 samples, 0.14%)bitcoind::leveldb::DB::Open (5,007,147,537 samples, 0.14%)bitcoind::leveldb::DBImpl::Recover (4,954,666,055 samples, 0.14%)bitcoind::leveldb::DBImpl::RecoverLogFile (4,954,666,055 samples, 0.14%)libc.so.6::__memmove_avx512_unaligned_erms (1,000,227,273 samples, 0.03%)[unknown] (1,000,227,273 samples, 0.03%)[unknown] (1,000,227,273 samples, 0.03%)[unknown] (947,397,460 samples, 0.03%)[unknown] (947,397,460 samples, 0.03%)[unknown] (947,397,460 samples, 0.03%)[unknown] (841,684,608 samples, 0.02%)[unknown] (841,684,608 samples, 0.02%)[unknown] (841,684,608 samples, 0.02%)[unknown] (788,837,171 samples, 0.02%)bitcoind::node::BlockManager::GetAllBlockIndices (356,174,463 samples, 0.01%)bitcoind::base_uint<256u>::operator/= (4,353,340,184 samples, 0.12%)bitcoind::base_uint<256u>::operator>>=(unsigned int) (1,651,178,228 samples, 0.05%)bitcoind::GetBlockProof (4,611,529,418 samples, 0.13%)bitcoind::CSHA256::Finalize (469,172,416 samples, 0.01%)bitcoind::CSHA256::Write (416,395,152 samples, 0.01%)bitcoind::CBlockHeader::GetHash (889,433,319 samples, 0.03%)bitcoind::CSHA256::Write (420,260,903 samples, 0.01%)bitcoind::CheckProofOfWorkImpl (628,054,325 samples, 0.02%)bitcoind::arith_uint256::SetCompact (474,577,125 samples, 0.01%)bitcoind::leveldb::(anonymous namespace)::MergingIterator::Next (359,545,524 samples, 0.01%)bitcoind::leveldb::(anonymous namespace)::DBIter::FindNextUserEntry (615,884,661 samples, 0.02%)bitcoind::std::_Hashtable<uint256, std::pair<uint256 const, CBlockIndex>, std::allocator<std::pair<uint256 const, CBlockIndex> >, std::__detail::_Select1st, std::equal_to<uint256>, BlockHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true> >::_M_rehash (412,349,637 samples, 0.01%)bitcoind::node::BlockManager::InsertBlockIndex (926,266,820 samples, 0.03%)bitcoind::std::_Hashtable<uint256, std::pair<uint256 const, CBlockIndex>, std::allocator<std::pair<uint256 const, CBlockIndex> >, std::__detail::_Select1st, std::equal_to<uint256>, BlockHasher, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true> >::_M_insert_unique_node (621,546,429 samples, 0.02%)bitcoind::kernel::BlockTreeDB::LoadBlockIndexGuts (4,371,914,721 samples, 0.12%)bitcoind::node::BlockManager::GetAllBlockIndices (360,699,633 samples, 0.01%)bitcoind::void std::__introsort_loop<__gnu_cxx::__normal_iterator<CBlockIndex**, std::vector<CBlockIndex*, std::allocator<CBlockIndex*> > >, long, __gnu_cxx::__ops::_Iter_comp_iter<node::CBlockIndexHeightOnlyComparator> > (487,613,426 samples, 0.01%)bitcoind::node::BlockManager::LoadBlockIndexDB (10,756,421,448 samples, 0.30%)bitcoind::node::BlockManager::LoadBlockIndex (10,397,563,911 samples, 0.29%)libc.so.6::__libc_start_call_main (17,915,410,780 samples, 0.50%)bitcoind::main (17,915,410,780 samples, 0.50%)bitcoind::AppInitMain (17,915,410,780 samples, 0.50%)bitcoind::InitAndLoadChainstate (17,915,410,780 samples, 0.50%)bitcoind::node::LoadChainstate (17,915,410,780 samples, 0.50%)bitcoind::node::CompleteChainstateInitialization (17,915,410,780 samples, 0.50%)bitcoind::ChainstateManager::LoadBlockIndex (12,499,349,673 samples, 0.35%)bitcoind::void std::__introsort_loop<__gnu_cxx::__normal_iterator<CBlockIndex**, std::vector<CBlockIndex*, std::allocator<CBlockIndex*> > >, long, __gnu_cxx::__ops::_Iter_comp_iter<node::CBlockIndexHeightOnlyComparator> > (711,414,524 samples, 0.02%)bitcoind::void std::__introsort_loop<__gnu_cxx::__normal_iterator<CBlockIndex**, std::vector<CBlockIndex*, std::allocator<CBlockIndex*> > >, long, __gnu_cxx::__ops::_Iter_comp_iter<node::CBlockIndexHeightOnlyComparator> > (401,238,745 samples, 0.01%)libc.so.6::_int_free (620,663,041 samples, 0.02%)bitcoind::leveldb::(anonymous namespace)::BytewiseComparatorImpl::Compare (868,925,227 samples, 0.02%)bitcoind::leveldb::(anonymous namespace)::MergingIterator::Valid (404,648,282 samples, 0.01%)bitcoind::leveldb::(anonymous namespace)::MergingIterator::value (764,967,422 samples, 0.02%)bitcoind::leveldb::Compaction::ShouldStopBefore (811,517,390 samples, 0.02%)bitcoind::leveldb::TableCache::Evict (924,140,736 samples, 0.03%)bitcoind::leveldb::(anonymous namespace)::ShardedLRUCache::Erase (924,140,736 samples, 0.03%)bitcoind::leveldb::(anonymous namespace)::LRUCache::FinishErase (924,140,736 samples, 0.03%)bitcoind::leveldb::(anonymous namespace)::LRUCache::Unref (924,140,736 samples, 0.03%)bitcoind::leveldb::DeleteEntry (924,140,736 samples, 0.03%)libc.so.6::__munmap (924,140,736 samples, 0.03%)[unknown] (924,140,736 samples, 0.03%)[unknown] (924,140,736 samples, 0.03%)[unknown] (924,140,736 samples, 0.03%)[unknown] (924,140,736 samples, 0.03%)[unknown] (924,140,736 samples, 0.03%)[unknown] (924,140,736 samples, 0.03%)[unknown] (924,140,736 samples, 0.03%)[unknown] (924,140,736 samples, 0.03%)[unknown] (924,140,736 samples, 0.03%)[unknown] (622,330,840 samples, 0.02%)[[jbd2]] (579,680,739 samples, 0.02%)bitcoind::leveldb::DBImpl::DeleteObsoleteFiles (5,386,961,700 samples, 0.15%)libc.so.6::__unlink (4,462,820,964 samples, 0.13%)[unknown] (4,462,820,964 samples, 0.13%)[unknown] (4,462,820,964 samples, 0.13%)[unknown] (4,462,820,964 samples, 0.13%)[unknown] (4,462,820,964 samples, 0.13%)[unknown] (4,462,820,964 samples, 0.13%)[[ext4]] (4,462,820,964 samples, 0.13%)[unknown] (4,413,928,808 samples, 0.12%)[unknown] (4,413,928,808 samples, 0.12%)[unknown] (3,629,480,214 samples, 0.10%)[unknown] (2,527,606,876 samples, 0.07%)[unknown] (1,289,801,972 samples, 0.04%)[unknown] (411,890,158 samples, 0.01%)bitcoind::leveldb::(anonymous namespace)::BytewiseComparatorImpl::Compare (1,451,370,022 samples, 0.04%)bitcoind::leveldb::(anonymous namespace)::BytewiseComparatorImpl::Compare (2,100,345,679 samples, 0.06%)bitcoind::leveldb::(anonymous namespace)::MergingIterator::FindSmallest (7,036,670,089 samples, 0.20%)bitcoind::leveldb::InternalKeyComparator::Compare (5,331,785,618 samples, 0.15%)libc.so.6::__memcmp_evex_movbe (467,739,292 samples, 0.01%)bitcoind::leveldb::Block::Iter::ParseNextKey (1,597,295,639 samples, 0.04%)bitcoind::leveldb::Block::Iter::key (719,412,755 samples, 0.02%)bitcoind::leveldb::(anonymous namespace)::TwoLevelIterator::Next (3,651,719,685 samples, 0.10%)[unknown] (775,514,001 samples, 0.02%)[unknown] (775,514,001 samples, 0.02%)[unknown] (775,514,001 samples, 0.02%)[unknown] (775,514,001 samples, 0.02%)[unknown] (723,468,265 samples, 0.02%)[unknown] (671,854,971 samples, 0.02%)[unknown] (620,745,631 samples, 0.02%)[unknown] (467,020,775 samples, 0.01%)bitcoind::leveldb::ReadBlock (5,036,746,240 samples, 0.14%)bitcoind::crc32c::ExtendSse42 (4,003,982,142 samples, 0.11%)bitcoind::leveldb::(anonymous namespace)::TwoLevelIterator::InitDataBlock (6,011,618,239 samples, 0.17%)bitcoind::leveldb::Table::BlockReader (5,654,181,527 samples, 0.16%)bitcoind::leveldb::(anonymous namespace)::TwoLevelIterator::SkipEmptyDataBlocksForward (6,370,810,843 samples, 0.18%)bitcoind::leveldb::Block::Iter::Valid (514,863,214 samples, 0.01%)bitcoind::leveldb::(anonymous namespace)::TwoLevelIterator::Next (11,878,686,935 samples, 0.33%)[unknown] (357,525,803 samples, 0.01%)bitcoind::leveldb::ReadBlock (1,021,671,534 samples, 0.03%)bitcoind::crc32c::ExtendSse42 (664,145,731 samples, 0.02%)bitcoind::leveldb::(anonymous namespace)::TwoLevelIterator::InitDataBlock (1,177,165,099 samples, 0.03%)bitcoind::leveldb::Table::BlockReader (1,073,054,446 samples, 0.03%)bitcoind::leveldb::(anonymous namespace)::TwoLevelIterator::SkipEmptyDataBlocksForward (3,944,657,665 samples, 0.11%)bitcoind::leveldb::(anonymous namespace)::TwoLevelIterator::Valid (468,344,432 samples, 0.01%)bitcoind::leveldb::Block::Iter::Valid (360,443,695 samples, 0.01%)bitcoind::leveldb::(anonymous namespace)::MergingIterator::Next (25,840,019,062 samples, 0.73%)bitcoind::leveldb::InternalKeyComparator::Compare (877,755,927 samples, 0.02%)bitcoind::leveldb::(anonymous namespace)::MergingIterator::value (460,966,118 samples, 0.01%)bitcoind::leveldb::(anonymous namespace)::TwoLevelIterator::value (1,168,083,499 samples, 0.03%)bitcoind::leveldb::(anonymous namespace)::BytewiseComparatorImpl::Compare (1,027,703,208 samples, 0.03%)bitcoind::leveldb::Compaction::IsBaseLevelForKey (3,331,453,084 samples, 0.09%)libc.so.6::__memcmp_evex_movbe (1,380,364,868 samples, 0.04%)bitcoind::leveldb::(anonymous namespace)::BytewiseComparatorImpl::Compare (1,428,625,135 samples, 0.04%)bitcoind::leveldb::Compaction::ShouldStopBefore (5,019,787,360 samples, 0.14%)bitcoind::leveldb::InternalKeyComparator::Compare (3,376,359,370 samples, 0.09%)libc.so.6::__memcmp_evex_movbe (1,229,056,330 samples, 0.03%)bitcoind::leveldb::DBImpl::DeleteObsoleteFiles (947,024,277 samples, 0.03%)libc.so.6::__unlink (947,024,277 samples, 0.03%)[unknown] (947,024,277 samples, 0.03%)[unknown] (947,024,277 samples, 0.03%)[unknown] (947,024,277 samples, 0.03%)[unknown] (947,024,277 samples, 0.03%)[unknown] (947,024,277 samples, 0.03%)[[ext4]] (947,024,277 samples, 0.03%)[unknown] (947,024,277 samples, 0.03%)[unknown] (947,024,277 samples, 0.03%)[unknown] (891,341,341 samples, 0.03%)[unknown] (632,138,490 samples, 0.02%)[unknown] (416,723,130 samples, 0.01%)bitcoind::leveldb::MemTableIterator::key (1,087,232,643 samples, 0.03%)bitcoind::leveldb::(anonymous namespace)::BytewiseComparatorImpl::Compare (7,618,678,897 samples, 0.21%)bitcoind::leveldb::(anonymous namespace)::BytewiseComparatorImpl::Compare (3,626,299,706 samples, 0.10%)bitcoind::leveldb::InternalKeyComparator::Compare (5,542,426,443 samples, 0.16%)bitcoind::leveldb::PutVarint32 (983,352,417 samples, 0.03%)bitcoind::leveldb::EncodeVarint32 (516,604,326 samples, 0.01%)bitcoind::leveldb::BlockBuilder::Add (15,702,002,539 samples, 0.44%)bitcoind::leveldb::FilterBlockBuilder::AddKey (412,090,761 samples, 0.01%)bitcoind::leveldb::(anonymous namespace)::BloomFilterPolicy::CreateFilter (2,066,056,339 samples, 0.06%)bitcoind::leveldb::Hash (665,922,831 samples, 0.02%)bitcoind::leveldb::FilterBlockBuilder::StartBlock (2,377,127,267 samples, 0.07%)bitcoind::leveldb::FilterBlockBuilder::GenerateFilter (2,377,127,267 samples, 0.07%)bitcoind::leveldb::InternalKeyComparator::Compare (774,664,618 samples, 0.02%)[[ext4]] (567,830,671 samples, 0.02%)[[ext4]] (929,740,986 samples, 0.03%)[unknown] (361,910,315 samples, 0.01%)[[ext4]] (3,216,007,087 samples, 0.09%)[unknown] (2,077,722,358 samples, 0.06%)[unknown] (2,025,638,088 samples, 0.06%)[unknown] (1,766,421,841 samples, 0.05%)[unknown] (1,349,297,830 samples, 0.04%)[unknown] (985,540,031 samples, 0.03%)[[ext4]] (4,245,378,964 samples, 0.12%)[unknown] (4,245,378,964 samples, 0.12%)[unknown] (821,567,389 samples, 0.02%)bitcoind::leveldb::TableBuilder::Flush (5,177,109,910 samples, 0.15%)libc.so.6::__GI___libc_write (4,762,579,653 samples, 0.13%)[unknown] (4,762,579,653 samples, 0.13%)[unknown] (4,762,579,653 samples, 0.13%)[unknown] (4,762,579,653 samples, 0.13%)[unknown] (4,607,316,631 samples, 0.13%)libc.so.6::__memcmp_evex_movbe (2,327,620,616 samples, 0.07%)bitcoind::leveldb::TableBuilder::Add (29,098,360,859 samples, 0.82%)libc.so.6::__memmove_avx512_unaligned_erms (880,376,005 samples, 0.02%)[[ext4]] (576,641,035 samples, 0.02%)[unknown] (419,245,830 samples, 0.01%)[[ext4]] (681,686,302 samples, 0.02%)[[ext4]] (886,736,982 samples, 0.02%)[[ext4]] (886,736,982 samples, 0.02%)[[ext4]] (886,736,982 samples, 0.02%)bitcoind::leveldb::BuildTable (31,594,879,610 samples, 0.89%)libc.so.6::fdatasync (990,237,376 samples, 0.03%)[unknown] (990,237,376 samples, 0.03%)[unknown] (990,237,376 samples, 0.03%)[unknown] (990,237,376 samples, 0.03%)[[ext4]] (990,237,376 samples, 0.03%)[unknown] (990,237,376 samples, 0.03%)[unknown] (990,237,376 samples, 0.03%)[unknown] (990,237,376 samples, 0.03%)[unknown] (990,237,376 samples, 0.03%)bitcoind::leveldb::DBImpl::CompactMemTable (32,644,397,020 samples, 0.92%)bitcoind::leveldb::DBImpl::WriteLevel0Table (31,697,372,743 samples, 0.89%)[[ext4]] (360,420,776 samples, 0.01%)bitcoind::leveldb::TableBuilder::Finish (565,702,739 samples, 0.02%)bitcoind::leveldb::TableBuilder::WriteRawBlock (411,712,919 samples, 0.01%)libc.so.6::__GI___libc_write (411,712,919 samples, 0.01%)[unknown] (411,712,919 samples, 0.01%)[unknown] (411,712,919 samples, 0.01%)[unknown] (411,712,919 samples, 0.01%)[unknown] (411,712,919 samples, 0.01%)[[ext4]] (411,712,919 samples, 0.01%)[unknown] (411,712,919 samples, 0.01%)[[ext4]] (2,407,378,967 samples, 0.07%)[unknown] (1,896,402,811 samples, 0.05%)[unknown] (1,223,588,483 samples, 0.03%)[unknown] (359,102,837 samples, 0.01%)[[ext4]] (3,282,391,421 samples, 0.09%)[unknown] (669,408,205 samples, 0.02%)[[nvme]] (410,427,902 samples, 0.01%)[[nvme]] (410,427,902 samples, 0.01%)[unknown] (410,427,902 samples, 0.01%)[unknown] (410,427,902 samples, 0.01%)[[ext4]] (5,846,551,102 samples, 0.16%)[unknown] (1,539,251,741 samples, 0.04%)[unknown] (1,332,958,992 samples, 0.04%)[unknown] (1,230,554,197 samples, 0.03%)[unknown] (1,230,554,197 samples, 0.03%)[unknown] (1,230,554,197 samples, 0.03%)[unknown] (410,593,098 samples, 0.01%)[[ext4]] (5,999,990,575 samples, 0.17%)[[ext4]] (5,999,990,575 samples, 0.17%)bitcoind::leveldb::DBImpl::FinishCompactionOutputFile (7,646,287,561 samples, 0.22%)libc.so.6::fdatasync (6,926,959,748 samples, 0.19%)[unknown] (6,926,959,748 samples, 0.19%)[unknown] (6,926,959,748 samples, 0.19%)[unknown] (6,926,959,748 samples, 0.19%)[[ext4]] (6,926,959,748 samples, 0.19%)[unknown] (6,926,959,748 samples, 0.19%)[unknown] (6,926,959,748 samples, 0.19%)[unknown] (6,926,959,748 samples, 0.19%)[unknown] (6,720,904,548 samples, 0.19%)[unknown] (618,546,651 samples, 0.02%)[unknown] (618,541,816 samples, 0.02%)[unknown] (618,541,816 samples, 0.02%)[unknown] (618,541,816 samples, 0.02%)[unknown] (513,838,124 samples, 0.01%)[unknown] (411,261,494 samples, 0.01%)bitcoind::leveldb::InternalKeyComparator::Compare (1,176,215,358 samples, 0.03%)bitcoind::leveldb::(anonymous namespace)::BytewiseComparatorImpl::Compare (1,597,987,748 samples, 0.04%)bitcoind::leveldb::EncodeVarint32 (667,083,479 samples, 0.02%)bitcoind::leveldb::(anonymous namespace)::BytewiseComparatorImpl::Compare (3,023,925,193 samples, 0.09%)bitcoind::leveldb::InternalKeyComparator::Compare (5,485,200,607 samples, 0.15%)libc.so.6::__memcmp_evex_movbe (768,462,744 samples, 0.02%)bitcoind::leveldb::BlockBuilder::Add (19,355,464,658 samples, 0.54%)bitcoind::leveldb::PutVarint32 (3,963,072,776 samples, 0.11%)bitcoind::leveldb::EncodeVarint32 (2,006,933,285 samples, 0.06%)bitcoind::leveldb::FilterBlockBuilder::AddKey (1,861,448,821 samples, 0.05%)bitcoind::leveldb::(anonymous namespace)::BloomFilterPolicy::CreateFilter (13,758,298,035 samples, 0.39%)bitcoind::leveldb::Hash (5,062,387,301 samples, 0.14%)bitcoind::leveldb::InternalFilterPolicy::CreateFilter (408,507,196 samples, 0.01%)bitcoind::std::vector<leveldb::Slice, std::allocator<leveldb::Slice> >::_M_default_append (1,029,970,476 samples, 0.03%)bitcoind::leveldb::FilterBlockBuilder::GenerateFilter (16,416,940,319 samples, 0.46%)bitcoind::leveldb::FilterBlockBuilder::StartBlock (16,468,035,714 samples, 0.46%)bitcoind::leveldb::(anonymous namespace)::BytewiseComparatorImpl::Compare (1,040,462,681 samples, 0.03%)bitcoind::leveldb::InternalKeyComparator::Compare (2,329,875,977 samples, 0.07%)bitcoind::leveldb::InternalKeyComparator::FindShortestSeparator (358,735,789 samples, 0.01%)bitcoind::leveldb::PutVarint32 (610,801,466 samples, 0.02%)bitcoind::crc32c::ExtendSse42 (874,382,210 samples, 0.02%)bitcoind::leveldb::TableBuilder::WriteBlock (1,806,524,733 samples, 0.05%)bitcoind::leveldb::TableBuilder::WriteRawBlock (1,390,163,236 samples, 0.04%)libc.so.6::__memmove_avx512_unaligned_erms (413,005,584 samples, 0.01%)[[ext4]] (720,896,427 samples, 0.02%)[[ext4]] (2,836,852,977 samples, 0.08%)[unknown] (1,356,279,497 samples, 0.04%)[[ext4]] (3,876,087,820 samples, 0.11%)[unknown] (634,447,162 samples, 0.02%)[[ext4]] (6,595,884,839 samples, 0.19%)[unknown] (2,409,927,037 samples, 0.07%)[unknown] (2,152,146,763 samples, 0.06%)[unknown] (1,946,544,284 samples, 0.05%)[unknown] (1,691,057,617 samples, 0.05%)[unknown] (1,332,315,567 samples, 0.04%)[unknown] (618,194,201 samples, 0.02%)[unknown] (411,783,313 samples, 0.01%)[[ext4]] (21,402,165,352 samples, 0.60%)[unknown] (13,825,328,165 samples, 0.39%)[unknown] (12,948,506,018 samples, 0.36%)[unknown] (10,591,496,268 samples, 0.30%)[unknown] (8,635,293,060 samples, 0.24%)[unknown] (5,512,816,463 samples, 0.16%)[unknown] (1,755,230,935 samples, 0.05%)[unknown] (358,610,982 samples, 0.01%)[[ext4]] (26,848,872,865 samples, 0.76%)[unknown] (26,183,441,807 samples, 0.74%)[unknown] (3,805,768,350 samples, 0.11%)[unknown] (2,522,380,066 samples, 0.07%)libc.so.6::__GI___libc_write (29,870,807,469 samples, 0.84%)[unknown] (29,663,737,328 samples, 0.83%)[unknown] (29,456,391,053 samples, 0.83%)[unknown] (29,306,607,963 samples, 0.82%)[unknown] (28,793,621,717 samples, 0.81%)[unknown] (869,287,921 samples, 0.02%)bitcoind::leveldb::TableBuilder::Flush (32,039,566,359 samples, 0.90%)bitcoind::leveldb::TableBuilder::status (2,416,608,293 samples, 0.07%)bitcoind::memcpy@plt (1,533,086,169 samples, 0.04%)libc.so.6::__memcmp_evex_movbe (11,663,095,994 samples, 0.33%)libc.so.6::__memmove_avx512_unaligned_erms (6,084,682,703 samples, 0.17%)bitcoind::leveldb::TableBuilder::Add (101,316,031,082 samples, 2.85%)bitc..bitcoind::leveldb::TableBuilder::NumEntries (460,667,349 samples, 0.01%)libc.so.6::__memcmp_evex_movbe (359,824,779 samples, 0.01%)bitcoind::leveldb::DBImpl::DoCompactionWork (188,768,693,249 samples, 5.31%)bitcoind:..libc.so.6::__memmove_avx512_unaligned_erms (972,425,560 samples, 0.03%)bitcoind::leveldb::TableBuilder::NumEntries (767,314,029 samples, 0.02%)bitcoind::leveldb::DBImpl::BackgroundCompaction (198,697,568,504 samples, 5.59%)bitcoind::..libc.so.6::__memmove_avx512_unaligned_erms (569,144,596 samples, 0.02%)bitcoind::leveldb::DBImpl::DeleteObsoleteFiles (591,819,871 samples, 0.02%)libc.so.6::__unlink (591,819,871 samples, 0.02%)[unknown] (591,819,871 samples, 0.02%)[unknown] (591,819,871 samples, 0.02%)[unknown] (591,819,871 samples, 0.02%)[unknown] (591,819,871 samples, 0.02%)[unknown] (591,819,871 samples, 0.02%)[[ext4]] (591,819,871 samples, 0.02%)[unknown] (591,819,871 samples, 0.02%)[unknown] (591,819,871 samples, 0.02%)[unknown] (479,954,726 samples, 0.01%)[unknown] (428,868,095 samples, 0.01%)bitcoind::leveldb::(anonymous namespace)::BytewiseComparatorImpl::Compare (4,760,717,074 samples, 0.13%)bitcoind::leveldb::(anonymous namespace)::BytewiseComparatorImpl::Compare (1,338,954,347 samples, 0.04%)bitcoind::leveldb::InternalKeyComparator::Compare (2,113,914,207 samples, 0.06%)bitcoind::leveldb::BlockBuilder::Add (8,483,080,141 samples, 0.24%)bitcoind::leveldb::PutVarint32 (468,110,226 samples, 0.01%)bitcoind::leveldb::(anonymous namespace)::BloomFilterPolicy::CreateFilter (981,556,026 samples, 0.03%)bitcoind::leveldb::Hash (364,078,664 samples, 0.01%)bitcoind::leveldb::FilterBlockBuilder::StartBlock (1,085,605,353 samples, 0.03%)bitcoind::leveldb::FilterBlockBuilder::GenerateFilter (1,085,605,353 samples, 0.03%)[[ext4]] (363,216,075 samples, 0.01%)[[ext4]] (414,434,148 samples, 0.01%)[[ext4]] (622,462,403 samples, 0.02%)[[ext4]] (2,219,690,360 samples, 0.06%)[unknown] (1,545,848,943 samples, 0.04%)[unknown] (1,545,848,943 samples, 0.04%)[unknown] (1,344,249,592 samples, 0.04%)[unknown] (1,034,709,836 samples, 0.03%)[unknown] (463,122,475 samples, 0.01%)[[ext4]] (2,730,864,687 samples, 0.08%)[unknown] (2,627,509,960 samples, 0.07%)bitcoind::leveldb::TableBuilder::Flush (2,941,422,377 samples, 0.08%)libc.so.6::__GI___libc_write (2,889,358,538 samples, 0.08%)[unknown] (2,889,358,538 samples, 0.08%)[unknown] (2,837,160,085 samples, 0.08%)[unknown] (2,837,160,085 samples, 0.08%)[unknown] (2,837,160,085 samples, 0.08%)libc.so.6::__memcmp_evex_movbe (870,026,684 samples, 0.02%)bitcoind::leveldb::TableBuilder::Add (14,671,945,001 samples, 0.41%)libc.so.6::__memmove_avx512_unaligned_erms (516,334,186 samples, 0.01%)[[ext4]] (366,285,823 samples, 0.01%)bitcoind::leveldb::BuildTable (15,764,968,843 samples, 0.44%)libc.so.6::fdatasync (522,804,809 samples, 0.01%)[unknown] (522,804,809 samples, 0.01%)[unknown] (522,804,809 samples, 0.01%)[unknown] (522,804,809 samples, 0.01%)[[ext4]] (522,804,809 samples, 0.01%)[unknown] (522,804,809 samples, 0.01%)[unknown] (522,804,809 samples, 0.01%)[unknown] (522,804,809 samples, 0.01%)[unknown] (522,804,809 samples, 0.01%)[[ext4]] (470,717,222 samples, 0.01%)[[ext4]] (470,717,222 samples, 0.01%)[[ext4]] (470,717,222 samples, 0.01%)libstdc++.so.6.0.32::execute_native_thread_routine (215,158,735,915 samples, 6.05%)libstdc++.s..bitcoind::leveldb::(anonymous namespace)::PosixEnv::BackgroundThreadEntryPoint (215,158,735,915 samples, 6.05%)bitcoind::l..bitcoind::leveldb::DBImpl::BackgroundCall (215,158,735,915 samples, 6.05%)bitcoind::l..bitcoind::leveldb::DBImpl::CompactMemTable (16,461,167,411 samples, 0.46%)bitcoind::leveldb::DBImpl::WriteLevel0Table (15,869,347,540 samples, 0.45%)bitcoind (236,278,709,104 samples, 6.65%)bitcoindall (3,555,551,407,309 samples, 100%) From 32f5a75c597214af92842658c17e1f8fa38ae846 Mon Sep 17 00:00:00 2001 From: willcl-ark Date: Thu, 13 Feb 2025 12:57:21 +0000 Subject: [PATCH 09/48] remove legacy assumeutxo bench This is still available in the testing repo: https://github.com/bitcoin-dev-tools/benchcoin-testing --- bench-ci/run-assumeutxo-bench.sh | 154 ------------------------------- 1 file changed, 154 deletions(-) delete mode 100755 bench-ci/run-assumeutxo-bench.sh diff --git a/bench-ci/run-assumeutxo-bench.sh b/bench-ci/run-assumeutxo-bench.sh deleted file mode 100755 index a1ee910ed428..000000000000 --- a/bench-ci/run-assumeutxo-bench.sh +++ /dev/null @@ -1,154 +0,0 @@ -#!/usr/bin/env bash - -set -euxo pipefail - -# Helper function to check and clean datadir -clean_datadir() { - set -euxo pipefail - - local TMP_DATADIR="$1" - - # Create the directory if it doesn't exist - mkdir -p "${TMP_DATADIR}" - - # If we're in CI, clean without confirmation - if [ -n "${CI:-}" ]; then - rm -Rf "${TMP_DATADIR:?}"/* - else - read -rp "Are you sure you want to delete everything in ${TMP_DATADIR}? [y/N] " response - if [[ "$response" =~ ^[Yy]$ ]]; then - rm -Rf "${TMP_DATADIR:?}"/* - else - echo "Aborting..." - exit 1 - fi - fi -} - -# Helper function to clear logs -clean_logs() { - set -euxo pipefail - - local TMP_DATADIR="$1" - local logfile="${TMP_DATADIR}/debug.log" - - echo "Checking for ${logfile}" - if [ -e "${logfile}" ]; then - echo "Removing ${logfile}" - rm "${logfile}" - fi -} - -# Execute CMD before each set of timing runs. -setup_assumeutxo_snapshot_run() { - set -euxo pipefail - - local TMP_DATADIR="$1" - local commit="$2" - clean_datadir "${TMP_DATADIR}" -} - -# Execute CMD before each timing run. -prepare_assumeutxo_snapshot_run() { - set -euxo pipefail - - local TMP_DATADIR="$1" - local UTXO_PATH="$2" - local CONNECT_ADDRESS="$3" - local CHAIN="$4" - local DBCACHE="$5" - local commit="$6" - local BINARIES_DIR="$7" - - # Run the actual preparation steps - clean_datadir "${TMP_DATADIR}" - # Use the pre-built binaries from BINARIES_DIR - "${BINARIES_DIR}/${commit}/bitcoind" --help - taskset -c 0-15 "${BINARIES_DIR}/${commit}/bitcoind" -datadir="${TMP_DATADIR}" -connect="${CONNECT_ADDRESS}" -daemon=0 -chain="${CHAIN}" -stopatheight=1 -printtoconsole=0 - taskset -c 0-15 "${BINARIES_DIR}/${commit}/bitcoind" -datadir="${TMP_DATADIR}" -connect="${CONNECT_ADDRESS}" -daemon=0 -chain="${CHAIN}" -dbcache="${DBCACHE}" -pausebackgroundsync=1 -loadutxosnapshot="${UTXO_PATH}" -printtoconsole=0 || true - clean_logs "${TMP_DATADIR}" -} - -# Executed after each timing run -conclude_assumeutxo_snapshot_run() { - set -euxo pipefail - - local commit="$1" - local TMP_DATADIR="$2" - local PNG_DIR="$3" - - # Search in subdirs e.g. $datadir/signet - debug_log=$(find "${TMP_DATADIR}" -name debug.log -print -quit) - if [ -n "${debug_log}" ]; then - echo "Generating plots from ${debug_log}" - if [ -x "bench-ci/parse_and_plot.py" ]; then - bench-ci/parse_and_plot.py "${debug_log}" "${PNG_DIR}" - else - ls -al "bench-ci/" - echo "parse_and_plot.py not found or not executable, skipping plot generation" - fi - else - ls -al "${TMP_DATADIR}/" - echo "debug.log not found, skipping plot generation" - fi - - # Move flamegraph if exists - if [ -e flamegraph.svg ]; then - mv flamegraph.svg "${commit}"-flamegraph.svg - fi -} - -# Execute CMD after the completion of all benchmarking runs for each individual -# command to be benchmarked. -cleanup_assumeutxo_snapshot_run() { - set -euxo pipefail - - local TMP_DATADIR="$1" - - # Clean up the datadir - clean_datadir "${TMP_DATADIR}" -} - -run_benchmark() { - local base_commit="$1" - local head_commit="$2" - local TMP_DATADIR="$3" - local UTXO_PATH="$4" - local results_file="$5" - local png_dir="$6" - local chain="$7" - local stop_at_height="$8" - local connect_address="$9" - local dbcache="${10}" - local BINARIES_DIR="${11}" - - # Export functions so they can be used by hyperfine - export -f setup_assumeutxo_snapshot_run - export -f prepare_assumeutxo_snapshot_run - export -f conclude_assumeutxo_snapshot_run - export -f cleanup_assumeutxo_snapshot_run - export -f clean_datadir - export -f clean_logs - - # Run hyperfine - hyperfine \ - --shell=bash \ - --setup "setup_assumeutxo_snapshot_run ${TMP_DATADIR} {commit}" \ - --prepare "prepare_assumeutxo_snapshot_run ${TMP_DATADIR} ${UTXO_PATH} ${connect_address} ${chain} ${dbcache} {commit} ${BINARIES_DIR}" \ - --conclude "conclude_assumeutxo_snapshot_run {commit} ${TMP_DATADIR} ${png_dir}" \ - --cleanup "cleanup_assumeutxo_snapshot_run ${TMP_DATADIR}" \ - --runs 1 \ - --export-json "${results_file}" \ - --command-name "base (${base_commit})" \ - --command-name "head (${head_commit})" \ - "taskset -c 1 flamegraph --palette bitcoin --title 'bitcoind assumeutxo IBD@{commit}' -c 'record -F 101 --call-graph fp' -- taskset -c 2-15 ${BINARIES_DIR}/{commit}/bitcoind -datadir=${TMP_DATADIR} -connect=${connect_address} -daemon=0 -chain=${chain} -stopatheight=${stop_at_height} -dbcache=${dbcache} -printtoconsole=0 -debug=coindb -debug=leveldb -debug=bench -debug=validation" \ - -L commit "base,head" -} - -# Main execution -if [ "$#" -ne 11 ]; then - echo "Usage: $0 base_commit head_commit TMP_DATADIR UTXO_PATH results_dir png_dir chain stop_at_height connect_address dbcache BINARIES_DIR" - exit 1 -fi - -run_benchmark "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" "$9" "${10}" "${11}" From 939f9280be34a5dded6b150287ec610efd3b6986 Mon Sep 17 00:00:00 2001 From: willcl-ark Date: Thu, 13 Feb 2025 13:06:42 +0000 Subject: [PATCH 10/48] use *instrumented for flame runs --- .github/workflows/benchmark.yml | 4 +- bench-ci/run-benchmark-instrumented.sh | 163 +++++++++++++++++++++++++ bench-ci/run-benchmark.sh | 29 +---- justfile | 10 +- 4 files changed, 177 insertions(+), 29 deletions(-) create mode 100755 bench-ci/run-benchmark-instrumented.sh diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index c8c1a992fa25..fa10b537a931 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -39,7 +39,7 @@ jobs: with: name: bitcoind-binaries path: ${{ runner.temp }}/binaries/ - assumeutxo: + instrumented: needs: build-binaries strategy: matrix: @@ -85,7 +85,7 @@ jobs: run: | env mkdir -p "$TMP_DATADIR" - nix-shell --command "just run-${{ matrix.network }}-ci $BASE_SHA $CHECKOUT_COMMIT $TMP_DATADIR $ORIGINAL_DATADIR ${{ runner.temp }}/results.json ${{ matrix.dbcache }} ${{ runner.temp }}/pngs $BINARIES_DIR" + nix-shell --command "just run-${{ matrix.network }}-ci-instrumented $BASE_SHA $CHECKOUT_COMMIT $TMP_DATADIR $ORIGINAL_DATADIR ${{ runner.temp }}/results.json ${{ matrix.dbcache }} ${{ runner.temp }}/pngs $BINARIES_DIR" - uses: actions/upload-artifact@v4 with: name: result-${{ matrix.name }} diff --git a/bench-ci/run-benchmark-instrumented.sh b/bench-ci/run-benchmark-instrumented.sh new file mode 100755 index 000000000000..db7f5be6ead1 --- /dev/null +++ b/bench-ci/run-benchmark-instrumented.sh @@ -0,0 +1,163 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +# Helper function to check and clean datadir +clean_datadir() { + set -euxo pipefail + + local TMP_DATADIR="$1" + + # Create the directory if it doesn't exist + mkdir -p "${TMP_DATADIR}" + + # If we're in CI, clean without confirmation + if [ -n "${CI:-}" ]; then + rm -Rf "${TMP_DATADIR:?}"/* + else + read -rp "Are you sure you want to delete everything in ${TMP_DATADIR}? [y/N] " response + if [[ "$response" =~ ^[Yy]$ ]]; then + rm -Rf "${TMP_DATADIR:?}"/* + else + echo "Aborting..." + exit 1 + fi + fi +} + +# Helper function to clear logs +clean_logs() { + set -euxo pipefail + + local TMP_DATADIR="$1" + local logfile="${TMP_DATADIR}/debug.log" + + echo "Checking for ${logfile}" + if [ -e "${logfile}" ]; then + echo "Removing ${logfile}" + rm "${logfile}" + fi +} + +# Execute CMD before each set of timing runs. +setup_run() { + set -euxo pipefail + + local TMP_DATADIR="$1" + local commit="$2" + clean_datadir "${TMP_DATADIR}" +} + +# Execute CMD before each timing run. +prepare_run() { + set -euxo pipefail + + local TMP_DATADIR="$1" + local ORIGINAL_DATADIR="$2" + + # Run the actual preparation steps + clean_datadir "${TMP_DATADIR}" + # Don't copy hidden files so use * + taskset -c 0-15 cp -r "$ORIGINAL_DATADIR"/* "$TMP_DATADIR" + clean_logs "${TMP_DATADIR}" +} + +# Executed after each timing run +conclude_run() { + set -euxo pipefail + + local commit="$1" + local TMP_DATADIR="$2" + local PNG_DIR="$3" + + # Search in subdirs e.g. $datadir/signet + debug_log=$(find "${TMP_DATADIR}" -name debug.log -print -quit) + if [ -n "${debug_log}" ]; then + echo "Generating plots from ${debug_log}" + if [ -x "bench-ci/parse_and_plot.py" ]; then + bench-ci/parse_and_plot.py "${debug_log}" "${PNG_DIR}" + else + ls -al "bench-ci/" + echo "parse_and_plot.py not found or not executable, skipping plot generation" + fi + else + ls -al "${TMP_DATADIR}/" + echo "debug.log not found, skipping plot generation" + fi + + # Move flamegraph if exists + if [ -e flamegraph.svg ]; then + mv flamegraph.svg "${commit}"-flamegraph.svg + fi +} + +# Execute CMD after the completion of all benchmarking runs for each individual +# command to be benchmarked. +cleanup_run() { + set -euxo pipefail + + local TMP_DATADIR="$1" + + # Clean up the datadir + clean_datadir "${TMP_DATADIR}" +} + +run_benchmark() { + local base_commit="$1" + local head_commit="$2" + local TMP_DATADIR="$3" + local ORIGINAL_DATADIR="$4" + local results_file="$5" + local png_dir="$6" + local chain="$7" + local stop_at_height="$8" + local connect_address="$9" + local dbcache="${10}" + local BINARIES_DIR="${11}" + + # Export functions so they can be used by hyperfine + export -f setup_run + export -f prepare_run + export -f conclude_run + export -f cleanup_run + export -f clean_datadir + export -f clean_logs + + # Debug: Print all variables being used + echo "=== Debug Information ===" + echo "TMP_DATADIR: ${TMP_DATADIR}" + echo "ORIGINAL_DATADIR: ${ORIGINAL_DATADIR}" + echo "BINARIES_DIR: ${BINARIES_DIR}" + echo "base_commit: ${base_commit}" + echo "head_commit: ${head_commit}" + echo "results_file: ${results_file}" + echo "png_dir: ${png_dir}" + echo "chain: ${chain}" + echo "stop_at_height: ${stop_at_height}" + echo "connect_address: ${connect_address}" + echo "dbcache: ${dbcache}" + echo "\n" + + # Run hyperfine + hyperfine \ + --shell=bash \ + --setup "setup_run ${TMP_DATADIR} {commit}" \ + --prepare "prepare_run ${TMP_DATADIR} ${ORIGINAL_DATADIR}" \ + --conclude "conclude_run {commit} ${TMP_DATADIR} ${png_dir}" \ + --cleanup "cleanup_run ${TMP_DATADIR}" \ + --runs 1 \ + --export-json "${results_file}" \ + --show-output \ + --command-name "base (${base_commit})" \ + --command-name "head (${head_commit})" \ + "taskset -c 1 flamegraph --palette bitcoin --title 'bitcoind IBD@{commit}' -c 'record -F 101 --call-graph fp' -- taskset -c 2-15 chrt -r 1 ${BINARIES_DIR}/{commit}/bitcoind -datadir=${TMP_DATADIR} -connect=${connect_address} -daemon=0 -prune=10000 -chain=${chain} -stopatheight=${stop_at_height} -dbcache=${dbcache} -printtoconsole=0 -debug=coindb -debug=leveldb -debug=bench -debug=validation" \ + -L commit "base,head" +} + +# Main execution +if [ "$#" -ne 11 ]; then + echo "Usage: $0 base_commit head_commit TMP_DATADIR ORIGINAL_DATADIR results_dir png_dir chain stop_at_height connect_address dbcache BINARIES_DIR" + exit 1 +fi + +run_benchmark "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" "$9" "${10}" "${11}" diff --git a/bench-ci/run-benchmark.sh b/bench-ci/run-benchmark.sh index dc190a5fa207..ca77427bb31f 100755 --- a/bench-ci/run-benchmark.sh +++ b/bench-ci/run-benchmark.sh @@ -65,30 +65,7 @@ prepare_run() { # Executed after each timing run conclude_run() { set -euxo pipefail - - local commit="$1" - local TMP_DATADIR="$2" - local PNG_DIR="$3" - - # Search in subdirs e.g. $datadir/signet - debug_log=$(find "${TMP_DATADIR}" -name debug.log -print -quit) - if [ -n "${debug_log}" ]; then - echo "Generating plots from ${debug_log}" - if [ -x "bench-ci/parse_and_plot.py" ]; then - bench-ci/parse_and_plot.py "${debug_log}" "${PNG_DIR}" - else - ls -al "bench-ci/" - echo "parse_and_plot.py not found or not executable, skipping plot generation" - fi - else - ls -al "${TMP_DATADIR}/" - echo "debug.log not found, skipping plot generation" - fi - - # Move flamegraph if exists - if [ -e flamegraph.svg ]; then - mv flamegraph.svg "${commit}"-flamegraph.svg - fi + return 0 } # Execute CMD after the completion of all benchmarking runs for each individual @@ -145,12 +122,12 @@ run_benchmark() { --prepare "prepare_run ${TMP_DATADIR} ${ORIGINAL_DATADIR}" \ --conclude "conclude_run {commit} ${TMP_DATADIR} ${png_dir}" \ --cleanup "cleanup_run ${TMP_DATADIR}" \ - --runs 2 \ + --runs 3 \ --export-json "${results_file}" \ --show-output \ --command-name "base (${base_commit})" \ --command-name "head (${head_commit})" \ - "taskset -c 1 flamegraph --palette bitcoin --title 'bitcoind IBD@{commit}' -c 'record -F 101 --call-graph fp' -- taskset -c 2-15 chrt -r 1 ${BINARIES_DIR}/{commit}/bitcoind -datadir=${TMP_DATADIR} -connect=${connect_address} -daemon=0 -prune=10000 -chain=${chain} -stopatheight=${stop_at_height} -dbcache=${dbcache} -printtoconsole=0 -debug=coindb -debug=leveldb -debug=bench -debug=validation" \ + "taskset -c 2-15 chrt -o 1 ${BINARIES_DIR}/{commit}/bitcoind -datadir=${TMP_DATADIR} -connect=${connect_address} -daemon=0 -prune=10000 -chain=${chain} -stopatheight=${stop_at_height} -dbcache=${dbcache} -printtoconsole=0" \ -L commit "base,head" } diff --git a/justfile b/justfile index 0ecd3e990f77..e87616602ce7 100644 --- a/justfile +++ b/justfile @@ -13,7 +13,7 @@ build-assumeutxo-binaries-guix base_commit head_commit: unset SOURCE_DATE_EPOCH # needed to run on NixOS ./bench-ci/build_binaries.sh {{ base_commit }} {{ head_commit }} -# Run mainnet benchmark workflow for large cache +# Run uninstrumented benchmarks on mainnet [group('ci')] run-mainnet-ci base_commit head_commit TMP_DATADIR ORIGINAL_DATADIR results_file dbcache png_dir binaries_dir: #!/usr/bin/env bash @@ -21,6 +21,14 @@ run-mainnet-ci base_commit head_commit TMP_DATADIR ORIGINAL_DATADIR results_file unset SOURCE_DATE_EPOCH # needed to run on NixOS ./bench-ci/run-benchmark.sh {{ base_commit }} {{ head_commit }} {{ TMP_DATADIR }} {{ ORIGINAL_DATADIR }} {{ results_file }} {{ png_dir }} main 855000 "148.251.128.115:33333" {{ dbcache }} {{ binaries_dir }} +# Run instrumented benchmarks on mainnet +[group('ci')] +run-mainnet-ci-instrumented base_commit head_commit TMP_DATADIR ORIGINAL_DATADIR results_file dbcache png_dir binaries_dir: + #!/usr/bin/env bash + set -euxo pipefail + unset SOURCE_DATE_EPOCH # needed to run on NixOS + ./bench-ci/run-benchmark-instrumented.sh {{ base_commit }} {{ head_commit }} {{ TMP_DATADIR }} {{ ORIGINAL_DATADIR }} {{ results_file }} {{ png_dir }} main 855000 "148.251.128.115:33333" {{ dbcache }} {{ binaries_dir }} + # Cherry-pick commits from a bitcoin core PR onto this branch [group('git')] pick-pr pr_number: From 6d8b70c9465b2d0debfb2c291f8855ba39f98e48 Mon Sep 17 00:00:00 2001 From: willcl-ark Date: Thu, 13 Feb 2025 13:19:50 +0000 Subject: [PATCH 11/48] add uninstrumented run --- .github/workflows/benchmark.yml | 64 +++++++++++++++++++++++++++++++++ bench-ci/run-benchmark.sh | 27 ++++++-------- justfile | 4 +-- 3 files changed, 77 insertions(+), 18 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index fa10b537a931..cedbd3b562e6 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -39,6 +39,70 @@ jobs: with: name: bitcoind-binaries path: ${{ runner.temp }}/binaries/ + uninstrumented: + needs: build-binaries + strategy: + matrix: + include: + - network: mainnet + name: mainnet-default + timeout: 600 + datadir_path: /data/pruned-840k + dbcache: 450 + - network: mainnet + name: mainnet-large + timeout: 600 + datadir_path: /data/pruned-840k + dbcache: 32000 + runs-on: [self-hosted, linux, x64] + timeout-minutes: ${{ matrix.timeout }} + env: + NIX_PATH: nixpkgs=channel:nixos-unstable + ORIGINAL_DATADIR: ${{ matrix.datadir_path }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + steps: + - name: Checkout repo + uses: actions/checkout@v4 + with: + fetch-depth: 1 + - name: Download binaries + uses: actions/download-artifact@v4 + with: + name: bitcoind-binaries + path: ${{ runner.temp }}/binaries + - name: Set binary permissions + run: | + chmod +x ${{ runner.temp }}/binaries/base/bitcoind + chmod +x ${{ runner.temp }}/binaries/head/bitcoind + - name: Fetch base commit + run: | + echo "CHECKOUT_COMMIT=$(git rev-parse HEAD)" >> "$GITHUB_ENV" + git fetch --depth=1 origin ${{ github.event.pull_request.base.sha }} + - name: Run AssumeUTXO ${{ matrix.network }} + env: + TMP_DATADIR: "${{ runner.temp }}/base_datadir" + BINARIES_DIR: "${{ runner.temp }}/binaries" + run: | + env + mkdir -p "$TMP_DATADIR" + nix-shell --command "just run-${{ matrix.network }}-ci $BASE_SHA $CHECKOUT_COMMIT $TMP_DATADIR $ORIGINAL_DATADIR ${{ runner.temp }}/results.json ${{ matrix.dbcache }} $BINARIES_DIR" + - uses: actions/upload-artifact@v4 + with: + name: result-${{ matrix.name }} + path: "${{ runner.temp }}/results.json" + - name: Write GitHub and runner context files + env: + GITHUB_CONTEXT: ${{ toJSON(github) }} + RUNNER_CONTEXT: ${{ toJSON(runner) }} + run: | + mkdir contexts + echo "$GITHUB_CONTEXT" | nix-shell -p jq --command "jq 'del(.token)' > contexts/github.json" + echo "$RUNNER_CONTEXT" > contexts/runner.json + - name: Upload context metadata as artifact + uses: actions/upload-artifact@v4 + with: + name: run-metadata-${{ matrix.name }} + path: ./contexts/ instrumented: needs: build-binaries strategy: diff --git a/bench-ci/run-benchmark.sh b/bench-ci/run-benchmark.sh index ca77427bb31f..d0e3ba60e1f1 100755 --- a/bench-ci/run-benchmark.sh +++ b/bench-ci/run-benchmark.sh @@ -44,7 +44,6 @@ setup_run() { set -euxo pipefail local TMP_DATADIR="$1" - local commit="$2" clean_datadir "${TMP_DATADIR}" } @@ -85,17 +84,15 @@ run_benchmark() { local TMP_DATADIR="$3" local ORIGINAL_DATADIR="$4" local results_file="$5" - local png_dir="$6" - local chain="$7" - local stop_at_height="$8" - local connect_address="$9" - local dbcache="${10}" - local BINARIES_DIR="${11}" + local chain="$6" + local stop_at_height="$7" + local connect_address="$8" + local dbcache="${9}" + local BINARIES_DIR="${10}" # Export functions so they can be used by hyperfine export -f setup_run export -f prepare_run - export -f conclude_run export -f cleanup_run export -f clean_datadir export -f clean_logs @@ -108,33 +105,31 @@ run_benchmark() { echo "base_commit: ${base_commit}" echo "head_commit: ${head_commit}" echo "results_file: ${results_file}" - echo "png_dir: ${png_dir}" echo "chain: ${chain}" echo "stop_at_height: ${stop_at_height}" echo "connect_address: ${connect_address}" echo "dbcache: ${dbcache}" - echo "\n" + printf \n # Run hyperfine hyperfine \ --shell=bash \ - --setup "setup_run ${TMP_DATADIR} {commit}" \ + --setup "setup_run ${TMP_DATADIR}" \ --prepare "prepare_run ${TMP_DATADIR} ${ORIGINAL_DATADIR}" \ - --conclude "conclude_run {commit} ${TMP_DATADIR} ${png_dir}" \ --cleanup "cleanup_run ${TMP_DATADIR}" \ --runs 3 \ --export-json "${results_file}" \ --show-output \ --command-name "base (${base_commit})" \ --command-name "head (${head_commit})" \ - "taskset -c 2-15 chrt -o 1 ${BINARIES_DIR}/{commit}/bitcoind -datadir=${TMP_DATADIR} -connect=${connect_address} -daemon=0 -prune=10000 -chain=${chain} -stopatheight=${stop_at_height} -dbcache=${dbcache} -printtoconsole=0" \ + "taskset -c 2-15 chrt -o 0 ${BINARIES_DIR}/{commit}/bitcoind -datadir=${TMP_DATADIR} -connect=${connect_address} -daemon=0 -prune=10000 -chain=${chain} -stopatheight=${stop_at_height} -dbcache=${dbcache} -printtoconsole=0" \ -L commit "base,head" } # Main execution -if [ "$#" -ne 11 ]; then - echo "Usage: $0 base_commit head_commit TMP_DATADIR ORIGINAL_DATADIR results_dir png_dir chain stop_at_height connect_address dbcache BINARIES_DIR" +if [ "$#" -ne 10 ]; then + echo "Usage: $0 base_commit head_commit TMP_DATADIR ORIGINAL_DATADIR results_dir chain stop_at_height connect_address dbcache BINARIES_DIR" exit 1 fi -run_benchmark "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" "$9" "${10}" "${11}" +run_benchmark "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" "${9}" "${10}" diff --git a/justfile b/justfile index e87616602ce7..86dbbab91115 100644 --- a/justfile +++ b/justfile @@ -15,11 +15,11 @@ build-assumeutxo-binaries-guix base_commit head_commit: # Run uninstrumented benchmarks on mainnet [group('ci')] -run-mainnet-ci base_commit head_commit TMP_DATADIR ORIGINAL_DATADIR results_file dbcache png_dir binaries_dir: +run-mainnet-ci base_commit head_commit TMP_DATADIR ORIGINAL_DATADIR results_file dbcache binaries_dir: #!/usr/bin/env bash set -euxo pipefail unset SOURCE_DATE_EPOCH # needed to run on NixOS - ./bench-ci/run-benchmark.sh {{ base_commit }} {{ head_commit }} {{ TMP_DATADIR }} {{ ORIGINAL_DATADIR }} {{ results_file }} {{ png_dir }} main 855000 "148.251.128.115:33333" {{ dbcache }} {{ binaries_dir }} + ./bench-ci/run-benchmark.sh {{ base_commit }} {{ head_commit }} {{ TMP_DATADIR }} {{ ORIGINAL_DATADIR }} {{ results_file }} main 855000 "148.251.128.115:33333" {{ dbcache }} {{ binaries_dir }} # Run instrumented benchmarks on mainnet [group('ci')] From 875b48fa41907b5641c36f04671d01add63f0eb6 Mon Sep 17 00:00:00 2001 From: willcl-ark Date: Fri, 14 Feb 2025 09:12:52 +0000 Subject: [PATCH 12/48] include instrumentation in name to avoid conflicts --- .github/workflows/benchmark.yml | 8 ++++---- .github/workflows/publish-results.yml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index cedbd3b562e6..5a4e1a4efe8c 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -45,12 +45,12 @@ jobs: matrix: include: - network: mainnet - name: mainnet-default + name: mainnet-default-uninstrumented timeout: 600 datadir_path: /data/pruned-840k dbcache: 450 - network: mainnet - name: mainnet-large + name: mainnet-large-uninstrumented timeout: 600 datadir_path: /data/pruned-840k dbcache: 32000 @@ -109,12 +109,12 @@ jobs: matrix: include: - network: mainnet - name: mainnet-default + name: mainnet-default-instrumented timeout: 600 datadir_path: /data/pruned-840k dbcache: 450 - network: mainnet - name: mainnet-large + name: mainnet-large-instrumented timeout: 600 datadir_path: /data/pruned-840k dbcache: 32000 diff --git a/.github/workflows/publish-results.yml b/.github/workflows/publish-results.yml index 62076d85b5eb..5cf97732430e 100644 --- a/.github/workflows/publish-results.yml +++ b/.github/workflows/publish-results.yml @@ -12,7 +12,7 @@ jobs: contents: write checks: read env: - NETWORKS: "mainnet-default,mainnet-large" + NETWORKS: "mainnet-default-instrumented,mainnet-large-instrumented,mainnet-default-uninstrumented,mainnet-large-uninstrumented" outputs: speedups: ${{ steps.organize.outputs.speedups }} pr-number: ${{ steps.organize.outputs.pr-number }} From 7b330f73a86f831fcc105576e12873c8b11a4633 Mon Sep 17 00:00:00 2001 From: willcl-ark Date: Fri, 14 Feb 2025 15:53:52 +0000 Subject: [PATCH 13/48] allow failing source guix profile --- bench-ci/guix/libexec/prelude.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench-ci/guix/libexec/prelude.bash b/bench-ci/guix/libexec/prelude.bash index 5756e856b240..0a145479b6a7 100644 --- a/bench-ci/guix/libexec/prelude.bash +++ b/bench-ci/guix/libexec/prelude.bash @@ -10,7 +10,7 @@ source contrib/shell/git-utils.bash # Source guix profile from the runner home directory GUIX_PROFILE=/home/github-runner/.config/guix/current -. "$GUIX_PROFILE/etc/profile" +. "$GUIX_PROFILE/etc/profile" || true echo "Using the following guix command:" command -v guix echo "Guix command symlink points to:" From d61b59b176b20dfabd265cc07a04076805bebf5e Mon Sep 17 00:00:00 2001 From: willcl-ark Date: Tue, 11 Mar 2025 10:01:22 +0000 Subject: [PATCH 14/48] use github guix mirror (faster) --- bench-ci/guix/libexec/prelude.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench-ci/guix/libexec/prelude.bash b/bench-ci/guix/libexec/prelude.bash index 0a145479b6a7..3cf568279f70 100644 --- a/bench-ci/guix/libexec/prelude.bash +++ b/bench-ci/guix/libexec/prelude.bash @@ -82,7 +82,7 @@ fi # across time. time-machine() { # shellcheck disable=SC2086 - guix time-machine --url=https://codeberg.org/guix/guix.git \ + guix time-machine --url=https://github.com/fanquake/guix.git \ --commit=5cb84f2013c5b1e48a7d0e617032266f1e6059e2 \ --cores="$JOBS" \ --keep-failed \ From ce5f3a77f4242d5821dd6cffad7d7822c6fd82fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C5=91rinc?= Date: Thu, 13 Mar 2025 13:41:36 +0100 Subject: [PATCH 15/48] Ignore speedup of instrumented runs --- .github/workflows/publish-results.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/publish-results.yml b/.github/workflows/publish-results.yml index 5cf97732430e..2a66dad88d59 100644 --- a/.github/workflows/publish-results.yml +++ b/.github/workflows/publish-results.yml @@ -262,6 +262,7 @@ jobs: // Set outputs for use in PR comment const resultUrl = `https://${context.repo.owner}.github.io/${context.repo.name}/results/pr-${prNumber}/${runId}/index.html`; const speedupString = Object.entries(combinedResults.speedups) + .filter(([network]) => network.includes('uninstrumented')) .map(([network, speedup]) => `${network}: ${speedup}%`) .join(', '); From 3ff2c3f4385a4feaeae82eb84c5c533e4173cceb Mon Sep 17 00:00:00 2001 From: willcl-ark Date: Mon, 17 Mar 2025 17:19:01 +0000 Subject: [PATCH 16/48] remove nightly upstream sync this just creates needless rebasing. Remove it. --- .github/workflows/sync_upstream.yml | 32 ----------------------------- 1 file changed, 32 deletions(-) delete mode 100644 .github/workflows/sync_upstream.yml diff --git a/.github/workflows/sync_upstream.yml b/.github/workflows/sync_upstream.yml deleted file mode 100644 index 11d9cfafc2e4..000000000000 --- a/.github/workflows/sync_upstream.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: Sync with Upstream -on: - schedule: - - cron: '0 3 * * *' # 03:00 UTC daily - workflow_dispatch: -permissions: - contents: write # Required for pushing to master -jobs: - sync: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - token: ${{ secrets.GITHUB_TOKEN }} - - name: Add upstream remote - run: | - git remote add upstream https://github.com/bitcoin/bitcoin.git - git remote -v - - name: Fetch upstream - run: git fetch upstream - - name: Configure Git - run: | - git config user.name github-actions - git config user.email github-actions@github.com - - name: Rebase onto upstream - run: | - git checkout master - git rebase upstream/master - - name: Push changes - run: git push --force-with-lease origin master From 61f072359051b965c9107675dbec63340a95ba01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C5=91rinc?= Date: Wed, 19 Mar 2025 14:08:59 +0100 Subject: [PATCH 17/48] Add commit id to the plots to make sure they're not overwritten --- .github/workflows/publish-results.yml | 3 ++- bench-ci/parse_and_plot.py | 33 +++++++++++++------------- bench-ci/run-benchmark-instrumented.sh | 2 +- 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/.github/workflows/publish-results.yml b/.github/workflows/publish-results.yml index 2a66dad88d59..2fa320db2072 100644 --- a/.github/workflows/publish-results.yml +++ b/.github/workflows/publish-results.yml @@ -46,7 +46,7 @@ jobs: if [ -d "pngs-${network}" ]; then mkdir -p "${network}-plots" - mv "pngs-${network}"/*.png "${network}-plots/" + mv "pngs-${network}"/* "${network}-plots/" fi done - name: Organize results @@ -172,6 +172,7 @@ jobs: const plotDir = `${resultDir}/${network}-plots`; const plots = fs.existsSync(plotDir) ? fs.readdirSync(plotDir) + .filter(plot => plot.startsWith(`${result.parameters.commit}-`)) .map(plot => ` ${plot} diff --git a/bench-ci/parse_and_plot.py b/bench-ci/parse_and_plot.py index db577417b2ff..4ff4d970a4f9 100755 --- a/bench-ci/parse_and_plot.py +++ b/bench-ci/parse_and_plot.py @@ -114,54 +114,55 @@ def generate_plot(x, y, x_label, y_label, title, output_file): if __name__ == "__main__": - if len(sys.argv) != 3: - print(f"Usage: {sys.argv[0]} ") + if len(sys.argv) != 4: + print(f"Usage: {sys.argv[0]} ") sys.exit(1) - log_file = sys.argv[1] + commit = sys.argv[1] + + log_file = sys.argv[2] if not os.path.isfile(log_file): print(f"File not found: {log_file}") sys.exit(1) - png_dir = sys.argv[2] + png_dir = sys.argv[3] os.makedirs(png_dir, exist_ok=True) update_tip_data, leveldb_compact_data, leveldb_gen_table_data, validation_txadd_data, coindb_write_batch_data, coindb_commit_data = parse_log_file(log_file) times, heights, tx_counts, cache_size, cache_count = zip(*update_tip_data) float_minutes = [(t - times[0]).total_seconds() / 60 for t in times] - generate_plot(float_minutes, heights, "Elapsed minutes", "Block Height", "Block Height vs Time", os.path.join(png_dir, "height_vs_time.png")) - generate_plot(heights, cache_size, "Block Height", "Cache Size (MiB)", "Cache Size vs Block Height", os.path.join(png_dir, "cache_vs_height.png")) - generate_plot(float_minutes, cache_size, "Elapsed minutes", "Cache Size (MiB)", "Cache Size vs Time", os.path.join(png_dir, "cache_vs_time.png")) - generate_plot(heights, tx_counts, "Block Height", "Transaction Count", "Transactions vs Block Height", os.path.join(png_dir, "tx_vs_height.png")) - generate_plot(times, cache_count, "Block Height", "Coins Cache Size", "Coins Cache Size vs Time", os.path.join(png_dir, "coins_cache_vs_time.png")) + generate_plot(float_minutes, heights, "Elapsed minutes", "Block Height", "Block Height vs Time", os.path.join(png_dir, f"{commit}-height_vs_time.png")) + generate_plot(heights, cache_size, "Block Height", "Cache Size (MiB)", "Cache Size vs Block Height", os.path.join(png_dir, f"{commit}-cache_vs_height.png")) + generate_plot(float_minutes, cache_size, "Elapsed minutes", "Cache Size (MiB)", "Cache Size vs Time", os.path.join(png_dir, f"{commit}-cache_vs_time.png")) + generate_plot(heights, tx_counts, "Block Height", "Transaction Count", "Transactions vs Block Height", os.path.join(png_dir, f"{commit}-tx_vs_height.png")) + generate_plot(times, cache_count, "Block Height", "Coins Cache Size", "Coins Cache Size vs Time", os.path.join(png_dir, f"{commit}-coins_cache_vs_time.png")) # LevelDB Compaction and Generated Tables if leveldb_compact_data: leveldb_compact_times = [(t - times[0]).total_seconds() / 60 for t in leveldb_compact_data] leveldb_compact_y = [1 for _ in leveldb_compact_times] # dummy y axis to mark compactions - generate_plot(leveldb_compact_times, leveldb_compact_y, "Elapsed minutes", "LevelDB Compaction", "LevelDB Compaction Events vs Time", os.path.join(png_dir, "leveldb_compact_vs_time.png")) + generate_plot(leveldb_compact_times, leveldb_compact_y, "Elapsed minutes", "LevelDB Compaction", "LevelDB Compaction Events vs Time", os.path.join(png_dir, f"{commit}-leveldb_compact_vs_time.png")) if leveldb_gen_table_data: leveldb_gen_table_times, leveldb_gen_table_keys, leveldb_gen_table_bytes = zip(*leveldb_gen_table_data) leveldb_gen_table_float_minutes = [(t - times[0]).total_seconds() / 60 for t in leveldb_gen_table_times] - generate_plot(leveldb_gen_table_float_minutes, leveldb_gen_table_keys, "Elapsed minutes", "Number of keys", "LevelDB Keys Generated vs Time", os.path.join(png_dir, "leveldb_gen_keys_vs_time.png")) - generate_plot(leveldb_gen_table_float_minutes, leveldb_gen_table_bytes, "Elapsed minutes", "Number of bytes", "LevelDB Bytes Generated vs Time", os.path.join(png_dir, "leveldb_gen_bytes_vs_time.png")) + generate_plot(leveldb_gen_table_float_minutes, leveldb_gen_table_keys, "Elapsed minutes", "Number of keys", "LevelDB Keys Generated vs Time", os.path.join(png_dir, f"{commit}-leveldb_gen_keys_vs_time.png")) + generate_plot(leveldb_gen_table_float_minutes, leveldb_gen_table_bytes, "Elapsed minutes", "Number of bytes", "LevelDB Bytes Generated vs Time", os.path.join(png_dir, f"{commit}-leveldb_gen_bytes_vs_time.png")) # validation mempool add transaction lines if validation_txadd_data: validation_txadd_times = [(t - times[0]).total_seconds() / 60 for t in validation_txadd_data] validation_txadd_y = [1 for _ in validation_txadd_times] # dummy y axis to mark transaction additions - generate_plot(validation_txadd_times, validation_txadd_y, "Elapsed minutes", "Transaction Additions", "Transaction Additions to Mempool vs Time", os.path.join(png_dir, "validation_txadd_vs_time.png")) + generate_plot(validation_txadd_times, validation_txadd_y, "Elapsed minutes", "Transaction Additions", "Transaction Additions to Mempool vs Time", os.path.join(png_dir, f"{commit}-validation_txadd_vs_time.png")) # coindb write batch lines if coindb_write_batch_data: coindb_write_batch_times, is_partial_strs, sizes_mb = zip(*coindb_write_batch_data) coindb_write_batch_float_minutes = [(t - times[0]).total_seconds() / 60 for t in coindb_write_batch_times] - generate_plot(coindb_write_batch_float_minutes, sizes_mb, "Elapsed minutes", "Batch Size MiB", "Coin Database Partial/Final Write Batch Size vs Time", os.path.join(png_dir, "coindb_write_batch_size_vs_time.png")) + generate_plot(coindb_write_batch_float_minutes, sizes_mb, "Elapsed minutes", "Batch Size MiB", "Coin Database Partial/Final Write Batch Size vs Time", os.path.join(png_dir, f"{commit}-coindb_write_batch_size_vs_time.png")) if coindb_commit_data: coindb_commit_times, txout_counts = zip(*coindb_commit_data) coindb_commit_float_minutes = [(t - times[0]).total_seconds() / 60 for t in coindb_commit_times] - generate_plot(coindb_commit_float_minutes, txout_counts, "Elapsed minutes", "Transaction Output Count", "Coin Database Transaction Output Committed vs Time", os.path.join(png_dir, "coindb_commit_txout_vs_time.png")) - + generate_plot(coindb_commit_float_minutes, txout_counts, "Elapsed minutes", "Transaction Output Count", "Coin Database Transaction Output Committed vs Time", os.path.join(png_dir, f"{commit}-coindb_commit_txout_vs_time.png")) print("Plots saved!") \ No newline at end of file diff --git a/bench-ci/run-benchmark-instrumented.sh b/bench-ci/run-benchmark-instrumented.sh index db7f5be6ead1..5c8a0481f4f4 100755 --- a/bench-ci/run-benchmark-instrumented.sh +++ b/bench-ci/run-benchmark-instrumented.sh @@ -75,7 +75,7 @@ conclude_run() { if [ -n "${debug_log}" ]; then echo "Generating plots from ${debug_log}" if [ -x "bench-ci/parse_and_plot.py" ]; then - bench-ci/parse_and_plot.py "${debug_log}" "${PNG_DIR}" + bench-ci/parse_and_plot.py "${commit}" "${debug_log}" "${PNG_DIR}" else ls -al "bench-ci/" echo "parse_and_plot.py not found or not executable, skipping plot generation" From dd762bcd8f78e7b828fa082bf345490c64aba57b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C5=91rinc?= Date: Mon, 7 Apr 2025 13:10:38 +0200 Subject: [PATCH 18/48] Plot coins_cache_vs_height instead of coins_cache_vs_time --- bench-ci/parse_and_plot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench-ci/parse_and_plot.py b/bench-ci/parse_and_plot.py index 4ff4d970a4f9..68a5c0fcb384 100755 --- a/bench-ci/parse_and_plot.py +++ b/bench-ci/parse_and_plot.py @@ -136,7 +136,7 @@ def generate_plot(x, y, x_label, y_label, title, output_file): generate_plot(heights, cache_size, "Block Height", "Cache Size (MiB)", "Cache Size vs Block Height", os.path.join(png_dir, f"{commit}-cache_vs_height.png")) generate_plot(float_minutes, cache_size, "Elapsed minutes", "Cache Size (MiB)", "Cache Size vs Time", os.path.join(png_dir, f"{commit}-cache_vs_time.png")) generate_plot(heights, tx_counts, "Block Height", "Transaction Count", "Transactions vs Block Height", os.path.join(png_dir, f"{commit}-tx_vs_height.png")) - generate_plot(times, cache_count, "Block Height", "Coins Cache Size", "Coins Cache Size vs Time", os.path.join(png_dir, f"{commit}-coins_cache_vs_time.png")) + generate_plot(heights, cache_count, "Block Height", "Coins Cache Size", "Coins Cache Size vs Height", os.path.join(png_dir, f"{commit}-coins_cache_vs_height.png")) # LevelDB Compaction and Generated Tables if leveldb_compact_data: From 1f9f28eea184ed755b681b3bae6187f8ee8a8f8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C5=91rinc?= Date: Mon, 7 Apr 2025 13:29:26 +0200 Subject: [PATCH 19/48] Add vertical lines for major protocol upgrades if this is a height-based plot --- bench-ci/parse_and_plot.py | 89 +++++++++++++++++++++++++++++++++++--- 1 file changed, 84 insertions(+), 5 deletions(-) diff --git a/bench-ci/parse_and_plot.py b/bench-ci/parse_and_plot.py index 68a5c0fcb384..2a8a112cc4cf 100755 --- a/bench-ci/parse_and_plot.py +++ b/bench-ci/parse_and_plot.py @@ -4,6 +4,7 @@ import re import datetime import matplotlib.pyplot as plt +from collections import OrderedDict def parse_updatetip_line(line): @@ -35,6 +36,7 @@ def parse_leveldb_generated_table_line(line): parsed_datetime = datetime.datetime.strptime(iso_str, "%Y-%m-%dT%H:%M:%SZ") return parsed_datetime, int(keys_count_str), int(bytes_count_str) + def parse_validation_txadd_line(line): match = re.match(r'^([\d\-:TZ]+) \[validation] TransactionAddedToMempool: txid=.+wtxid=.+', line) if not match: @@ -61,6 +63,7 @@ def parse_coindb_commit_line(line): parsed_datetime = datetime.datetime.strptime(iso_str, "%Y-%m-%dT%H:%M:%SZ") return parsed_datetime, int(txout_count_str) + def parse_log_file(log_file): with open(log_file, 'r', encoding='utf-8') as f: update_tip_data = [] @@ -94,7 +97,7 @@ def parse_log_file(log_file): return update_tip_data, leveldb_compact_data, leveldb_gen_table_data, validation_txadd_data, coindb_write_batch_data, coindb_commit_data -def generate_plot(x, y, x_label, y_label, title, output_file): +def generate_plot(x, y, x_label, y_label, title, output_file, is_height_based=False): if not x or not y: print(f"Skipping plot '{title}' as there is no data.") return @@ -105,6 +108,82 @@ def generate_plot(x, y, x_label, y_label, title, output_file): plt.xlabel(x_label, fontsize=16) plt.ylabel(y_label, fontsize=16) plt.grid(True) + + # Make sure the x-axis covers the full data range + min_x, max_x = min(x), max(x) + plt.xlim(min_x, max_x) + + # Add vertical lines for major protocol upgrades if this is a height-based plot + if is_height_based: + # Define all notable heights from the chainparams file + fork_heights = OrderedDict([ + ('BIP34', 227931), # Block v2, coinbase includes height + ('BIP66', 363725), # Strict DER signatures + ('BIP65', 388381), # OP_CHECKLOCKTIMEVERIFY + ('CSV', 419328), # BIP68, 112, 113 - OP_CHECKSEQUENCEVERIFY + ('Segwit', 481824), # BIP141, 143, 144, 145 - Segregated Witness + ('Taproot', 709632), # BIP341, 342 - Schnorr signatures & Taproot + ('Halving 1', 210000), # First halving + ('Halving 2', 420000), # Second halving + ('Halving 3', 630000), # Third halving + ('Halving 4', 840000), # Fourth halving + ]) + + # Colors for the different types of events + fork_colors = { + 'BIP34': 'blue', + 'BIP66': 'blue', + 'BIP65': 'blue', + 'CSV': 'blue', + 'Segwit': 'green', + 'Taproot': 'red', + 'Halving 1': 'purple', + 'Halving 2': 'purple', + 'Halving 3': 'purple', + 'Halving 4': 'purple', + } + + # Line styles for different types of events + fork_styles = { + 'BIP34': '--', + 'BIP66': '--', + 'BIP65': '--', + 'CSV': '--', + 'Segwit': '--', + 'Taproot': '--', + 'Halving 1': ':', + 'Halving 2': ':', + 'Halving 3': ':', + 'Halving 4': ':', + } + + max_y = max(y) + + # Position text labels at different heights to avoid overlap + text_positions = {} + position_increment = max_y * 0.05 + current_position = max_y * 0.9 + + # Add lines for forks that are in range + for fork_name, height in fork_heights.items(): + if min_x <= height <= max_x: + plt.axvline(x=height, color=fork_colors[fork_name], + linestyle=fork_styles[fork_name]) + + # Avoid label overlaps by staggering vertical positions + if height in text_positions: + # If this x position already has a label, adjust position + text_positions[height] -= position_increment + else: + text_positions[height] = current_position + current_position -= position_increment + if current_position < max_y * 0.1: + current_position = max_y * 0.9 # Reset if we're too low + + plt.text(height, text_positions[height], f'{fork_name} ({height})', + rotation=90, verticalalignment='top', + color=fork_colors[fork_name]) + plt.xticks(rotation=90, fontsize=12) plt.yticks(fontsize=12) plt.tight_layout() @@ -133,10 +212,10 @@ def generate_plot(x, y, x_label, y_label, title, output_file): float_minutes = [(t - times[0]).total_seconds() / 60 for t in times] generate_plot(float_minutes, heights, "Elapsed minutes", "Block Height", "Block Height vs Time", os.path.join(png_dir, f"{commit}-height_vs_time.png")) - generate_plot(heights, cache_size, "Block Height", "Cache Size (MiB)", "Cache Size vs Block Height", os.path.join(png_dir, f"{commit}-cache_vs_height.png")) + generate_plot(heights, cache_size, "Block Height", "Cache Size (MiB)", "Cache Size vs Block Height", os.path.join(png_dir, f"{commit}-cache_vs_height.png"), is_height_based=True) generate_plot(float_minutes, cache_size, "Elapsed minutes", "Cache Size (MiB)", "Cache Size vs Time", os.path.join(png_dir, f"{commit}-cache_vs_time.png")) - generate_plot(heights, tx_counts, "Block Height", "Transaction Count", "Transactions vs Block Height", os.path.join(png_dir, f"{commit}-tx_vs_height.png")) - generate_plot(heights, cache_count, "Block Height", "Coins Cache Size", "Coins Cache Size vs Height", os.path.join(png_dir, f"{commit}-coins_cache_vs_height.png")) + generate_plot(heights, tx_counts, "Block Height", "Transaction Count", "Transactions vs Block Height", os.path.join(png_dir, f"{commit}-tx_vs_height.png"), is_height_based=True) + generate_plot(heights, cache_count, "Block Height", "Coins Cache Size", "Coins Cache Size vs Height", os.path.join(png_dir, f"{commit}-coins_cache_vs_height.png"), is_height_based=True) # LevelDB Compaction and Generated Tables if leveldb_compact_data: @@ -165,4 +244,4 @@ def generate_plot(x, y, x_label, y_label, title, output_file): coindb_commit_float_minutes = [(t - times[0]).total_seconds() / 60 for t in coindb_commit_times] generate_plot(coindb_commit_float_minutes, txout_counts, "Elapsed minutes", "Transaction Output Count", "Coin Database Transaction Output Committed vs Time", os.path.join(png_dir, f"{commit}-coindb_commit_txout_vs_time.png")) - print("Plots saved!") \ No newline at end of file + print("Plots saved!") From 6e37e2431ac41ff3ced2014852ae8ac99d6c8414 Mon Sep 17 00:00:00 2001 From: will Date: Fri, 5 Dec 2025 09:42:48 +0000 Subject: [PATCH 20/48] drop page cache before each run --- bench-ci/run-benchmark-instrumented.sh | 2 ++ bench-ci/run-benchmark.sh | 2 ++ 2 files changed, 4 insertions(+) diff --git a/bench-ci/run-benchmark-instrumented.sh b/bench-ci/run-benchmark-instrumented.sh index 5c8a0481f4f4..29d9b51875cb 100755 --- a/bench-ci/run-benchmark-instrumented.sh +++ b/bench-ci/run-benchmark-instrumented.sh @@ -59,6 +59,8 @@ prepare_run() { clean_datadir "${TMP_DATADIR}" # Don't copy hidden files so use * taskset -c 0-15 cp -r "$ORIGINAL_DATADIR"/* "$TMP_DATADIR" + # Clear page caches + sync && echo 3 | sudo tee /proc/sys/vm/drop_caches > /dev/null clean_logs "${TMP_DATADIR}" } diff --git a/bench-ci/run-benchmark.sh b/bench-ci/run-benchmark.sh index d0e3ba60e1f1..d008dc0fa799 100755 --- a/bench-ci/run-benchmark.sh +++ b/bench-ci/run-benchmark.sh @@ -58,6 +58,8 @@ prepare_run() { clean_datadir "${TMP_DATADIR}" # Don't copy hidden files so use * taskset -c 0-15 cp -r "$ORIGINAL_DATADIR"/* "$TMP_DATADIR" + # Clear page caches + sync && echo 3 | sudo tee /proc/sys/vm/drop_caches > /dev/null clean_logs "${TMP_DATADIR}" } From beccd59dab3ad2e995e75a04ae5aafdb570b7a85 Mon Sep 17 00:00:00 2001 From: will Date: Fri, 5 Dec 2025 09:49:30 +0000 Subject: [PATCH 21/48] refactor shared functions --- bench-ci/prelude.sh | 78 +++++++++++++++++++++++ bench-ci/run-benchmark-instrumented.sh | 86 ++------------------------ bench-ci/run-benchmark.sh | 83 ++----------------------- 3 files changed, 89 insertions(+), 158 deletions(-) create mode 100644 bench-ci/prelude.sh diff --git a/bench-ci/prelude.sh b/bench-ci/prelude.sh new file mode 100644 index 000000000000..4424f8ab25b0 --- /dev/null +++ b/bench-ci/prelude.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# Shared functions + +set -euxo pipefail + +clean_datadir() { + set -euxo pipefail + + local TMP_DATADIR="$1" + + mkdir -p "${TMP_DATADIR}" + + # If we're in CI, clean without confirmation + if [ -n "${CI:-}" ]; then + rm -Rf "${TMP_DATADIR:?}"/* + else + read -rp "Are you sure you want to delete everything in ${TMP_DATADIR}? [y/N] " response + if [[ "$response" =~ ^[Yy]$ ]]; then + rm -Rf "${TMP_DATADIR:?}"/* + else + echo "Aborting..." + exit 1 + fi + fi +} + +clean_logs() { + set -euxo pipefail + + local TMP_DATADIR="$1" + local logfile="${TMP_DATADIR}/debug.log" + + echo "Checking for ${logfile}" + if [ -e "${logfile}" ]; then + echo "Removing ${logfile}" + rm "${logfile}" + fi +} + +# Executes once before each *set* of timing runs. +setup_run() { + set -euxo pipefail + + local TMP_DATADIR="$1" + clean_datadir "${TMP_DATADIR}" +} + +# Executes before each timing run. +prepare_run() { + set -euxo pipefail + + local TMP_DATADIR="$1" + local ORIGINAL_DATADIR="$2" + + clean_datadir "${TMP_DATADIR}" + # Don't copy hidden files so use * + taskset -c 0-15 cp -r "$ORIGINAL_DATADIR"/* "$TMP_DATADIR" + # Clear page caches + sync && echo 3 | sudo tee /proc/sys/vm/drop_caches > /dev/null + clean_logs "${TMP_DATADIR}" +} + +# Executes after the completion of all benchmarking runs for each individual +# command to be benchmarked. +cleanup_run() { + set -euxo pipefail + local TMP_DATADIR="$1" + clean_datadir "${TMP_DATADIR}" +} + +# Export all shared functions for use by hyperfine subshells +export_shared_functions() { + export -f clean_datadir + export -f clean_logs + export -f setup_run + export -f prepare_run + export -f cleanup_run +} diff --git a/bench-ci/run-benchmark-instrumented.sh b/bench-ci/run-benchmark-instrumented.sh index 29d9b51875cb..6cb307bf395b 100755 --- a/bench-ci/run-benchmark-instrumented.sh +++ b/bench-ci/run-benchmark-instrumented.sh @@ -2,69 +2,10 @@ set -euxo pipefail -# Helper function to check and clean datadir -clean_datadir() { - set -euxo pipefail - - local TMP_DATADIR="$1" - - # Create the directory if it doesn't exist - mkdir -p "${TMP_DATADIR}" - - # If we're in CI, clean without confirmation - if [ -n "${CI:-}" ]; then - rm -Rf "${TMP_DATADIR:?}"/* - else - read -rp "Are you sure you want to delete everything in ${TMP_DATADIR}? [y/N] " response - if [[ "$response" =~ ^[Yy]$ ]]; then - rm -Rf "${TMP_DATADIR:?}"/* - else - echo "Aborting..." - exit 1 - fi - fi -} - -# Helper function to clear logs -clean_logs() { - set -euxo pipefail - - local TMP_DATADIR="$1" - local logfile="${TMP_DATADIR}/debug.log" - - echo "Checking for ${logfile}" - if [ -e "${logfile}" ]; then - echo "Removing ${logfile}" - rm "${logfile}" - fi -} +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/prelude.sh" -# Execute CMD before each set of timing runs. -setup_run() { - set -euxo pipefail - - local TMP_DATADIR="$1" - local commit="$2" - clean_datadir "${TMP_DATADIR}" -} - -# Execute CMD before each timing run. -prepare_run() { - set -euxo pipefail - - local TMP_DATADIR="$1" - local ORIGINAL_DATADIR="$2" - - # Run the actual preparation steps - clean_datadir "${TMP_DATADIR}" - # Don't copy hidden files so use * - taskset -c 0-15 cp -r "$ORIGINAL_DATADIR"/* "$TMP_DATADIR" - # Clear page caches - sync && echo 3 | sudo tee /proc/sys/vm/drop_caches > /dev/null - clean_logs "${TMP_DATADIR}" -} - -# Executed after each timing run +# Executed after each timing run - generates plots and handles flamegraph conclude_run() { set -euxo pipefail @@ -93,17 +34,6 @@ conclude_run() { fi } -# Execute CMD after the completion of all benchmarking runs for each individual -# command to be benchmarked. -cleanup_run() { - set -euxo pipefail - - local TMP_DATADIR="$1" - - # Clean up the datadir - clean_datadir "${TMP_DATADIR}" -} - run_benchmark() { local base_commit="$1" local head_commit="$2" @@ -118,12 +48,8 @@ run_benchmark() { local BINARIES_DIR="${11}" # Export functions so they can be used by hyperfine - export -f setup_run - export -f prepare_run + export_shared_functions export -f conclude_run - export -f cleanup_run - export -f clean_datadir - export -f clean_logs # Debug: Print all variables being used echo "=== Debug Information ===" @@ -138,12 +64,12 @@ run_benchmark() { echo "stop_at_height: ${stop_at_height}" echo "connect_address: ${connect_address}" echo "dbcache: ${dbcache}" - echo "\n" + printf '\n' # Run hyperfine hyperfine \ --shell=bash \ - --setup "setup_run ${TMP_DATADIR} {commit}" \ + --setup "setup_run ${TMP_DATADIR}" \ --prepare "prepare_run ${TMP_DATADIR} ${ORIGINAL_DATADIR}" \ --conclude "conclude_run {commit} ${TMP_DATADIR} ${png_dir}" \ --cleanup "cleanup_run ${TMP_DATADIR}" \ diff --git a/bench-ci/run-benchmark.sh b/bench-ci/run-benchmark.sh index d008dc0fa799..bce7857919ac 100755 --- a/bench-ci/run-benchmark.sh +++ b/bench-ci/run-benchmark.sh @@ -2,84 +2,15 @@ set -euxo pipefail -# Helper function to check and clean datadir -clean_datadir() { - set -euxo pipefail - - local TMP_DATADIR="$1" - - # Create the directory if it doesn't exist - mkdir -p "${TMP_DATADIR}" - - # If we're in CI, clean without confirmation - if [ -n "${CI:-}" ]; then - rm -Rf "${TMP_DATADIR:?}"/* - else - read -rp "Are you sure you want to delete everything in ${TMP_DATADIR}? [y/N] " response - if [[ "$response" =~ ^[Yy]$ ]]; then - rm -Rf "${TMP_DATADIR:?}"/* - else - echo "Aborting..." - exit 1 - fi - fi -} - -# Helper function to clear logs -clean_logs() { - set -euxo pipefail - - local TMP_DATADIR="$1" - local logfile="${TMP_DATADIR}/debug.log" - - echo "Checking for ${logfile}" - if [ -e "${logfile}" ]; then - echo "Removing ${logfile}" - rm "${logfile}" - fi -} - -# Execute CMD before each set of timing runs. -setup_run() { - set -euxo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/prelude.sh" - local TMP_DATADIR="$1" - clean_datadir "${TMP_DATADIR}" -} - -# Execute CMD before each timing run. -prepare_run() { - set -euxo pipefail - - local TMP_DATADIR="$1" - local ORIGINAL_DATADIR="$2" - - # Run the actual preparation steps - clean_datadir "${TMP_DATADIR}" - # Don't copy hidden files so use * - taskset -c 0-15 cp -r "$ORIGINAL_DATADIR"/* "$TMP_DATADIR" - # Clear page caches - sync && echo 3 | sudo tee /proc/sys/vm/drop_caches > /dev/null - clean_logs "${TMP_DATADIR}" -} - -# Executed after each timing run +# Executed after each timing run (no-op for uninstrumented) conclude_run() { set -euxo pipefail return 0 } -# Execute CMD after the completion of all benchmarking runs for each individual -# command to be benchmarked. -cleanup_run() { - set -euxo pipefail - - local TMP_DATADIR="$1" - - # Clean up the datadir - clean_datadir "${TMP_DATADIR}" -} - run_benchmark() { local base_commit="$1" local head_commit="$2" @@ -93,11 +24,7 @@ run_benchmark() { local BINARIES_DIR="${10}" # Export functions so they can be used by hyperfine - export -f setup_run - export -f prepare_run - export -f cleanup_run - export -f clean_datadir - export -f clean_logs + export_shared_functions # Debug: Print all variables being used echo "=== Debug Information ===" @@ -111,7 +38,7 @@ run_benchmark() { echo "stop_at_height: ${stop_at_height}" echo "connect_address: ${connect_address}" echo "dbcache: ${dbcache}" - printf \n + printf '\n' # Run hyperfine hyperfine \ From 95ccc4434ade0374166af60138058b818475790f Mon Sep 17 00:00:00 2001 From: will Date: Fri, 5 Dec 2025 20:49:26 +0000 Subject: [PATCH 22/48] fixup! drop page cache before each run --- bench-ci/prelude.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench-ci/prelude.sh b/bench-ci/prelude.sh index 4424f8ab25b0..98a5232b40fb 100644 --- a/bench-ci/prelude.sh +++ b/bench-ci/prelude.sh @@ -56,7 +56,7 @@ prepare_run() { # Don't copy hidden files so use * taskset -c 0-15 cp -r "$ORIGINAL_DATADIR"/* "$TMP_DATADIR" # Clear page caches - sync && echo 3 | sudo tee /proc/sys/vm/drop_caches > /dev/null + /run/wrappers/bin/drop-caches clean_logs "${TMP_DATADIR}" } From 3577c3c6207520489b1428a2178db3955f93dbd1 Mon Sep 17 00:00:00 2001 From: will Date: Fri, 5 Dec 2025 21:56:24 +0000 Subject: [PATCH 23/48] use SCHED_OTHER in instrumented run --- bench-ci/run-benchmark-instrumented.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench-ci/run-benchmark-instrumented.sh b/bench-ci/run-benchmark-instrumented.sh index 6cb307bf395b..1f4be0b0b2dd 100755 --- a/bench-ci/run-benchmark-instrumented.sh +++ b/bench-ci/run-benchmark-instrumented.sh @@ -78,7 +78,7 @@ run_benchmark() { --show-output \ --command-name "base (${base_commit})" \ --command-name "head (${head_commit})" \ - "taskset -c 1 flamegraph --palette bitcoin --title 'bitcoind IBD@{commit}' -c 'record -F 101 --call-graph fp' -- taskset -c 2-15 chrt -r 1 ${BINARIES_DIR}/{commit}/bitcoind -datadir=${TMP_DATADIR} -connect=${connect_address} -daemon=0 -prune=10000 -chain=${chain} -stopatheight=${stop_at_height} -dbcache=${dbcache} -printtoconsole=0 -debug=coindb -debug=leveldb -debug=bench -debug=validation" \ + "taskset -c 1 flamegraph --palette bitcoin --title 'bitcoind IBD@{commit}' -c 'record -F 101 --call-graph fp' -- taskset -c 2-15 chrt -o 0 ${BINARIES_DIR}/{commit}/bitcoind -datadir=${TMP_DATADIR} -connect=${connect_address} -daemon=0 -prune=10000 -chain=${chain} -stopatheight=${stop_at_height} -dbcache=${dbcache} -printtoconsole=0 -debug=coindb -debug=leveldb -debug=bench -debug=validation" \ -L commit "base,head" } From 1557d2358eaccf4320ff70fb7502f7902e09b45f Mon Sep 17 00:00:00 2001 From: will Date: Sat, 6 Dec 2025 12:03:40 +0000 Subject: [PATCH 24/48] test clearer results chart --- .github/workflows/publish-results.yml | 163 ++++++++++++++++---------- 1 file changed, 102 insertions(+), 61 deletions(-) diff --git a/.github/workflows/publish-results.yml b/.github/workflows/publish-results.yml index 2fa320db2072..11d0327192c5 100644 --- a/.github/workflows/publish-results.yml +++ b/.github/workflows/publish-results.yml @@ -141,6 +141,14 @@ jobs: fs.writeFileSync(`${resultDir}/results.json`, JSON.stringify(combinedResults, null, 2)); // Create index.html for this run + // Sort results by network then by command type (base first) + const sortedResults = combinedResults.results.sort((a, b) => { + if (a.network !== b.network) return a.network.localeCompare(b.network); + const aIsBase = a.command.includes('base'); + const bIsBase = b.command.includes('base'); + return bIsBase - aIsBase; // base first + }); + const indexHtml = ` @@ -152,70 +160,103 @@ jobs:

Benchmark Results

PR #${prNumber} - Run ${runId}

- ${networks.map(network => ` -
-

- ${network} Results - ${combinedResults.speedups[network] ? - `(${combinedResults.speedups[network]}% speedup)` - : ''} -

-
- ${combinedResults.results - .filter(result => result.network === network) - .map(result => { - const commitShortId = result.parameters.commit.slice(0, 8); - const flameGraphFile = `${network}-${result.parameters.commit}-flamegraph.svg`; - const flameGraphPath = `${resultDir}/${network}-${result.parameters.commit}-flamegraph.svg`; - // Query PNG files dynamically - const plotDir = `${resultDir}/${network}-plots`; - const plots = fs.existsSync(plotDir) - ? fs.readdirSync(plotDir) - .filter(plot => plot.startsWith(`${result.parameters.commit}-`)) - .map(plot => ` - - ${plot} - - `) - .join('') - : ''; + +

Run Data

+
+ + + + + + + + + + + + + ${sortedResults.map(result => ` + + + + + + + + + `).join('')} + +
NetworkCommandMean (s)Std DevUser (s)System (s)
${result.network} + ${result.command.replace( + /\((\w+)\)/, + (_, commit) => `(${commit.slice(0, 8)})` + )} + ${result.mean.toFixed(3)}${result.stddev?.toFixed(3) || 'N/A'}${result.user.toFixed(3)}${result.system.toFixed(3)}
+
+ + +

Speedup Summary

+
+ + + + + + + + + ${Object.entries(combinedResults.speedups).map(([network, speedup]) => ` + + + + + `).join('')} + +
NetworkSpeedup (%)
${network}${speedup}%
+
+ + + ${networks.filter(network => network.includes('instrumented')).map(network => { + const networkResults = combinedResults.results.filter(r => r.network === network); + const graphsHtml = networkResults.map(result => { + const flameGraphFile = `${network}-${result.parameters.commit}-flamegraph.svg`; + const flameGraphPath = `${resultDir}/${network}-${result.parameters.commit}-flamegraph.svg`; + + const plotDir = `${resultDir}/${network}-plots`; + const plots = fs.existsSync(plotDir) + ? fs.readdirSync(plotDir) + .filter(plot => plot.startsWith(`${result.parameters.commit}-`)) + .map(plot => ` + + ${plot} + + `) + .join('') + : ''; + + if (!fs.existsSync(flameGraphPath) && !plots) return ''; + + return ` +
+

${result.command.replace(/\((\w+)\)/, (_, commit) => `(${commit.slice(0, 8)})`)}

+ ${fs.existsSync(flameGraphPath) ? ` + + ` : ''} + ${plots} +
+ `; + }).join(''); + + if (!graphsHtml.trim()) return ''; - return ` - - - - - - - - - - - - - - - - - - - -
CommandMean (s)Std DevUser (s)System (s)
- ${result.command.replace( - /\((\w+)\)/, - (_, commit) => `(${commit.slice(0, 8)})` - )} - ${result.mean.toFixed(3)}${result.stddev?.toFixed(3) || 'N/A'}${result.user.toFixed(3)}${result.system.toFixed(3)}
- ${fs.existsSync(flameGraphPath) ? ` - - ` : ''} - ${plots} - `; - }).join('')} + return ` +
+

${network} Graphs

+ ${graphsHtml}
-
- `).join('')} + `; + }).join('')}
From 2c754e6a02d26d557df0c8d9962b12d7dcf1878d Mon Sep 17 00:00:00 2001 From: willcl-ark Date: Thu, 4 Dec 2025 11:06:06 +0000 Subject: [PATCH 25/48] bump to nixos 25.11 --- shell.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shell.nix b/shell.nix index b1914a75d99a..35a9fdbf49c2 100644 --- a/shell.nix +++ b/shell.nix @@ -1,6 +1,6 @@ # Copyright 0xB10C, willcl-ark { pkgs ? import - (fetchTarball "https://github.com/nixos/nixpkgs/archive/nixos-24.11.tar.gz") + (fetchTarball "https://github.com/nixos/nixpkgs/archive/nixos-25.11.tar.gz") { }, }: let inherit (pkgs.lib) optionals strings; From c6aaa0031fc0aed0883d68538049a3edfb083e22 Mon Sep 17 00:00:00 2001 From: will Date: Mon, 8 Dec 2025 09:57:45 +0000 Subject: [PATCH 26/48] add nix flake --- flake.lock | 27 +++++++++ flake.nix | 167 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 194 insertions(+) create mode 100644 flake.lock create mode 100644 flake.nix diff --git a/flake.lock b/flake.lock new file mode 100644 index 000000000000..fc1308c520fa --- /dev/null +++ b/flake.lock @@ -0,0 +1,27 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1764983851, + "narHash": "sha256-y7RPKl/jJ/KAP/VKLMghMgXTlvNIJMHKskl8/Uuar7o=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "d9bc5c7dceb30d8d6fafa10aeb6aa8a48c218454", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-25.11", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 000000000000..f769290ebca5 --- /dev/null +++ b/flake.nix @@ -0,0 +1,167 @@ +{ + description = "bitcoind for benchmarking"; + + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.11"; + + outputs = + { self, nixpkgs }: + let + systems = [ + "x86_64-linux" + "aarch64-darwin" + ]; + + forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f system); + + pkgsFor = system: import nixpkgs { inherit system; }; + + mkBitcoinCore = + system: + let + pkgs = pkgsFor system; + inherit (pkgs) lib; + + pname = "bitcoin-core"; + version = self.shortRev or "dirty"; + + CFlags = toString [ + "-O2" + "-g" + ]; + CXXFlags = "${CFlags} -fno-omit-frame-pointer"; + + nativeBuildInputs = with pkgs; [ + cmake + ninja + pkg-config + python3 + ]; + + buildInputs = with pkgs; [ + boost188.dev + libevent.dev + ]; + + cmakeFlags = [ + "-DBUILD_BENCH=OFF" + "-DBUILD_BITCOIN_BIN=OFF" + "-DBUILD_CLI=OFF" + "-DBUILD_DAEMON=ON" + "-DBUILD_FUZZ_BINARY=OFF" + "-DBUILD_GUI_TESTS=OFF" + "-DBUILD_TESTS=OFF" + "-DBUILD_TX=OFF" + "-DBUILD_UTIL=OFF" + "-DBUILD_WALLET_TOOL=OFF" + "-DCMAKE_BUILD_TYPE=RelWithDebInfo" + "-DCMAKE_SKIP_RPATH=ON" + "-DENABLE_EXTERNAL_SIGNER=OFF" + "-DENABLE_IPC=OFF" + "-DENABLE_WALLET=OFF" + "-DREDUCE_EXPORTS=ON" + "-DWITH_ZMQ=OFF" + ]; + in + pkgs.stdenv.mkDerivation { + inherit + pname + version + nativeBuildInputs + buildInputs + cmakeFlags + ; + + preConfigure = '' + cmakeFlagsArray+=( + "-DAPPEND_CFLAGS=${CFlags}" + "-DAPPEND_CXXFLAGS=${CXXFlags}" + "-DAPPEND_LDFLAGS=-Wl,--as-needed -Wl,-O2" + ) + ''; + + src = builtins.path { + path = ./.; + name = "source"; + }; + + env = { + CMAKE_GENERATOR = "Ninja"; + LC_ALL = "C"; + LIBRARY_PATH = ""; + CPATH = ""; + C_INCLUDE_PATH = ""; + CPLUS_INCLUDE_PATH = ""; + OBJC_INCLUDE_PATH = ""; + OBJCPLUS_INCLUDE_PATH = ""; + }; + + dontStrip = true; + + meta = { + description = "bitcoind for benchmarking"; + homepage = "https://bitcoincore.org/"; + license = lib.licenses.mit; + }; + }; + in + { + packages = forAllSystems (system: { + default = mkBitcoinCore system; + }); + + formatter = forAllSystems (system: (pkgsFor system).nixfmt-tree); + + devShells = forAllSystems ( + system: + let + pkgs = pkgsFor system; + inherit (pkgs) stdenv; + + # Override the default cargo-flamegraph with a custom fork including bitcoin highlighting + cargo-flamegraph = pkgs.rustPlatform.buildRustPackage rec { + pname = "flamegraph"; + version = "bitcoin-core"; + + src = pkgs.fetchFromGitHub { + owner = "willcl-ark"; + repo = "flamegraph"; + rev = "bitcoin-core"; + sha256 = "sha256-tQbr3MYfAiOxeT12V9au5KQK5X5JeGuV6p8GR/Sgen4="; + }; + + doCheck = false; + cargoHash = "sha256-QWPqTyTFSZNJNayNqLmsQSu0rX26XBKfdLROZ9tRjrg="; + + nativeBuildInputs = pkgs.lib.optionals stdenv.hostPlatform.isLinux [ pkgs.makeWrapper ]; + buildInputs = pkgs.lib.optionals stdenv.hostPlatform.isDarwin [ + pkgs.darwin.apple_sdk.frameworks.Security + ]; + + postFixup = pkgs.lib.optionalString stdenv.hostPlatform.isLinux '' + wrapProgram $out/bin/cargo-flamegraph \ + --set-default PERF ${pkgs.perf}/bin/perf + wrapProgram $out/bin/flamegraph \ + --set-default PERF ${pkgs.perf}/bin/perf + ''; + }; + in + { + default = pkgs.mkShell { + buildInputs = [ + # Benchmarking + pkgs.cargo-flamegraph + pkgs.flamegraph + pkgs.hyperfine + pkgs.jq + pkgs.perf + pkgs.perf-tools + pkgs.util-linux + + # Binary patching + pkgs.patchelf + ]; + }; + } + ); + }; +} From eae6b40bb4c115fd622a1776f02a110b7be081a1 Mon Sep 17 00:00:00 2001 From: will Date: Mon, 8 Dec 2025 10:17:48 +0000 Subject: [PATCH 27/48] switch to nix build --- .github/workflows/benchmark.yml | 42 +- bench-ci/build_binaries.sh | 20 +- bench-ci/guix/INSTALL.md | 814 ------------------ bench-ci/guix/README.md | 430 --------- bench-ci/guix/guix-attest | 263 ------ bench-ci/guix/guix-build | 474 ---------- bench-ci/guix/guix-clean | 83 -- bench-ci/guix/guix-codesign | 384 --------- bench-ci/guix/guix-verify | 174 ---- bench-ci/guix/libexec/build.sh | 416 --------- bench-ci/guix/libexec/codesign.sh | 153 ---- bench-ci/guix/libexec/prelude.bash | 114 --- bench-ci/guix/manifest.scm | 610 ------------- .../patches/binutils-unaligned-default.patch | 22 - .../guix/patches/gcc-remap-guix-store.patch | 20 - .../guix/patches/glibc-2.42-guix-prefix.patch | 47 - bench-ci/guix/patches/glibc-guix-prefix.patch | 16 - .../guix/patches/glibc-riscv-jumptarget.patch | 57 -- bench-ci/guix/patches/lief-scikit-0-9.patch | 21 - .../patches/oscrypto-hard-code-openssl.patch | 13 - .../winpthreads-remap-guix-store.patch | 17 - bench-ci/guix/security-check.py | 297 ------- bench-ci/guix/symbol-check.py | 338 -------- flake.nix | 5 +- justfile | 3 +- shell.nix | 101 --- 26 files changed, 25 insertions(+), 4909 deletions(-) delete mode 100644 bench-ci/guix/INSTALL.md delete mode 100644 bench-ci/guix/README.md delete mode 100755 bench-ci/guix/guix-attest delete mode 100755 bench-ci/guix/guix-build delete mode 100755 bench-ci/guix/guix-clean delete mode 100755 bench-ci/guix/guix-codesign delete mode 100755 bench-ci/guix/guix-verify delete mode 100755 bench-ci/guix/libexec/build.sh delete mode 100755 bench-ci/guix/libexec/codesign.sh delete mode 100644 bench-ci/guix/libexec/prelude.bash delete mode 100644 bench-ci/guix/manifest.scm delete mode 100644 bench-ci/guix/patches/binutils-unaligned-default.patch delete mode 100644 bench-ci/guix/patches/gcc-remap-guix-store.patch delete mode 100644 bench-ci/guix/patches/glibc-2.42-guix-prefix.patch delete mode 100644 bench-ci/guix/patches/glibc-guix-prefix.patch delete mode 100644 bench-ci/guix/patches/glibc-riscv-jumptarget.patch delete mode 100644 bench-ci/guix/patches/lief-scikit-0-9.patch delete mode 100644 bench-ci/guix/patches/oscrypto-hard-code-openssl.patch delete mode 100644 bench-ci/guix/patches/winpthreads-remap-guix-store.patch delete mode 100755 bench-ci/guix/security-check.py delete mode 100755 bench-ci/guix/symbol-check.py delete mode 100644 shell.nix diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 5a4e1a4efe8c..65e667e88fe1 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -7,7 +7,6 @@ jobs: build-binaries: runs-on: [self-hosted, linux, x64] env: - NIX_PATH: nixpkgs=channel:nixos-unstable BASE_SHA: ${{ github.event.pull_request.base.sha }} steps: - name: Checkout repo @@ -18,22 +17,17 @@ jobs: run: | echo "CHECKOUT_COMMIT=$(git rev-parse HEAD)" >> "$GITHUB_ENV" git fetch --depth=1 origin ${{ github.event.pull_request.base.sha }} - - name: Setup ccache - run: | - mkdir -p /data/ccache - export CCACHE_DIR=/data/ccache - export CCACHE_MAXSIZE=50G - ccache -M 50G - ccache -s - name: Build both binaries env: - CCACHE_DIR: /data/ccache + CCACHE_DIR: /nix/var/cache/ccache run: | mkdir -p ${{ runner.temp }}/binaries/base mkdir -p ${{ runner.temp }}/binaries/head - nix-shell --command "just build-assumeutxo-binaries-guix $BASE_SHA $CHECKOUT_COMMIT" - cp binaries/base/bitcoind ${{ runner.temp }}/binaries/base/bitcoind - cp binaries/head/bitcoind ${{ runner.temp }}/binaries/head/bitcoind + nix develop --command bash -c ' + just build-binaries $BASE_SHA $CHECKOUT_COMMIT + cp binaries/base/bitcoind ${{ runner.temp }}/binaries/base/bitcoind + cp binaries/head/bitcoind ${{ runner.temp }}/binaries/head/bitcoind + ' - name: Upload binaries uses: actions/upload-artifact@v4 with: @@ -57,7 +51,6 @@ jobs: runs-on: [self-hosted, linux, x64] timeout-minutes: ${{ matrix.timeout }} env: - NIX_PATH: nixpkgs=channel:nixos-unstable ORIGINAL_DATADIR: ${{ matrix.datadir_path }} BASE_SHA: ${{ github.event.pull_request.base.sha }} steps: @@ -83,9 +76,8 @@ jobs: TMP_DATADIR: "${{ runner.temp }}/base_datadir" BINARIES_DIR: "${{ runner.temp }}/binaries" run: | - env mkdir -p "$TMP_DATADIR" - nix-shell --command "just run-${{ matrix.network }}-ci $BASE_SHA $CHECKOUT_COMMIT $TMP_DATADIR $ORIGINAL_DATADIR ${{ runner.temp }}/results.json ${{ matrix.dbcache }} $BINARIES_DIR" + nix develop --command just run-${{ matrix.network }}-ci $BASE_SHA $CHECKOUT_COMMIT $TMP_DATADIR $ORIGINAL_DATADIR ${{ runner.temp }}/results.json ${{ matrix.dbcache }} $BINARIES_DIR - uses: actions/upload-artifact@v4 with: name: result-${{ matrix.name }} @@ -95,9 +87,11 @@ jobs: GITHUB_CONTEXT: ${{ toJSON(github) }} RUNNER_CONTEXT: ${{ toJSON(runner) }} run: | - mkdir contexts - echo "$GITHUB_CONTEXT" | nix-shell -p jq --command "jq 'del(.token)' > contexts/github.json" - echo "$RUNNER_CONTEXT" > contexts/runner.json + mkdir -p contexts + nix develop --command bash -c ' + echo "$GITHUB_CONTEXT" | jq "del(.token)" > contexts/github.json + echo "$RUNNER_CONTEXT" > contexts/runner.json + ' - name: Upload context metadata as artifact uses: actions/upload-artifact@v4 with: @@ -121,7 +115,6 @@ jobs: runs-on: [self-hosted, linux, x64] timeout-minutes: ${{ matrix.timeout }} env: - NIX_PATH: nixpkgs=channel:nixos-unstable ORIGINAL_DATADIR: ${{ matrix.datadir_path }} BASE_SHA: ${{ github.event.pull_request.base.sha }} steps: @@ -147,9 +140,8 @@ jobs: TMP_DATADIR: "${{ runner.temp }}/base_datadir" BINARIES_DIR: "${{ runner.temp }}/binaries" run: | - env mkdir -p "$TMP_DATADIR" - nix-shell --command "just run-${{ matrix.network }}-ci-instrumented $BASE_SHA $CHECKOUT_COMMIT $TMP_DATADIR $ORIGINAL_DATADIR ${{ runner.temp }}/results.json ${{ matrix.dbcache }} ${{ runner.temp }}/pngs $BINARIES_DIR" + nix develop --command just run-${{ matrix.network }}-ci-instrumented $BASE_SHA $CHECKOUT_COMMIT $TMP_DATADIR $ORIGINAL_DATADIR ${{ runner.temp }}/results.json ${{ matrix.dbcache }} ${{ runner.temp }}/pngs $BINARIES_DIR - uses: actions/upload-artifact@v4 with: name: result-${{ matrix.name }} @@ -167,9 +159,11 @@ jobs: GITHUB_CONTEXT: ${{ toJSON(github) }} RUNNER_CONTEXT: ${{ toJSON(runner) }} run: | - mkdir contexts - echo "$GITHUB_CONTEXT" | nix-shell -p jq --command "jq 'del(.token)' > contexts/github.json" - echo "$RUNNER_CONTEXT" > contexts/runner.json + mkdir -p contexts + nix develop --command bash -c ' + echo "$GITHUB_CONTEXT" | jq "del(.token)" > contexts/github.json + echo "$RUNNER_CONTEXT" > contexts/runner.json + ' - name: Upload context metadata as artifact uses: actions/upload-artifact@v4 with: diff --git a/bench-ci/build_binaries.sh b/bench-ci/build_binaries.sh index 9a396a00659f..242456250d8d 100755 --- a/bench-ci/build_binaries.sh +++ b/bench-ci/build_binaries.sh @@ -25,23 +25,9 @@ for build in "base:${base_commit}" "head:${head_commit}"; do name="${build%%:*}" commit="${build#*:}" git checkout "$commit" - # Use environment variables if set, otherwise use defaults - HOSTS="${HOSTS:-x86_64-linux-gnu}" \ - SOURCES_PATH="${SOURCES_PATH:-/data/SOURCES_PATH}" \ - BASE_CACHE="${BASE_CACHE:-/data/BASE_CACHE}" \ - taskset -c 2-15 chrt -f 1 bench-ci/guix/guix-build - - # Truncate commit hash to 12 characters - short_commit=$(echo "$commit" | cut -c 1-12) - - # Extract the Guix output - tar -xzf "guix-build-${short_commit}/output/x86_64-linux-gnu/bitcoin-${short_commit}-x86_64-linux-gnu.tar.gz" - - # Copy the binary to our binaries directory - cp "bitcoin-${short_commit}/bin/bitcoind" "binaries/${name}/bitcoind" - - # Cleanup extracted files - rm -rf "bitcoin-${short_commit}" + taskset -c 2-15 chrt -f 1 nix build -L + cp "./result/bin/bitcoind" "./binaries/${name}/bitcoind" + rm -rf "./result" done # Restore initial git state diff --git a/bench-ci/guix/INSTALL.md b/bench-ci/guix/INSTALL.md deleted file mode 100644 index f9a79f66349c..000000000000 --- a/bench-ci/guix/INSTALL.md +++ /dev/null @@ -1,814 +0,0 @@ -# Guix Installation and Setup - -This only needs to be done once per machine. If you have already completed the -installation and setup, please proceed to [perform a build](./README.md). - -Otherwise, you may choose from one of the following options to install Guix: - -1. Using the official **shell installer script** [⤓ skip to section][install-script] - - Maintained by Guix developers - - Easiest (automatically performs *most* setup) - - Works on nearly all Linux distributions - - Only installs latest release - - Binary installation only, requires high level of trust - - Note: The script needs to be run as root, so it should be inspected before it's run -2. Using the official **binary tarball** [⤓ skip to section][install-bin-tarball] - - Maintained by Guix developers - - Normal difficulty (full manual setup required) - - Works on nearly all Linux distributions - - Installs any release - - Binary installation only, requires high level of trust -3. Using fanquake's **container image** [↗︎ external instructions][install-fanquake-container] - - Maintained by fanquake - - Easy (automatically performs *some* setup) - - Works wherever container images work (Docker/Podman) - - Installs any release - - Binary installation only, requires high level of trust -4. Using a **distribution-maintained package** [⤓ skip to section][install-distro-pkg] - - Maintained by distribution's Guix package maintainer - - Normal difficulty (manual setup required) - - Works only on distributions with Guix packaged, see: https://repology.org/project/guix/versions - - Installs a release decided on by package maintainer - - Source or binary installation depending on the distribution -5. Building **from source** [⤓ skip to section][install-source] - - Maintained by you - - Hard, but rewarding - - Can be made to work on most Linux distributions - - Installs any commit (more granular) - - Source installation, requires lower level of trust - -## Options 1 and 2: Using the official shell installer script or binary tarball - -The installation instructions for both the official shell installer script and -the binary tarballs can be found in the GNU Guix Manual's [Binary Installation -section](https://guix.gnu.org/manual/en/html_node/Binary-Installation.html). - -Note that running through the binary tarball installation steps is largely -equivalent to manually performing what the shell installer script does. - -Note that at the time of writing (July 5th, 2021), the shell installer script -automatically creates an `/etc/profile.d` entry which the binary tarball -installation instructions do not ask you to create. However, you will likely -need this entry for better desktop integration. Please see [this -section](#add-an-etcprofiled-entry) for instructions on how to add a -`/etc/profile.d/guix.sh` entry. - -Regardless of which installation option you chose, the changes to -`/etc/profile.d` will not take effect until the next shell or desktop session, -so you should log out and log back in. - -## Option 3: Using fanquake's container image - -Please refer to fanquake's instructions -[here](https://github.com/fanquake/core-review/tree/master/guix). - -## Option 4: Using a distribution-maintained package - -Note that this section is based on the distro packaging situation at the time of -writing (July 2021). Guix is expected to be more widely packaged over time. For -an up-to-date view on Guix's package status/version across distros, please see: -https://repology.org/project/guix/versions - -### Debian / Ubuntu - -Guix is available as a distribution package in [Debian -](https://packages.debian.org/search?keywords=guix) and [Ubuntu -](https://packages.ubuntu.com/search?keywords=guix). - -To install: -```sh -sudo apt install guix -``` - -### Arch Linux - -Guix is available in the AUR as -[`guix`](https://aur.archlinux.org/packages/guix/), please follow the -installation instructions in the Arch Linux Wiki ([live -link](https://wiki.archlinux.org/index.php/Guix#AUR_Package_Installation), -[2021/03/30 -permalink](https://wiki.archlinux.org/index.php?title=Guix&oldid=637559#AUR_Package_Installation)) -to install Guix. - -At the time of writing (2021/03/30), the `check` phase will fail if the path to -guix's build directory is longer than 36 characters due to an anachronistic -character limit on the shebang line. Since the `check` phase happens after the -`build` phase, which may take quite a long time, it is recommended that users -either: - -1. Skip the `check` phase - - For `makepkg`: `makepkg --nocheck ...` - - For `yay`: `yay --mflags="--nocheck" ...` - - For `paru`: `paru --nocheck ...` -2. Or, check their build directory's length beforehand - - For those building with `makepkg`: `pwd | wc -c` - -## Option 5: Building from source - -Building Guix from source is a rather involved process but a rewarding one for -those looking to minimize trust and maximize customizability (e.g. building a -particular commit of Guix). Previous experience with using autotools-style build -systems to build packages from source will be helpful. *hic sunt dracones.* - -I strongly urge you to at least skim through the entire section once before you -start issuing commands, as it will save you a lot of unnecessary pain and -anguish. - -### Installing common build tools - -There are a few basic build tools that are required for most things we'll build, -so let's install them now: - -Text transformation/i18n: -- `autopoint` (sometimes packaged in `gettext`) -- `help2man` -- `po4a` -- `texinfo` - -Build system tools: -- `g++` w/ C++11 support -- `libtool` -- `autoconf` -- `automake` -- `pkg-config` (sometimes packaged as `pkgconf`) -- `make` -- `cmake` - -Miscellaneous: -- `git` -- `gnupg` -- `python3` - -### Building and Installing Guix's dependencies - -In order to build Guix itself from source, we need to first make sure that the -necessary dependencies are installed and discoverable. The most up-to-date list -of Guix's dependencies is kept in the ["Requirements" -section](https://guix.gnu.org/manual/en/html_node/Requirements.html) of the Guix -Reference Manual. - -Depending on your distribution, most or all of these dependencies may already be -packaged and installable without manually building and installing. - -For reference, the graphic below outlines Guix v1.3.0's dependency graph: - -![bootstrap map](https://user-images.githubusercontent.com/6399679/125064185-a9a59880-e0b0-11eb-82c1-9b8e5dc9950d.png) - -If you do not care about building each dependency from source, and Guix is -already packaged for your distribution, you can easily install only the build -dependencies of Guix. For example, to enable deb-src and install the Guix build -dependencies on Ubuntu/Debian: - -```sh -sed -i 's|# deb-src|deb-src|g' /etc/apt/sources.list -apt update -apt-get build-dep -y guix -``` - -If this succeeded, you can likely skip to section -["Building and Installing Guix itself"](#building-and-installing-guix-itself). - -#### Guile - -###### Corner case: Multiple versions of Guile on one system - -It is recommended to only install the required version of Guile, so that build systems do -not get confused about which Guile to use. - -However, if you insist on having more versions of Guile installed on -your system, then you need to **consistently** specify -`GUILE_EFFECTIVE_VERSION=3.0` to all -`./configure` invocations for Guix and its dependencies. - -##### Installing Guile - -If your distribution splits packages into `-dev`-suffixed and -non-`-dev`-suffixed sub-packages (as is the case for Debian-derived -distributions), please make sure to install both. For example, to install Guile -v3.0 on Debian/Ubuntu: - -```sh -apt install guile-3.0 guile-3.0-dev -``` - -#### Mixing distribution packages and source-built packages - -At the time of writing, most distributions have _some_ of Guix's dependencies -packaged, but not all. This means that you may want to install the distribution -package for some dependencies, and manually build-from-source for others. - -Distribution packages usually install to `/usr`, which is different from the -default `./configure` prefix of source-built packages: `/usr/local`. - -This means that if you mix-and-match distribution packages and source-built -packages and do not specify exactly `--prefix=/usr` to `./configure` for -source-built packages, you will need to augment the `GUILE_LOAD_PATH` and -`GUILE_LOAD_COMPILED_PATH` environment variables so that Guile will look -under the right prefix and find your source-built packages. - -For example, if you are using Guile v3.0, and have Guile packages in the -`/usr/local` prefix, either add the following lines to your `.profile` or -`.bash_profile` so that the environment variable is properly set for all future -shell logins, or paste the lines into a POSIX-style shell to temporarily modify -the environment variables of your current shell session. - -```sh -# Help Guile v3.0.x find packages in /usr/local -export GUILE_LOAD_PATH="/usr/local/share/guile/site/3.0${GUILE_LOAD_PATH:+:}$GUILE_LOAD_PATH" -export GUILE_LOAD_COMPILED_PATH="/usr/local/lib/guile/3.0/site-ccache${GUILE_LOAD_COMPILED_PATH:+:}$GUILE_COMPILED_LOAD_PATH" -``` - -Note that these environment variables are used to check for packages during -`./configure`, so they should be set as soon as possible should you want to use -a prefix other than `/usr`. - -#### Building and installing source-built packages - -***IMPORTANT**: A few dependencies have non-obvious quirks/errata which are -documented in the sub-sections immediately below. Please read these sections -before proceeding to build and install these packages.* - -Although you should always refer to the README or INSTALL files for the most -accurate information, most of these dependencies use autoconf-style build -systems (check if there's a `configure.ac` file), and will likely do the right -thing with the following: - -Clone the repository and check out the latest release: -```sh -git clone /.git -cd -git tag -l # check for the latest release -git checkout -``` - -For autoconf-based build systems (if `./autogen.sh` or `configure.ac` exists at -the root of the repository): - -```sh -./autogen.sh || autoreconf -vfi -./configure --prefix= -make -sudo make install -``` - -For CMake-based build systems (if `CMakeLists.txt` exists at the root of the -repository): - -```sh -mkdir build && cd build -cmake .. -DCMAKE_INSTALL_PREFIX= -sudo cmake --build . --target install -``` - -If you choose not to specify exactly `--prefix=/usr` to `./configure`, please -make sure you've carefully read the [previous section] on mixing distribution -packages and source-built packages. - -##### Binding packages require `-dev`-suffixed packages - -Relevant for: -- Everyone - -When building bindings, the `-dev`-suffixed version of the original package -needs to be installed. For example, building `Guile-zlib` on Debian-derived -distributions requires that `zlib1g-dev` is installed. - -When using bindings, the `-dev`-suffixed version of the original package still -needs to be installed. This is particularly problematic when distribution -packages are mispackaged like `guile-sqlite3` is in Ubuntu Focal such that -installing `guile-sqlite3` does not automatically install `libsqlite3-dev` as a -dependency. - -Below is a list of relevant Guile bindings and their corresponding `-dev` -packages in Debian at the time of writing. - -| Guile binding package | -dev Debian package | -|-----------------------|---------------------| -| guile-gcrypt | libgcrypt-dev | -| guile-git | libgit2-dev | -| guile-gnutls | (none) | -| guile-json | (none) | -| guile-lzlib | liblz-dev | -| guile-ssh | libssh-dev | -| guile-sqlite3 | libsqlite3-dev | -| guile-zlib | zlib1g-dev | - -##### `guile-git` actually depends on `libgit2 >= 1.1` - -Relevant for: -- Those building `guile-git` from source against `libgit2 < 1.1` -- Those installing `guile-git` from their distribution where `guile-git` is - built against `libgit2 < 1.1` - -As of v0.5.2, `guile-git` claims to only require `libgit2 >= 0.28.0`, however, -it actually requires `libgit2 >= 1.1`, otherwise, it will be confused by a -reference of `origin/keyring`: instead of interpreting the reference as "the -'keyring' branch of the 'origin' remote", the reference is interpreted as "the -branch literally named 'origin/keyring'" - -This is especially notable because Ubuntu Focal packages `libgit2 v0.28.4`, and -`guile-git` is built against it. - -Should you be in this situation, you need to build both `libgit2 v1.1.x` and -`guile-git` from source. - -Source: https://logs.guix.gnu.org/guix/2020-11-12.log#232527 - -### Building and Installing Guix itself - -Start by cloning Guix: - -``` -git clone https://codeberg.org/guix/guix.git -cd guix -``` - -You will likely want to build the latest release. -At the time of writing (November 2023), the latest release was `v1.4.0`. - -``` -git branch -a -l 'origin/version-*' # check for the latest release -git checkout -``` - -Bootstrap the build system: -``` -./bootstrap -``` - -Configure with the recommended `--localstatedir` flag: -``` -./configure --localstatedir=/var -``` - -Note: If you intend to hack on Guix in the future, you will need to supply the -same `--localstatedir=` flag for all future Guix `./configure` invocations. See -the last paragraph of this -[section](https://guix.gnu.org/manual/en/html_node/Requirements.html) for more -details. - -Build Guix (this will take a while): -``` -make -j$(nproc) -``` - -Install Guix: - -``` -sudo make install -``` - -### Post-"build from source" Setup - -#### Creating and starting a `guix-daemon-original` service with a fixed `argv[0]` - -At this point, guix will be installed to `${bindir}`, which is likely -`/usr/local/bin` if you did not override directory variables at -`./configure`-time. More information on standard Automake directory variables -can be found -[here](https://www.gnu.org/software/automake/manual/html_node/Standard-Directory-Variables.html). - -However, the Guix init scripts and service configurations for Upstart, systemd, -SysV, and OpenRC are installed (in `${libdir}`) to launch -`${localstatedir}/guix/profiles/per-user/root/current-guix/bin/guix-daemon`, -which does not yet exist, and will only exist after [`root` performs their first -`guix pull`](#guix-pull-as-root). - -We need to create a `-original` version of these init scripts that's pointed to -the binaries we just built and `make install`'ed in `${bindir}` (normally, -`/usr/local/bin`). - -Example for `systemd`, run as `root`: - -```sh -# Create guix-daemon-original.service by modifying guix-daemon.service -libdir=# set according to your PREFIX (default is /usr/local/lib) -bindir="$(dirname $(command -v guix-daemon))" -sed -E -e "s|/\S*/guix/profiles/per-user/root/current-guix/bin/guix-daemon|${bindir}/guix-daemon|" "${libdir}"/systemd/system/guix-daemon.service > /etc/systemd/system/guix-daemon-original.service -chmod 664 /etc/systemd/system/guix-daemon-original.service - -# Make systemd recognize the new service -systemctl daemon-reload - -# Make sure that the non-working guix-daemon.service is stopped and disabled -systemctl stop guix-daemon -systemctl disable guix-daemon - -# Make sure that the working guix-daemon-original.service is started and enabled -systemctl enable guix-daemon-original -systemctl start guix-daemon-original -``` - -#### Creating `guix-daemon` users / groups - -Please see the [relevant -section](https://guix.gnu.org/manual/en/html_node/Build-Environment-Setup.html) -in the Guix Reference Manual for more details. - -## Optional setup - -At this point, you are set up to [use Guix to build Bitcoin -Core](./README.md#usage). However, if you want to polish your setup a bit and -make it "what Guix intended", then read the next few subsections. - -### Add an `/etc/profile.d` entry - -This section definitely does not apply to you if you installed Guix using: -1. The shell installer script -2. fanquake's container image -3. Debian's `guix` package - -#### Background - -Although Guix knows how to update itself and its packages, it does so in a -non-invasive way (it does not modify `/usr/local/bin/guix`). - -Instead, it does the following: - -- After a `guix pull`, it updates - `/var/guix/profiles/per-user/$USER/current-guix`, and creates a symlink - targeting this directory at `$HOME/.config/guix/current` - -- After a `guix install`, it updates - `/var/guix/profiles/per-user/$USER/guix-profile`, and creates a symlink - targeting this directory at `$HOME/.guix-profile` - -Therefore, in order for these operations to affect your shell/desktop sessions -(and for the principle of least astonishment to hold), their corresponding -directories have to be added to well-known environment variables like `$PATH`, -`$INFOPATH`, `$XDG_DATA_DIRS`, etc. - -In other words, if `$HOME/.config/guix/current/bin` does not exist in your -`$PATH`, a `guix pull` will have no effect on what `guix` you are using. Same -goes for `$HOME/.guix-profile/bin`, `guix install`, and installed packages. - -Helpfully, after a `guix pull` or `guix install`, a message will be printed like -so: - -``` -hint: Consider setting the necessary environment variables by running: - - GUIX_PROFILE="$HOME/.guix-profile" - . "$GUIX_PROFILE/etc/profile" - -Alternately, see `guix package --search-paths -p "$HOME/.guix-profile"'. -``` - -However, this is somewhat tedious to do for both `guix pull` and `guix install` -for each user on the system that wants to properly use `guix`. I recommend that -you add an entry to `/etc/profile.d` instead. This is done by default -when installing the Debian package later than 1.2.0-4 and when using the shell -script installer. - -#### Instructions - -Create `/etc/profile.d/guix.sh` with the following content: -```sh -# _GUIX_PROFILE: `guix pull` profile -_GUIX_PROFILE="$HOME/.config/guix/current" -if [ -L $_GUIX_PROFILE ]; then - export PATH="$_GUIX_PROFILE/bin${PATH:+:}$PATH" - # Export INFOPATH so that the updated info pages can be found - # and read by both /usr/bin/info and/or $GUIX_PROFILE/bin/info - # When INFOPATH is unset, add a trailing colon so that Emacs - # searches 'Info-default-directory-list'. - export INFOPATH="$_GUIX_PROFILE/share/info:$INFOPATH" -fi - -# GUIX_PROFILE: User's default profile -GUIX_PROFILE="$HOME/.guix-profile" -[ -L $GUIX_PROFILE ] || return -GUIX_LOCPATH="$GUIX_PROFILE/lib/locale" -export GUIX_PROFILE GUIX_LOCPATH - -[ -f "$GUIX_PROFILE/etc/profile" ] && . "$GUIX_PROFILE/etc/profile" - -# set XDG_DATA_DIRS to include Guix installations -export XDG_DATA_DIRS="$GUIX_PROFILE/share:${XDG_DATA_DIRS:-/usr/local/share/:/usr/share/}" -``` - -Please note that this will not take effect until the next shell or desktop -session (log out and log back in). - -### `guix pull` as root - -Before you do this, you need to read the section on [choosing your security -model][security-model] and adjust `guix` and `guix-daemon` flags according to -your choice, as invoking `guix pull` may pull substitutes from substitute -servers (which you may not want). - -As mentioned in a previous section, Guix expects -`${localstatedir}/guix/profiles/per-user/root/current-guix` to be populated with -`root`'s Guix profile, `guix pull`-ed and built by some former version of Guix. -However, this is not the case when we build from source. Therefore, we need to -perform a `guix pull` as `root`: - -```sh -sudo --login guix pull --branch=version- -# or -sudo --login guix pull --commit= -``` - -`guix pull` is quite a long process (especially if you're using -`--no-substitutes`). If you encounter build problems, please refer to the -[troubleshooting section](#troubleshooting). - -Note that running a bare `guix pull` with no commit or branch specified will -pull the latest commit on Guix's master branch, which is likely fine, but not -recommended. - -If you installed Guix from source, you may get an error like the following: -```sh -error: while creating symlink '/root/.config/guix/current' No such file or directory -``` -To resolve this, simply: -``` -sudo mkdir -p /root/.config/guix -``` -Then try the `guix pull` command again. - -After the `guix pull` finishes successfully, -`${localstatedir}/guix/profiles/per-user/root/current-guix` should be populated. - -#### Using the newly-pulled `guix` by restarting the daemon - -Depending on how you installed Guix, you should now make sure that your init -scripts and service configurations point to the newly-pulled `guix-daemon`. - -##### If you built Guix from source - -If you followed the instructions for [fixing argv\[0\]][fix-argv0], you can now -do the following: - -```sh -systemctl stop guix-daemon-original -systemctl disable guix-daemon-original - -systemctl enable guix-daemon -systemctl start guix-daemon -``` - -Remember to set `--no-substitutes` in `$libdir/systemd/system/guix-daemon.service` and other customizations if you used them for `guix-daemon-original.service`. - -##### If you installed Guix via the Debian/Ubuntu distribution packages - -You will need to create a `guix-daemon-latest` service which points to the new -`guix` rather than a pinned one. - -```sh -# Create guix-daemon-latest.service by modifying guix-daemon.service -sed -E -e "s|/usr/bin/guix-daemon|/var/guix/profiles/per-user/root/current-guix/bin/guix-daemon|" /etc/systemd/system/guix-daemon.service > /lib/systemd/system/guix-daemon-latest.service -chmod 664 /lib/systemd/system/guix-daemon-latest.service - -# Make systemd recognize the new service -systemctl daemon-reload - -# Make sure that the old guix-daemon.service is stopped and disabled -systemctl stop guix-daemon -systemctl disable guix-daemon - -# Make sure that the new guix-daemon-latest.service is started and enabled -systemctl enable guix-daemon-latest -systemctl start guix-daemon-latest -``` - -##### If you installed Guix via lantw44's Arch Linux AUR package - -At the time of writing (July 5th, 2021) the systemd unit for "updated Guix" is -`guix-daemon-latest.service`, therefore, you should do the following: - -```sh -systemctl stop guix-daemon -systemctl disable guix-daemon - -systemctl enable guix-daemon-latest -systemctl start guix-daemon-latest -``` - -##### Otherwise... - -Simply do: - -```sh -systemctl restart guix-daemon -``` - -### Checking everything - -If you followed all the steps above to make your Guix setup "prim and proper," -you can check that you did everything properly by running through this -checklist. - -1. `/etc/profile.d/guix.sh` should exist and be sourced at each shell login - -2. `guix describe` should not print `guix describe: error: failed to determine - origin`, but rather something like: - - ``` - Generation 38 Feb 22 2021 16:39:31 (current) - guix f350df4 - repository URL: https://codeberg.org/guix/guix.git - branch: version-1.2.0 - commit: f350df405fbcd5b9e27e6b6aa500da7f101f41e7 - ``` - -3. `guix-daemon` should be running from `${localstatedir}/guix/profiles/per-user/root/current-guix` - -# Troubleshooting - -## Derivation failed to build - -When you see a build failure like below: - -``` -building /gnu/store/...-foo-3.6.12.drv... -/ 'check' phasenote: keeping build directory `/tmp/guix-build-foo-3.6.12.drv-0' -builder for `/gnu/store/...-foo-3.6.12.drv' failed with exit code 1 -build of /gnu/store/...-foo-3.6.12.drv failed -View build log at '/var/log/guix/drvs/../...-foo-3.6.12.drv.bz2'. -cannot build derivation `/gnu/store/...-qux-7.69.1.drv': 1 dependencies couldn't be built -cannot build derivation `/gnu/store/...-bar-3.16.5.drv': 1 dependencies couldn't be built -cannot build derivation `/gnu/store/...-baz-2.0.5.drv': 1 dependencies couldn't be built -guix time-machine: error: build of `/gnu/store/...-baz-2.0.5.drv' failed -``` - -It means that `guix` failed to build a package named `foo`, which was a -dependency of `qux`, `bar`, and `baz`. Importantly, note that the last "failed" -line is not necessarily the root cause, the first "failed" line is. - -Most of the time, the build failure is due to a spurious test failure or the -package's build system/test suite breaking when running multi-threaded. To -rebuild _just_ this derivation in a single-threaded fashion (please don't forget -to add other `guix` flags like `--no-substitutes` as appropriate): - -```sh -$ guix build --cores=1 /gnu/store/...-foo-3.6.12.drv -``` - -If the single-threaded rebuild did not succeed, you may need to dig deeper. -You may view `foo`'s build logs in `less` like so (please replace paths with the -path you see in the build failure output): - -```sh -$ bzcat /var/log/guix/drvs/../...-foo-3.6.12.drv.bz2 | less -``` - -`foo`'s build directory is also preserved and available at -`/tmp/guix-build-foo-3.6.12.drv-0`. However, if you fail to build `foo` multiple -times, it may be `/tmp/...drv-1` or `/tmp/...drv-2`. Always consult the build -failure output for the most accurate, up-to-date information. - -### python(-minimal): [Errno 84] Invalid or incomplete multibyte or wide character - -This error occurs when your `$TMPDIR` (default: /tmp) exists on a filesystem -which rejects characters not present in the UTF-8 character code set. An example -is ZFS with the utf8only=on option set. - -More information: https://github.com/python/cpython/issues/81765 - -### openssl-1.1.1l and openssl-1.1.1n - -OpenSSL includes tests that will fail once some certificate has expired. -The workarounds from the GnuTLS section immediately below can be used. - -For openssl-1.1.1l use 2022-05-01 as the date. - -### GnuTLS: test-suite FAIL: status-request-revoked - -*The derivation is likely identified by: `/gnu/store/vhphki5sg9xkdhh2pbc8gi6vhpfzryf0-gnutls-3.6.12.drv`* - -This unfortunate error is most common for non-substitute builders who installed -Guix v1.2.0. The problem stems from the fact that one of GnuTLS's tests uses a -hardcoded certificate which expired on 2020-10-24. - -What's more unfortunate is that this GnuTLS derivation is somewhat special in -Guix's dependency graph and is not affected by the package transformation flags -like `--without-tests=`. - -The easiest solution for those encountering this problem is to install a newer -version of Guix. However, there are ways to work around this issue: - -#### Workaround 1: Using substitutes for this single derivation - -If you've authorized the official Guix build farm's key (more info -[here](./README.md#step-1-authorize-the-signing-keys)), then you can use -substitutes just for this single derivation by invoking the following: - -```sh -guix build --substitute-urls="https://ci.guix.gnu.org" /gnu/store/vhphki5sg9xkdhh2pbc8gi6vhpfzryf0-gnutls-3.6.12.drv -``` - -See [this section](./README.md#removing-authorized-keys) for instructions on how -to remove authorized keys if you don't want to keep the build farm's key -authorized. - -#### Workaround 2: Temporarily setting the system clock back - -This workaround was described [here](https://issues.guix.gnu.org/44559#5). - -Basically: - -1. Turn off NTP -2. Set system time to 2020-10-01 -3. guix build --no-substitutes /gnu/store/vhphki5sg9xkdhh2pbc8gi6vhpfzryf0-gnutls-3.6.12.drv -4. Set system time back to accurate current time -5. Turn NTP back on - -For example, - -```sh -sudo timedatectl set-ntp no -sudo date --set "01 oct 2020 15:00:00" -guix build /gnu/store/vhphki5sg9xkdhh2pbc8gi6vhpfzryf0-gnutls-3.6.12.drv -sudo timedatectl set-ntp yes -``` - -#### Workaround 3: Disable the tests in the Guix source code for this single derivation - -If all of the above workarounds fail, you can also disable the `tests` phase of -the derivation via the `arguments` option, as described in the official -[`package` -reference](https://guix.gnu.org/manual/en/html_node/package-Reference.html). - -For example, to disable the openssl-1.1 check phase: - -```diff -diff --git a/gnu/packages/tls.scm b/gnu/packages/tls.scm -index f1e844b..1077c4b 100644 ---- a/gnu/packages/tls.scm -+++ b/gnu/packages/tls.scm -@@ -494,4 +494,5 @@ (define-public openssl-1.1 - (arguments - `(#:parallel-tests? #f -+ #:tests? #f - #:test-target "test" -``` - -### coreutils: FAIL: tests/tail-2/inotify-dir-recreate - -The inotify-dir-create test fails on "remote" filesystems such as overlayfs -(Docker's default filesystem) due to the filesystem being mistakenly recognized -as non-remote. - -A relatively easy workaround to this is to make sure that a somewhat traditional -filesystem is mounted at `/tmp` (where `guix-daemon` performs its builds). For -Docker users, this might mean [using a volume][docker/volumes], [binding -mounting][docker/bind-mnt] from host, or (for those with enough RAM and swap) -[mounting a tmpfs][docker/tmpfs] using the `--tmpfs` flag. - -Please see the following links for more details: - -- An upstream coreutils bug has been filed: [debbugs#47940](https://debbugs.gnu.org/cgi/bugreport.cgi?bug=47940) -- A Guix bug detailing the underlying problem has been filed: [guix-issues#47935](https://issues.guix.gnu.org/47935), [guix-issues#49985](https://issues.guix.gnu.org/49985#5) -- A commit to skip this test is included since Guix 1.4.0: -[codeberg/guix@6ba1058](https://codeberg.org/guix/guix/commit/6ba1058df0c4ce5611c2367531ae5c3cdc729ab4) - - -[install-script]: #options-1-and-2-using-the-official-shell-installer-script-or-binary-tarball -[install-bin-tarball]: #options-1-and-2-using-the-official-shell-installer-script-or-binary-tarball -[install-fanquake-container]: #option-3-using-fanquakes-container-image -[install-distro-pkg]: #option-4-using-a-distribution-maintained-package -[install-source]: #option-5-building-from-source - -[fix-argv0]: #creating-and-starting-a-guix-daemon-original-service-with-a-fixed-argv0 -[security-model]: ./README.md#choosing-your-security-model - -[docker/volumes]: https://docs.docker.com/storage/volumes/ -[docker/bind-mnt]: https://docs.docker.com/storage/bind-mounts/ -[docker/tmpfs]: https://docs.docker.com/storage/tmpfs/ - -# Purging/Uninstalling Guix - -In the extraordinarily rare case where you messed up your Guix installation in -an irreversible way, you may want to completely purge Guix from your system and -start over. - -1. Uninstall Guix itself according to the way you installed it (e.g. `sudo apt - purge guix` for Ubuntu packaging, `sudo make uninstall` for a build from source). -2. Remove all build users and groups - - You may check for relevant users and groups using: - - ``` - getent passwd | grep guix - getent group | grep guix - ``` - - Then, you may remove users and groups using: - - ``` - sudo userdel - sudo groupdel - ``` - -3. Remove all possible Guix-related directories - - `/var/guix/` - - `/var/log/guix/` - - `/gnu/` - - `/etc/guix/` - - `/home/*/.config/guix/` - - `/home/*/.cache/guix/` - - `/home/*/.guix-profile/` - - `/root/.config/guix/` - - `/root/.cache/guix/` - - `/root/.guix-profile/` diff --git a/bench-ci/guix/README.md b/bench-ci/guix/README.md deleted file mode 100644 index 7f6b8232bba5..000000000000 --- a/bench-ci/guix/README.md +++ /dev/null @@ -1,430 +0,0 @@ -# Bootstrappable Bitcoin Core Builds - -This directory contains the files necessary to perform bootstrappable Bitcoin -Core builds. - -[Bootstrappability][b17e] furthers our binary security guarantees by allowing us -to _audit and reproduce_ our toolchain instead of blindly _trusting_ binary -downloads. - -We achieve bootstrappability by using Guix as a functional package manager. - -# Requirements - -Conservatively, you will need: - -- 16GB of free disk space on the partition that /gnu/store will reside in -- 8GB of free disk space **per platform triple** you're planning on building - (see the `HOSTS` [environment variable description][env-vars-list]) - -# Installation and Setup - -If you don't have Guix installed and set up, please follow the instructions in -[INSTALL.md](./INSTALL.md) - -# Usage - -If you haven't considered your security model yet, please read [the relevant -section](#choosing-your-security-model) before proceeding to perform a build. - -## Making the Xcode SDK available for macOS cross-compilation - -In order to perform a build for macOS (which is included in the default set of -platform triples to build), you'll need to extract the macOS SDK tarball using -tools found in the [`macdeploy` directory](../macdeploy/README.md#sdk-extraction). - -You can then either point to the SDK using the `SDK_PATH` environment variable: - -```sh -# Extract the SDK tarball to /path/to/parent/dir/of/extracted/SDK/Xcode---extracted-SDK-with-libcxx-headers -tar -C /path/to/parent/dir/of/extracted/SDK -xaf /path/to/Xcode---extracted-SDK-with-libcxx-headers.tar.gz - -# Indicate where to locate the SDK tarball -export SDK_PATH=/path/to/parent/dir/of/extracted/SDK -``` - -or extract it into `depends/SDKs`: - -```sh -mkdir -p depends/SDKs -tar -C depends/SDKs -xaf /path/to/SDK/tarball -``` - -## Building - -*The author highly recommends at least reading over the [common usage patterns -and examples](#common-guix-build-invocation-patterns-and-examples) section below -before starting a build. For a full list of customization options, see the -[recognized environment variables][env-vars-list] section.* - -To build Bitcoin Core reproducibly with all default options, invoke the -following from the top of a clean repository: - -```sh -./contrib/guix/guix-build -``` - -## Codesigning build outputs - -The `guix-codesign` command attaches codesignatures (produced by codesigners) to -existing non-codesigned outputs. Please see the [release process -documentation](/doc/release-process.md#codesigning) for more context. - -It respects many of the same environment variable flags as `guix-build`, with 2 -crucial differences: - -1. Since only Windows and macOS build outputs require codesigning, the `HOSTS` - environment variable will have a sane default value of `x86_64-w64-mingw32 - x86_64-apple-darwin arm64-apple-darwin` instead of all the platforms. -2. The `guix-codesign` command ***requires*** a `DETACHED_SIGS_REPO` flag. - * _**DETACHED_SIGS_REPO**_ - - Set the directory where detached codesignatures can be found for the current - Bitcoin Core version being built. - - _REQUIRED environment variable_ - -An invocation with all default options would look like: - -``` -env DETACHED_SIGS_REPO= ./contrib/guix/guix-codesign -``` - -## Cleaning intermediate work directories - -By default, `guix-build` leaves all intermediate files or "work directories" -(e.g. `depends/work`, `guix-build-*/distsrc-*`) intact at the end of a build so -that they are available to the user (to aid in debugging, etc.). However, these -directories usually take up a large amount of disk space. Therefore, a -`guix-clean` convenience script is provided which cleans the current `git` -worktree to save disk space: - -``` -./contrib/guix/guix-clean -``` - - -## Attesting to build outputs - -Much like how Gitian build outputs are attested to in a `gitian.sigs` -repository, Guix build outputs are attested to in the [`guix.sigs` -repository](https://github.com/bitcoin-core/guix.sigs). - -After you've cloned the `guix.sigs` repository, to attest to the current -worktree's commit/tag: - -``` -env GUIX_SIGS_REPO= SIGNER= ./contrib/guix/guix-attest -``` - -See `./contrib/guix/guix-attest --help` for more information on the various ways -`guix-attest` can be invoked. - -## Verifying build output attestations - -After at least one other signer has uploaded their signatures to the `guix.sigs` -repository: - -``` -git -C pull -env GUIX_SIGS_REPO= ./contrib/guix/guix-verify -``` - - -## Common `guix-build` invocation patterns and examples - -### Keeping caches and SDKs outside of the worktree - -If you perform a lot of builds and have a bunch of worktrees, you may find it -more efficient to keep the depends tree's download cache, build cache, and SDKs -outside of the worktrees to avoid duplicate downloads and unnecessary builds. To -help with this situation, the `guix-build` script honours the `SOURCES_PATH`, -`BASE_CACHE`, and `SDK_PATH` environment variables and will pass them on to the -depends tree so that you can do something like: - -```sh -env SOURCES_PATH="$HOME/depends-SOURCES_PATH" BASE_CACHE="$HOME/depends-BASE_CACHE" SDK_PATH="$HOME/macOS-SDKs" ./contrib/guix/guix-build -``` - -Note that the paths that these environment variables point to **must be -directories**, and **NOT symlinks to directories**. - -See the [recognized environment variables][env-vars-list] section for more -details. - -### Building a subset of platform triples - -Sometimes you only want to build a subset of the supported platform triples, in -which case you can override the default list by setting the space-separated -`HOSTS` environment variable: - -```sh -env HOSTS='x86_64-w64-mingw32 x86_64-apple-darwin' ./contrib/guix/guix-build -``` - -See the [recognized environment variables][env-vars-list] section for more -details. - -### Controlling the number of threads used by `guix` build commands - -Depending on your system's RAM capacity, you may want to decrease the number of -threads used to decrease RAM usage or vice versa. - -By default, the scripts under `./contrib/guix` will invoke all `guix` build -commands with `--cores="$JOBS"`. Note that `$JOBS` defaults to `$(nproc)` if not -specified. However, astute manual readers will also notice that `guix` build -commands also accept a `--max-jobs=` flag (which defaults to 1 if unspecified). - -Here is the difference between `--cores=` and `--max-jobs=`: - -> Note: When I say "derivation," think "package" - -`--cores=` - - - controls the number of CPU cores to build each derivation. This is the value - passed to `make`'s `--jobs=` flag. - -`--max-jobs=` - - - controls how many derivations can be built in parallel - - defaults to 1 - -Therefore, the default is for `guix` build commands to build one derivation at a -time, utilizing `$JOBS` threads. - -Specifying the `$JOBS` environment variable will only modify `--cores=`, but you -can also modify the value for `--max-jobs=` by specifying -`$ADDITIONAL_GUIX_COMMON_FLAGS`. For example, if you have a LOT of memory, you -may want to set: - -```sh -export ADDITIONAL_GUIX_COMMON_FLAGS='--max-jobs=8' -``` - -Which allows for a maximum of 8 derivations to be built at the same time, each -utilizing `$JOBS` threads. - -Or, if you'd like to avoid spurious build failures caused by issues with -parallelism within a single package, but would still like to build multiple -packages when the dependency graph allows for it, you may want to try: - -```sh -export JOBS=1 ADDITIONAL_GUIX_COMMON_FLAGS='--max-jobs=8' -``` - -See the [recognized environment variables][env-vars-list] section for more -details. - -## Recognized environment variables - -* _**HOSTS**_ - - Override the space-separated list of platform triples for which to perform a - bootstrappable build. - - _(defaults to "x86\_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu - riscv64-linux-gnu powerpc64-linux-gnu powerpc64le-linux-gnu - x86\_64-w64-mingw32 x86\_64-apple-darwin arm64-apple-darwin")_ - -* _**SOURCES_PATH**_ - - Set the depends tree download cache for sources. This is passed through to the - depends tree. Setting this to the same directory across multiple builds of the - depends tree can eliminate unnecessary redownloading of package sources. - - The path that this environment variable points to **must be a directory**, and - **NOT a symlink to a directory**. - -* _**BASE_CACHE**_ - - Set the depends tree cache for built packages. This is passed through to the - depends tree. Setting this to the same directory across multiple builds of the - depends tree can eliminate unnecessary building of packages. - - The path that this environment variable points to **must be a directory**, and - **NOT a symlink to a directory**. - -* _**SDK_PATH**_ - - Set the path where _extracted_ SDKs can be found. This is passed through to - the depends tree. Note that this should be set to the _parent_ directory of - the actual SDK (e.g. `SDK_PATH=$HOME/Downloads/macOS-SDKs` instead of - `$HOME/Downloads/macOS-SDKs/Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers`). - - The path that this environment variable points to **must be a directory**, and - **NOT a symlink to a directory**. - -* _**JOBS**_ - - Override the number of jobs to run simultaneously, you might want to do so on - a memory-limited machine. This may be passed to: - - - `guix` build commands as in `guix shell --cores="$JOBS"` - - `make` as in `make --jobs="$JOBS"` - - `cmake` as in `cmake --build build -j "$JOBS"` - - `xargs` as in `xargs -P"$JOBS"` - - See [here](#controlling-the-number-of-threads-used-by-guix-build-commands) for - more details. - - _(defaults to the value of `nproc` outside the container)_ - -* _**SOURCE_DATE_EPOCH**_ - - Override the reference UNIX timestamp used for bit-for-bit reproducibility, - the variable name conforms to [standard][r12e/source-date-epoch]. - - _(defaults to the output of `$(git log --format=%at -1)`)_ - -* _**V**_ - - If non-empty, will pass `V=1` to all `make` invocations, making `make` output - verbose. - - Note that any given value is ignored. The variable is only checked for - emptiness. More concretely, this means that `V=` (setting `V` to the empty - string) is interpreted the same way as not setting `V` at all, and that `V=0` - has the same effect as `V=1`. - -* _**SUBSTITUTE_URLS**_ - - A whitespace-delimited list of URLs from which to download pre-built packages. - A URL is only used if its signing key is authorized (refer to the [substitute - servers section](#option-1-building-with-substitutes) for more details). - -* _**ADDITIONAL_GUIX_COMMON_FLAGS**_ - - Additional flags to be passed to all `guix` commands. - -* _**ADDITIONAL_GUIX_TIMEMACHINE_FLAGS**_ - - Additional flags to be passed to `guix time-machine`. - -* _**ADDITIONAL_GUIX_ENVIRONMENT_FLAGS**_ - - Additional flags to be passed to the invocation of `guix shell` inside - `guix time-machine`. - -# Choosing your security model - -No matter how you installed Guix, you need to decide on your security model for -building packages with Guix. - -Guix allows us to achieve better binary security by using our CPU time to build -everything from scratch. However, it doesn't sacrifice user choice in pursuit of -this: users can decide whether or not to use **substitutes** (pre-built -packages). - -## Option 1: Building with substitutes - -### Step 1: Authorize the signing keys - -Depending on the installation procedure you followed, you may have already -authorized the Guix build farm key. In particular, the official shell installer -script asks you if you want the key installed, and the debian distribution -package authorized the key during installation. - -You can check the current list of authorized keys at `/etc/guix/acl`. - -At the time of writing, a `/etc/guix/acl` with just the Guix build farm key -authorized looks something like: - -```lisp -(acl - (entry - (public-key - (ecc - (curve Ed25519) - (q #8D156F295D24B0D9A86FA5741A840FF2D24F60F7B6C4134814AD55625971B394#) - ) - ) - (tag - (guix import) - ) - ) - ) -``` - -If you've determined that the official Guix build farm key hasn't been -authorized, and you would like to authorize it, run the following as root: - -``` -guix archive --authorize < /var/guix/profiles/per-user/root/current-guix/share/guix/ci.guix.gnu.org.pub -``` - -If -`/var/guix/profiles/per-user/root/current-guix/share/guix/ci.guix.gnu.org.pub` -doesn't exist, try: - -```sh -guix archive --authorize < /share/guix/ci.guix.gnu.org.pub -``` - -Where `` is likely: -- `/usr` if you installed from a distribution package -- `/usr/local` if you installed Guix from source and didn't supply any - prefix-modifying flags to Guix's `./configure` - -#### Removing authorized keys - -To remove previously authorized keys, simply edit `/etc/guix/acl` and remove the -`(entry (public-key ...))` entry. - -### Step 2: Specify the substitute servers - -Once its key is authorized, the official Guix build farm at -https://ci.guix.gnu.org is automatically used unless the `--no-substitutes` flag -is supplied. This default list of substitute servers is overridable both on a -`guix-daemon` level and when you invoke `guix` commands. See examples below for -the various ways of adding a substitute server after having [authorized -its signing key](#step-1-authorize-the-signing-keys). - -Change the **default list** of substitute servers by starting `guix-daemon` with -the `--substitute-urls` option (you will likely need to edit your init script): - -```sh -guix-daemon --substitute-urls='https://bordeaux.guix.gnu.org https://ci.guix.gnu.org' -``` - -Override the default list of substitute servers by passing the -`--substitute-urls` option for invocations of `guix` commands: - -```sh -guix --substitute-urls='https://bordeaux.guix.gnu.org https://ci.guix.gnu.org' -``` - -For scripts under `./contrib/guix`, set the `SUBSTITUTE_URLS` environment -variable: - -```sh -export SUBSTITUTE_URLS='https://bordeaux.guix.gnu.org https://ci.guix.gnu.org' -``` - -## Option 2: Disabling substitutes on an ad-hoc basis - -If you prefer not to use any substitutes, make sure to supply `--no-substitutes` -like in the following snippet. The first build will take a while, but the -resulting packages will be cached for future builds. - -For direct invocations of `guix`: -```sh -guix --no-substitutes -``` - -For the scripts under `./contrib/guix/`: -```sh -export ADDITIONAL_GUIX_COMMON_FLAGS='--no-substitutes' -``` - -## Option 3: Disabling substitutes by default - -`guix-daemon` accepts a `--no-substitutes` flag, which will make sure that, -unless otherwise overridden by a command line invocation, no substitutes will be -used. - -If you start `guix-daemon` using an init script, you can edit said script to -supply this flag. - -[b17e]: https://bootstrappable.org/ -[r12e/source-date-epoch]: https://reproducible-builds.org/docs/source-date-epoch/ -[env-vars-list]: #recognized-environment-variables diff --git a/bench-ci/guix/guix-attest b/bench-ci/guix/guix-attest deleted file mode 100755 index b0ef28dc3f92..000000000000 --- a/bench-ci/guix/guix-attest +++ /dev/null @@ -1,263 +0,0 @@ -#!/usr/bin/env bash -export LC_ALL=C -set -e -o pipefail - -# Source the common prelude, which: -# 1. Checks if we're at the top directory of the Bitcoin Core repository -# 2. Defines a few common functions and variables -# -# shellcheck source=libexec/prelude.bash -source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" - - -################### -## Sanity Checks ## -################### - -################ -# Required non-builtin commands should be invokable -################ - -check_tools cat env basename mkdir diff sort - -if [ -z "$NO_SIGN" ]; then - # make it possible to override the gpg binary - GPG=${GPG:-gpg} - - # $GPG can contain extra arguments passed to the binary - # so let's check only the existence of arg[0] - # shellcheck disable=SC2206 - GPG_ARRAY=($GPG) - check_tools "${GPG_ARRAY[0]}" -fi - -################ -# Required env vars should be non-empty -################ - -cmd_usage() { -cat < \\ - SIGNER=GPG_KEY_NAME[=SIGNER_NAME] \\ - [ NO_SIGN=1 ] - ./contrib/guix/guix-attest - -Example w/o overriding signing name: - - env GUIX_SIGS_REPO=/home/achow101/guix.sigs \\ - SIGNER=achow101 \\ - ./contrib/guix/guix-attest - -Example overriding signing name: - - env GUIX_SIGS_REPO=/home/dongcarl/guix.sigs \\ - SIGNER=0x96AB007F1A7ED999=dongcarl \\ - ./contrib/guix/guix-attest - -Example w/o signing, just creating SHA256SUMS: - - env GUIX_SIGS_REPO=/home/achow101/guix.sigs \\ - SIGNER=achow101 \\ - NO_SIGN=1 \\ - ./contrib/guix/guix-attest - -EOF -} - -if [ -z "$GUIX_SIGS_REPO" ] || [ -z "$SIGNER" ]; then - cmd_usage - exit 1 -fi - -################ -# GUIX_SIGS_REPO should exist as a directory -################ - -if [ ! -d "$GUIX_SIGS_REPO" ]; then -cat << EOF -ERR: The specified GUIX_SIGS_REPO is not an existent directory: - - '$GUIX_SIGS_REPO' - -Hint: Please clone the guix.sigs repository and point to it with the - GUIX_SIGS_REPO environment variable. - -EOF -cmd_usage -exit 1 -fi - -################ -# The key specified in SIGNER should be usable -################ - -IFS='=' read -r gpg_key_name signer_name <<< "$SIGNER" -if [ -z "${signer_name}" ]; then - signer_name="$gpg_key_name" -fi - -if [ -z "$NO_SIGN" ] && ! ${GPG} --dry-run --list-secret-keys "${gpg_key_name}" >/dev/null 2>&1; then - echo "ERR: GPG can't seem to find any key named '${gpg_key_name}'" - exit 1 -fi - -################ -# We should be able to find at least one output -################ - -echo "Looking for build output SHA256SUMS fragments in ${OUTDIR_BASE}" - -shopt -s nullglob -sha256sum_fragments=( "$OUTDIR_BASE"/*/SHA256SUMS.part ) # This expands to an array of directories... -shopt -u nullglob - -noncodesigned_fragments=() -codesigned_fragments=() - -if (( ${#sha256sum_fragments[@]} )); then - echo "Found build output SHA256SUMS fragments:" - for outdir in "${sha256sum_fragments[@]}"; do - echo " '$outdir'" - case "$outdir" in - "$OUTDIR_BASE"/*-codesigned/SHA256SUMS.part) - codesigned_fragments+=("$outdir") - ;; - *) - noncodesigned_fragments+=("$outdir") - ;; - esac - done - echo -else - echo "ERR: Could not find any build output SHA256SUMS fragments in ${OUTDIR_BASE}" - exit 1 -fi - -############## -## Attest ## -############## - -# Usage: out_name $outdir -# -# HOST: The output directory being attested -# -out_name() { - basename "$(dirname "$1")" -} - -shasum_already_exists() { -cat < "$temp_noncodesigned" - if [ -e noncodesigned.SHA256SUMS ]; then - # The SHA256SUMS already exists, make sure it's exactly what we - # expect, error out if not - if diff -u noncodesigned.SHA256SUMS "$temp_noncodesigned"; then - echo "A noncodesigned.SHA256SUMS file already exists for '${VERSION}' and is up-to-date." - else - shasum_already_exists noncodesigned.SHA256SUMS - exit 1 - fi - else - mv "$temp_noncodesigned" noncodesigned.SHA256SUMS - fi - else - echo "ERR: No noncodesigned outputs found for '${VERSION}', exiting..." - exit 1 - fi - - temp_all="$(mktemp)" - trap 'rm -rf -- "$temp_all"' EXIT - - if (( ${#codesigned_fragments[@]} )); then - # Note: all.SHA256SUMS attests to all of $sha256sum_fragments, but is - # not needed if there are no $codesigned_fragments - cat "${sha256sum_fragments[@]}" \ - | sort -u \ - | sort -k2 \ - | basenameify_SHA256SUMS \ - > "$temp_all" - if [ -e all.SHA256SUMS ]; then - # The SHA256SUMS already exists, make sure it's exactly what we - # expect, error out if not - if diff -u all.SHA256SUMS "$temp_all"; then - echo "An all.SHA256SUMS file already exists for '${VERSION}' and is up-to-date." - else - shasum_already_exists all.SHA256SUMS - exit 1 - fi - else - mv "$temp_all" all.SHA256SUMS - fi - else - # It is fine to have the codesigned outputs be missing (perhaps the - # detached codesigs have not been published yet), just print a log - # message instead of erroring out - echo "INFO: No codesigned outputs found for '${VERSION}', skipping..." - fi - - if [ -z "$NO_SIGN" ]; then - echo "Signing SHA256SUMS to produce SHA256SUMS.asc" - for i in *.SHA256SUMS; do - if [ ! -e "$i".asc ]; then - ${GPG} --detach-sign \ - --digest-algo sha256 \ - --local-user "$gpg_key_name" \ - --armor \ - --output "$i".asc "$i" - else - echo "Signature already there" - fi - done - else - echo "Not signing SHA256SUMS as \$NO_SIGN is not empty" - fi - echo "" -) diff --git a/bench-ci/guix/guix-build b/bench-ci/guix/guix-build deleted file mode 100755 index 84d4f201b259..000000000000 --- a/bench-ci/guix/guix-build +++ /dev/null @@ -1,474 +0,0 @@ -#!/usr/bin/env bash -export LC_ALL=C -set -e -o pipefail - -# Source the common prelude, which: -# 1. Checks if we're at the top directory of the Bitcoin Core repository -# 2. Defines a few common functions and variables -# -# shellcheck source=libexec/prelude.bash -source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" - - -################### -## SANITY CHECKS ## -################### - -################ -# Required non-builtin commands should be invocable -################ - -check_tools cat mkdir make getent curl git guix - -################ -# GUIX_BUILD_OPTIONS should be empty -################ -# -# GUIX_BUILD_OPTIONS is an environment variable recognized by guix commands that -# can perform builds. This seems like what we want instead of -# ADDITIONAL_GUIX_COMMON_FLAGS, but the value of GUIX_BUILD_OPTIONS is actually -# _appended_ to normal command-line options. Meaning that they will take -# precedence over the command-specific ADDITIONAL_GUIX__FLAGS. -# -# This seems like a poor user experience. Thus we check for GUIX_BUILD_OPTIONS's -# existence here and direct users of this script to use our (more flexible) -# custom environment variables. -if [ -n "$GUIX_BUILD_OPTIONS" ]; then -cat << EOF -Error: Environment variable GUIX_BUILD_OPTIONS is not empty: - '$GUIX_BUILD_OPTIONS' - -Unfortunately this script is incompatible with GUIX_BUILD_OPTIONS, please unset -GUIX_BUILD_OPTIONS and use ADDITIONAL_GUIX_COMMON_FLAGS to set build options -across guix commands or ADDITIONAL_GUIX__FLAGS to set build options for a -specific guix command. - -See contrib/guix/README.md for more details. -EOF -exit 1 -fi - -################ -# The git worktree should not be dirty -################ - -if ! git diff-index --quiet HEAD -- && [ -z "$FORCE_DIRTY_WORKTREE" ]; then -cat << EOF -ERR: The current git worktree is dirty, which may lead to broken builds. - - Aborting... - -Hint: To make your git worktree clean, You may want to: - 1. Commit your changes, - 2. Stash your changes, or - 3. Set the 'FORCE_DIRTY_WORKTREE' environment variable if you insist on - using a dirty worktree -EOF -exit 1 -fi - -mkdir -p "$VERSION_BASE" - -################ -# SOURCE_DATE_EPOCH should not unintentionally be set -################ - -check_source_date_epoch - -################ -# Build directories should not exist -################ - -# Default to building for all supported HOSTs (overridable by environment) -# powerpc64le-linux-gnu currently disabled due non-determinism issues across build arches. -export HOSTS="${HOSTS:-x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu powerpc64-linux-gnu - x86_64-w64-mingw32 - x86_64-apple-darwin arm64-apple-darwin}" - -# Usage: distsrc_for_host HOST -# -# HOST: The current platform triple we're building for -# -distsrc_for_host() { - echo "${DISTSRC_BASE}/distsrc-${VERSION}-${1}" -} - -# Accumulate a list of build directories that already exist... -hosts_distsrc_exists="" -for host in $HOSTS; do - if [ -e "$(distsrc_for_host "$host")" ]; then - hosts_distsrc_exists+=" ${host}" - fi -done - -if [ -n "$hosts_distsrc_exists" ]; then -# ...so that we can print them out nicely in an error message -cat << EOF -ERR: Build directories for this commit already exist for the following platform - triples you're attempting to build, probably because of previous builds. - Please remove, or otherwise deal with them prior to starting another build. - - Aborting... - -Hint: To blow everything away, you may want to use: - - $ ./contrib/guix/guix-clean - -Specifically, this will remove all files without an entry in the index, -excluding the SDK directory, the depends download cache, the depends built -packages cache, the garbage collector roots for Guix environments, and the -output directory. -EOF -for host in $hosts_distsrc_exists; do - echo " ${host} '$(distsrc_for_host "$host")'" -done -exit 1 -else - mkdir -p "$DISTSRC_BASE" -fi - -################ -# When building for darwin, the macOS SDK should exist -################ - -for host in $HOSTS; do - case "$host" in - *darwin*) - OSX_SDK="$(make -C "${PWD}/depends" --no-print-directory HOST="$host" print-OSX_SDK | sed 's@^[^=]\+=@@g')" - if [ -e "$OSX_SDK" ]; then - echo "Found macOS SDK at '${OSX_SDK}', using..." - break - else - echo "macOS SDK does not exist at '${OSX_SDK}', please place the extracted, untarred SDK there to perform darwin builds, or define SDK_PATH environment variable. Exiting..." - exit 1 - fi - ;; - esac -done - -################ -# VERSION_BASE should have enough space -################ - -avail_KiB="$(df -Pk "$VERSION_BASE" | sed 1d | tr -s ' ' | cut -d' ' -f4)" -total_required_KiB=0 -for host in $HOSTS; do - case "$host" in - *darwin*) required_KiB=440000 ;; - *mingw*) required_KiB=7600000 ;; - *) required_KiB=6400000 ;; - esac - total_required_KiB=$((total_required_KiB+required_KiB)) -done - -if (( total_required_KiB > avail_KiB )); then - total_required_GiB=$((total_required_KiB / 1048576)) - avail_GiB=$((avail_KiB / 1048576)) - echo "Performing a Bitcoin Core Guix build for the selected HOSTS requires ${total_required_GiB} GiB, however, only ${avail_GiB} GiB is available. Please free up some disk space before performing the build." - exit 1 -fi - -################ -# Check that we can connect to the guix-daemon -################ - -cat << EOF -Checking that we can connect to the guix-daemon... - -Hint: If this hangs, you may want to try turning your guix-daemon off and on - again. - -EOF -if ! guix gc --list-failures > /dev/null; then -cat << EOF - -ERR: Failed to connect to the guix-daemon, please ensure that one is running and - reachable. -EOF -exit 1 -fi - -# Developer note: we could use `guix repl` for this check and run: -# -# (import (guix store)) (close-connection (open-connection)) -# -# However, the internal API is likely to change more than the CLI invocation - -################ -# Services database must have basic entries -################ - -if ! getent services http https ftp > /dev/null 2>&1; then -cat << EOF -ERR: Your system's C library cannot find service database entries for at least - one of the following services: http, https, ftp. - -Hint: Most likely, /etc/services does not exist yet (common for docker images - and minimal distros), or you don't have permissions to access it. - - If /etc/services does not exist yet, you may want to install the - appropriate package for your distro which provides it. - - On Debian/Ubuntu: netbase - On Arch Linux: iana-etc - - For more information, see: getent(1), services(5) - -EOF - -fi - -######### -# SETUP # -######### - -# Determine the maximum number of jobs to run simultaneously (overridable by -# environment) -JOBS="${JOBS:-$(nproc)}" - -# Usage: host_to_commonname HOST -# -# HOST: The current platform triple we're building for -# -host_to_commonname() { - case "$1" in - *darwin*) echo osx ;; - *mingw*) echo win ;; - *linux*) echo linux ;; - *) exit 1 ;; - esac -} - -# Determine the reference time used for determinism (overridable by environment) -SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git -c log.showSignature=false log --format=%at -1)}" - -# Precious directories are those which should not be cleaned between successive -# guix builds -depends_precious_dir_names='SOURCES_PATH BASE_CACHE SDK_PATH' -precious_dir_names="${depends_precious_dir_names} OUTDIR_BASE PROFILES_BASE" - -# Usage: contains IFS-SEPARATED-LIST ITEM -contains() { - for i in ${1}; do - if [ "$i" = "${2}" ]; then - return 0 # Found! - fi - done - return 1 -} - -# If the user explicitly specified a precious directory, create it so we -# can map it into the container -for precious_dir_name in $precious_dir_names; do - precious_dir_path="${!precious_dir_name}" - if [ -n "$precious_dir_path" ]; then - if [ ! -e "$precious_dir_path" ]; then - mkdir -p "$precious_dir_path" - elif [ -L "$precious_dir_path" ]; then - echo "ERR: ${precious_dir_name} cannot be a symbolic link" - exit 1 - elif [ ! -d "$precious_dir_path" ]; then - echo "ERR: ${precious_dir_name} must be a directory" - exit 1 - fi - fi -done - -mkdir -p "$VAR_BASE" - -# Record the _effective_ values of precious directories such that guix-clean can -# avoid clobbering them if appropriate. -# -# shellcheck disable=SC2046,SC2086 -{ - # Get depends precious dir definitions from depends - make -C "${PWD}/depends" \ - --no-print-directory \ - -- $(printf "print-%s\n" $depends_precious_dir_names) - - # Get remaining precious dir definitions from the environment - for precious_dir_name in $precious_dir_names; do - precious_dir_path="${!precious_dir_name}" - if ! contains "$depends_precious_dir_names" "$precious_dir_name"; then - echo "${precious_dir_name}=${precious_dir_path}" - fi - done -} > "${VAR_BASE}/precious_dirs" - -# Make sure an output directory exists for our builds -OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}" -mkdir -p "$OUTDIR_BASE" - -# Download the depends sources now as we won't have internet access in the build -# container -for host in $HOSTS; do - make -C "${PWD}/depends" -j"$JOBS" download-"$(host_to_commonname "$host")" ${V:+V=1} ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} -done - -# Usage: outdir_for_host HOST SUFFIX -# -# HOST: The current platform triple we're building for -# -outdir_for_host() { - echo "${OUTDIR_BASE}/${1}${2:+-${2}}" -} - -# Usage: profiledir_for_host HOST SUFFIX -# -# HOST: The current platform triple we're building for -# -profiledir_for_host() { - echo "${PROFILES_BASE}/${1}${2:+-${2}}" -} - - -######### -# BUILD # -######### - -# Function to be called when building for host ${1} and the user interrupts the -# build -int_trap() { -cat << EOF -** INT received while building ${1}, you may want to clean up the relevant - work directories (e.g. distsrc-*) before rebuilding - -Hint: To blow everything away, you may want to use: - - $ ./contrib/guix/guix-clean - -Specifically, this will remove all files without an entry in the index, -excluding the SDK directory, the depends download cache, the depends built -packages cache, the garbage collector roots for Guix environments, and the -output directory. -EOF -} - -# Deterministically build Bitcoin Core -# shellcheck disable=SC2153 -for host in $HOSTS; do - - # Display proper warning when the user interrupts the build - trap 'int_trap ${host}' INT - - ( - # Required for 'contrib/guix/manifest.scm' to output the right manifest - # for the particular $HOST we're building for - export HOST="$host" - - # shellcheck disable=SC2030 -cat << EOF -INFO: Building ${VERSION:?not set} for platform triple ${HOST:?not set}: - ...using reference timestamp: ${SOURCE_DATE_EPOCH:?not set} - ...running at most ${JOBS:?not set} jobs - ...from worktree directory: '${PWD}' - ...bind-mounted in container to: '/bitcoin' - ...in build directory: '$(distsrc_for_host "$HOST")' - ...bind-mounted in container to: '$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")' - ...outputting in: '$(outdir_for_host "$HOST")' - ...bind-mounted in container to: '$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST")' - ADDITIONAL FLAGS (if set) - ADDITIONAL_GUIX_COMMON_FLAGS: ${ADDITIONAL_GUIX_COMMON_FLAGS} - ADDITIONAL_GUIX_ENVIRONMENT_FLAGS: ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} - ADDITIONAL_GUIX_TIMEMACHINE_FLAGS: ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS} -EOF - - # Run the build script 'contrib/guix/libexec/build.sh' in the build - # container specified by 'contrib/guix/manifest.scm'. - # - # Explanation of `guix shell` flags: - # - # --container run command within an isolated container - # - # Running in an isolated container minimizes build-time differences - # between machines and improves reproducibility - # - # --pure unset existing environment variables - # - # Same rationale as --container - # - # --no-cwd do not share current working directory with an - # isolated container - # - # When --container is specified, the default behavior is to share - # the current working directory with the isolated container at the - # same exact path (e.g. mapping '/home/satoshi/bitcoin/' to - # '/home/satoshi/bitcoin/'). This means that the $PWD inside the - # container becomes a source of irreproducibility. --no-cwd disables - # this behaviour. - # - # --share=SPEC for containers, share writable host file system - # according to SPEC - # - # --share="$PWD"=/bitcoin - # - # maps our current working directory to /bitcoin - # inside the isolated container, which we later cd - # into. - # - # While we don't want to map our current working directory to the - # same exact path (as this introduces irreproducibility), we do want - # it to be at a _fixed_ path _somewhere_ inside the isolated - # container so that we have something to build. '/bitcoin' was - # chosen arbitrarily. - # - # ${SOURCES_PATH:+--share="$SOURCES_PATH"} - # - # make the downloaded depends sources path available - # inside the isolated container - # - # The isolated container has no network access as it's in a - # different network namespace from the main machine, so we have to - # make the downloaded depends sources available to it. The sources - # should have been downloaded prior to this invocation. - # - # --keep-failed keep build tree of failed builds - # - # When builds of the Guix environment itself (not Bitcoin Core) - # fail, it is useful for the build tree to be kept for debugging - # purposes. - # - # ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} - # - # fetch substitute from SUBSTITUTE_URLS if they are - # authorized - # - # Depending on the user's security model, it may be desirable to use - # substitutes (pre-built packages) from servers that the user trusts. - # Please read the README.md in the same directory as this file for - # more information. - # - # shellcheck disable=SC2086,SC2031 - time-machine shell --manifest="${PWD}/bench-ci/guix/manifest.scm" \ - --container \ - --pure \ - --no-cwd \ - --share="$PWD"=/bitcoin \ - --share="$DISTSRC_BASE"=/distsrc-base \ - --share="$OUTDIR_BASE"=/outdir-base \ - --expose="$(git rev-parse --git-common-dir)" \ - ${SOURCES_PATH:+--share="$SOURCES_PATH"} \ - ${BASE_CACHE:+--share="$BASE_CACHE"} \ - ${SDK_PATH:+--share="$SDK_PATH"} \ - --cores="$JOBS" \ - --keep-failed \ - --fallback \ - --link-profile \ - --root="$(profiledir_for_host "${HOST}")" \ - ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ - ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ - -- env HOST="$host" \ - DISTNAME="$DISTNAME" \ - JOBS="$JOBS" \ - SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \ - ${V:+V=1} \ - ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \ - ${BASE_CACHE:+BASE_CACHE="$BASE_CACHE"} \ - ${SDK_PATH:+SDK_PATH="$SDK_PATH"} \ - DISTSRC="$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")" \ - OUTDIR="$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST")" \ - DIST_ARCHIVE_BASE=/outdir-base/dist-archive \ - bash -c "cd /bitcoin && bash bench-ci/guix/libexec/build.sh" - ) - -done diff --git a/bench-ci/guix/guix-clean b/bench-ci/guix/guix-clean deleted file mode 100755 index 9af0a793cff7..000000000000 --- a/bench-ci/guix/guix-clean +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env bash -export LC_ALL=C -set -e -o pipefail - -# Source the common prelude, which: -# 1. Checks if we're at the top directory of the Bitcoin Core repository -# 2. Defines a few common functions and variables -# -# shellcheck source=libexec/prelude.bash -source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" - - -################### -## Sanity Checks ## -################### - -################ -# Required non-builtin commands should be invokable -################ - -check_tools cat mkdir make git guix - - -############# -## Clean ## -############# - -# Usage: under_dir MAYBE_PARENT MAYBE_CHILD -# -# If MAYBE_CHILD is a subdirectory of MAYBE_PARENT, print the relative path -# from MAYBE_PARENT to MAYBE_CHILD. Otherwise, return 1 as the error code. -# -# NOTE: This does not perform any symlink-resolving or path canonicalization. -# -under_dir() { - local path_residue - path_residue="${2##"${1}"}" - if [ -z "$path_residue" ] || [ "$path_residue" = "$2" ]; then - return 1 - else - echo "$path_residue" - fi -} - -# Usage: dir_under_git_root MAYBE_CHILD -# -# If MAYBE_CHILD is under the current git repository and exists, print the -# relative path from the git repository's top-level directory to MAYBE_CHILD, -# otherwise, exit with an error code. -# -dir_under_git_root() { - local rv - rv="$(under_dir "$(git_root)" "$1")" - [ -n "$rv" ] && echo "$rv" -} - -shopt -s nullglob -found_precious_dirs_files=( "${version_base_prefix}"*/"${var_base_basename}/precious_dirs" ) # This expands to an array of directories... -shopt -u nullglob - -exclude_flags=() - -for precious_dirs_file in "${found_precious_dirs_files[@]}"; do - # Make sure the precious directories (e.g. SOURCES_PATH, BASE_CACHE, SDK_PATH) - # are excluded from git-clean - echo "Found precious_dirs file: '${precious_dirs_file}'" - - # Exclude the precious_dirs file itself - if dirs_file_exclude_fragment=$(dir_under_git_root "$(dirname "$precious_dirs_file")"); then - exclude_flags+=( --exclude="${dirs_file_exclude_fragment}/precious_dirs" ) - fi - - # Read each 'name=dir' pair from the precious_dirs file - while IFS='=' read -r name dir; do - # Add an exclusion flag if the precious directory is under the git root. - if under=$(dir_under_git_root "$dir"); then - echo "Avoiding ${name}: ${under}" - exclude_flags+=( --exclude="$under" ) - fi - done < "$precious_dirs_file" -done - -git clean -xdff "${exclude_flags[@]}" diff --git a/bench-ci/guix/guix-codesign b/bench-ci/guix/guix-codesign deleted file mode 100755 index ac7aae3a1802..000000000000 --- a/bench-ci/guix/guix-codesign +++ /dev/null @@ -1,384 +0,0 @@ -#!/usr/bin/env bash -export LC_ALL=C -set -e -o pipefail - -# Source the common prelude, which: -# 1. Checks if we're at the top directory of the Bitcoin Core repository -# 2. Defines a few common functions and variables -# -# shellcheck source=libexec/prelude.bash -source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" - - -################### -## SANITY CHECKS ## -################### - -################ -# Required non-builtin commands should be invocable -################ - -check_tools cat mkdir git guix - -################ -# Required env vars should be non-empty -################ - -cmd_usage() { - cat < \\ - ./contrib/guix/guix-codesign - -EOF -} - -if [ -z "$DETACHED_SIGS_REPO" ]; then - cmd_usage - exit 1 -fi - -################ -# GUIX_BUILD_OPTIONS should be empty -################ -# -# GUIX_BUILD_OPTIONS is an environment variable recognized by guix commands that -# can perform builds. This seems like what we want instead of -# ADDITIONAL_GUIX_COMMON_FLAGS, but the value of GUIX_BUILD_OPTIONS is actually -# _appended_ to normal command-line options. Meaning that they will take -# precedence over the command-specific ADDITIONAL_GUIX__FLAGS. -# -# This seems like a poor user experience. Thus we check for GUIX_BUILD_OPTIONS's -# existence here and direct users of this script to use our (more flexible) -# custom environment variables. -if [ -n "$GUIX_BUILD_OPTIONS" ]; then -cat << EOF -Error: Environment variable GUIX_BUILD_OPTIONS is not empty: - '$GUIX_BUILD_OPTIONS' - -Unfortunately this script is incompatible with GUIX_BUILD_OPTIONS, please unset -GUIX_BUILD_OPTIONS and use ADDITIONAL_GUIX_COMMON_FLAGS to set build options -across guix commands or ADDITIONAL_GUIX__FLAGS to set build options for a -specific guix command. - -See contrib/guix/README.md for more details. -EOF -exit 1 -fi - -################ -# SOURCE_DATE_EPOCH should not unintentionally be set -################ - -check_source_date_epoch - -################ -# The codesignature git worktree should not be dirty -################ - -if ! git -C "$DETACHED_SIGS_REPO" diff-index --quiet HEAD -- && [ -z "$FORCE_DIRTY_WORKTREE" ]; then - cat << EOF -ERR: The DETACHED CODESIGNATURE git worktree is dirty, which may lead to broken builds. - - Aborting... - -Hint: To make your git worktree clean, You may want to: - 1. Commit your changes, - 2. Stash your changes, or - 3. Set the 'FORCE_DIRTY_WORKTREE' environment variable if you insist on - using a dirty worktree -EOF - exit 1 -fi - -################ -# Build directories should not exist -################ - -# Default to building for all supported HOSTs (overridable by environment) -export HOSTS="${HOSTS:-x86_64-w64-mingw32 x86_64-apple-darwin arm64-apple-darwin}" - -# Usage: distsrc_for_host HOST -# -# HOST: The current platform triple we're building for -# -distsrc_for_host() { - echo "${DISTSRC_BASE}/distsrc-${VERSION}-${1}-codesigned" -} - -# Accumulate a list of build directories that already exist... -hosts_distsrc_exists="" -for host in $HOSTS; do - if [ -e "$(distsrc_for_host "$host")" ]; then - hosts_distsrc_exists+=" ${host}" - fi -done - -if [ -n "$hosts_distsrc_exists" ]; then -# ...so that we can print them out nicely in an error message -cat << EOF -ERR: Build directories for this commit already exist for the following platform - triples you're attempting to build, probably because of previous builds. - Please remove, or otherwise deal with them prior to starting another build. - - Aborting... - -Hint: To blow everything away, you may want to use: - - $ ./contrib/guix/guix-clean - -Specifically, this will remove all files without an entry in the index, -excluding the SDK directory, the depends download cache, the depends built -packages cache, the garbage collector roots for Guix environments, and the -output directory. -EOF -for host in $hosts_distsrc_exists; do - echo " ${host} '$(distsrc_for_host "$host")'" -done -exit 1 -else - mkdir -p "$DISTSRC_BASE" -fi - - -################ -# Codesigning tarballs SHOULD exist -################ - -# Usage: outdir_for_host HOST SUFFIX -# -# HOST: The current platform triple we're building for -# -outdir_for_host() { - echo "${OUTDIR_BASE}/${1}${2:+-${2}}" -} - - -codesigning_tarball_for_host() { - case "$1" in - *mingw*) - echo "$(outdir_for_host "$1")/${DISTNAME}-win64-codesigning.tar.gz" - ;; - *darwin*) - echo "$(outdir_for_host "$1")/${DISTNAME}-${1}-codesigning.tar.gz" - ;; - *) - exit 1 - ;; - esac -} - -# Accumulate a list of build directories that already exist... -hosts_codesigning_tarball_missing="" -for host in $HOSTS; do - if [ ! -e "$(codesigning_tarball_for_host "$host")" ]; then - hosts_codesigning_tarball_missing+=" ${host}" - fi -done - -if [ -n "$hosts_codesigning_tarball_missing" ]; then - # ...so that we can print them out nicely in an error message - cat << EOF -ERR: Codesigning tarballs do not exist -... - -EOF -for host in $hosts_codesigning_tarball_missing; do - echo " ${host} '$(codesigning_tarball_for_host "$host")'" -done -exit 1 -fi - -################ -# Check that we can connect to the guix-daemon -################ - -cat << EOF -Checking that we can connect to the guix-daemon... - -Hint: If this hangs, you may want to try turning your guix-daemon off and on - again. - -EOF -if ! guix gc --list-failures > /dev/null; then - cat << EOF - -ERR: Failed to connect to the guix-daemon, please ensure that one is running and - reachable. -EOF - exit 1 -fi - -# Developer note: we could use `guix repl` for this check and run: -# -# (import (guix store)) (close-connection (open-connection)) -# -# However, the internal API is likely to change more than the CLI invocation - - -######### -# SETUP # -######### - -# Determine the maximum number of jobs to run simultaneously (overridable by -# environment) -JOBS="${JOBS:-$(nproc)}" - -# Determine the reference time used for determinism (overridable by environment) -SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git -c log.showSignature=false log --format=%at -1)}" - -# Make sure an output directory exists for our builds -OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}" -mkdir -p "$OUTDIR_BASE" - -# Usage: profiledir_for_host HOST SUFFIX -# -# HOST: The current platform triple we're building for -# -profiledir_for_host() { - echo "${PROFILES_BASE}/${1}${2:+-${2}}" -} - -######### -# BUILD # -######### - -# Function to be called when codesigning for host ${1} and the user interrupts -# the codesign -int_trap() { -cat << EOF -** INT received while codesigning ${1}, you may want to clean up the relevant - work directories (e.g. distsrc-*) before recodesigning - -Hint: To blow everything away, you may want to use: - - $ ./contrib/guix/guix-clean - -Specifically, this will remove all files without an entry in the index, -excluding the SDK directory, the depends download cache, the depends built -packages cache, the garbage collector roots for Guix environments, and the -output directory. -EOF -} - -# Deterministically build Bitcoin Core -# shellcheck disable=SC2153 -for host in $HOSTS; do - - # Display proper warning when the user interrupts the build - trap 'int_trap ${host}' INT - - ( - # Required for 'contrib/guix/manifest.scm' to output the right manifest - # for the particular $HOST we're building for - export HOST="$host" - - # shellcheck disable=SC2030 -cat << EOF -INFO: Codesigning ${VERSION:?not set} for platform triple ${HOST:?not set}: - ...using reference timestamp: ${SOURCE_DATE_EPOCH:?not set} - ...from worktree directory: '${PWD}' - ...bind-mounted in container to: '/bitcoin' - ...in build directory: '$(distsrc_for_host "$HOST")' - ...bind-mounted in container to: '$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")' - ...outputting in: '$(outdir_for_host "$HOST" codesigned)' - ...bind-mounted in container to: '$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST" codesigned)' - ...using detached signatures in: '${DETACHED_SIGS_REPO:?not set}' - ...bind-mounted in container to: '/detached-sigs' -EOF - - - # Run the build script 'contrib/guix/libexec/build.sh' in the build - # container specified by 'contrib/guix/manifest.scm'. - # - # Explanation of `guix shell` flags: - # - # --container run command within an isolated container - # - # Running in an isolated container minimizes build-time differences - # between machines and improves reproducibility - # - # --pure unset existing environment variables - # - # Same rationale as --container - # - # --no-cwd do not share current working directory with an - # isolated container - # - # When --container is specified, the default behavior is to share - # the current working directory with the isolated container at the - # same exact path (e.g. mapping '/home/satoshi/bitcoin/' to - # '/home/satoshi/bitcoin/'). This means that the $PWD inside the - # container becomes a source of irreproducibility. --no-cwd disables - # this behaviour. - # - # --share=SPEC for containers, share writable host file system - # according to SPEC - # - # --share="$PWD"=/bitcoin - # - # maps our current working directory to /bitcoin - # inside the isolated container, which we later cd - # into. - # - # While we don't want to map our current working directory to the - # same exact path (as this introduces irreproducibility), we do want - # it to be at a _fixed_ path _somewhere_ inside the isolated - # container so that we have something to build. '/bitcoin' was - # chosen arbitrarily. - # - # ${SOURCES_PATH:+--share="$SOURCES_PATH"} - # - # make the downloaded depends sources path available - # inside the isolated container - # - # The isolated container has no network access as it's in a - # different network namespace from the main machine, so we have to - # make the downloaded depends sources available to it. The sources - # should have been downloaded prior to this invocation. - # - # ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} - # - # fetch substitute from SUBSTITUTE_URLS if they are - # authorized - # - # Depending on the user's security model, it may be desirable to use - # substitutes (pre-built packages) from servers that the user trusts. - # Please read the README.md in the same directory as this file for - # more information. - # - # shellcheck disable=SC2086,SC2031 - time-machine shell --manifest="${PWD}/contrib/guix/manifest.scm" \ - --container \ - --pure \ - --no-cwd \ - --share="$PWD"=/bitcoin \ - --share="$DISTSRC_BASE"=/distsrc-base \ - --share="$OUTDIR_BASE"=/outdir-base \ - --share="$DETACHED_SIGS_REPO"=/detached-sigs \ - --expose="$(git rev-parse --git-common-dir)" \ - --expose="$(git -C "$DETACHED_SIGS_REPO" rev-parse --git-common-dir)" \ - ${SOURCES_PATH:+--share="$SOURCES_PATH"} \ - --cores="$JOBS" \ - --keep-failed \ - --fallback \ - --link-profile \ - --root="$(profiledir_for_host "${HOST}" codesigned)" \ - ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ - ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ - -- env HOST="$host" \ - DISTNAME="$DISTNAME" \ - JOBS="$JOBS" \ - SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \ - ${V:+V=1} \ - ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \ - DISTSRC="$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")" \ - OUTDIR="$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST" codesigned)" \ - DIST_ARCHIVE_BASE=/outdir-base/dist-archive \ - DETACHED_SIGS_REPO=/detached-sigs \ - CODESIGNING_TARBALL="$(OUTDIR_BASE=/outdir-base && codesigning_tarball_for_host "$HOST")" \ - bash -c "cd /bitcoin && bash contrib/guix/libexec/codesign.sh" - ) - -done diff --git a/bench-ci/guix/guix-verify b/bench-ci/guix/guix-verify deleted file mode 100755 index 02ae022741ba..000000000000 --- a/bench-ci/guix/guix-verify +++ /dev/null @@ -1,174 +0,0 @@ -#!/usr/bin/env bash -export LC_ALL=C -set -e -o pipefail - -# Source the common prelude, which: -# 1. Checks if we're at the top directory of the Bitcoin Core repository -# 2. Defines a few common functions and variables -# -# shellcheck source=libexec/prelude.bash -source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" - - -################### -## Sanity Checks ## -################### - -################ -# Required non-builtin commands should be invokable -################ - -check_tools cat diff gpg - -################ -# Required env vars should be non-empty -################ - -cmd_usage() { -cat < [ SIGNER= ] ./contrib/guix/guix-verify - -Example overriding signer's manifest to use as base - - env GUIX_SIGS_REPO=/home/dongcarl/guix.sigs SIGNER=achow101 ./contrib/guix/guix-verify - -EOF -} - -if [ -z "$GUIX_SIGS_REPO" ]; then - cmd_usage - exit 1 -fi - -################ -# GUIX_SIGS_REPO should exist as a directory -################ - -if [ ! -d "$GUIX_SIGS_REPO" ]; then -cat << EOF -ERR: The specified GUIX_SIGS_REPO is not an existent directory: - - '$GUIX_SIGS_REPO' - -Hint: Please clone the guix.sigs repository and point to it with the - GUIX_SIGS_REPO environment variable. - -EOF -cmd_usage -exit 1 -fi - -############## -## Verify ## -############## - -OUTSIGDIR_BASE="${GUIX_SIGS_REPO}/${VERSION}" -echo "Looking for signature directories in '${OUTSIGDIR_BASE}'" -echo "" - -# Usage: verify compare_manifest current_manifest -verify() { - local compare_manifest="$1" - local current_manifest="$2" - if ! gpg --quiet --batch --verify "$current_manifest".asc "$current_manifest" 1>&2; then - echo "ERR: Failed to verify GPG signature in '${current_manifest}'" - echo "" - echo "Hint: Either the signature is invalid or the public key is missing" - echo "" - failure=1 - elif ! diff --report-identical "$compare_manifest" "$current_manifest" 1>&2; then - echo "ERR: The SHA256SUMS attestation in these two directories differ:" - echo " '${compare_manifest}'" - echo " '${current_manifest}'" - echo "" - failure=1 - else - echo "Verified: '${current_manifest}'" - echo "" - fi -} - -shopt -s nullglob -all_noncodesigned=( "$OUTSIGDIR_BASE"/*/noncodesigned.SHA256SUMS ) -shopt -u nullglob - -echo "--------------------" -echo "" -if (( ${#all_noncodesigned[@]} )); then - compare_noncodesigned="${all_noncodesigned[0]}" - if [[ -n "$SIGNER" ]]; then - signer_noncodesigned="$OUTSIGDIR_BASE/$SIGNER/noncodesigned.SHA256SUMS" - if [[ -f "$signer_noncodesigned" ]]; then - echo "Using $SIGNER's manifest as the base to compare against" - compare_noncodesigned="$signer_noncodesigned" - else - echo "Unable to find $SIGNER's manifest, using the first one found" - fi - else - echo "No SIGNER provided, using the first manifest found" - fi - - for current_manifest in "${all_noncodesigned[@]}"; do - verify "$compare_noncodesigned" "$current_manifest" - done - - echo "DONE: Checking output signatures for noncodesigned.SHA256SUMS" - echo "" -else - echo "WARN: No signature directories with noncodesigned.SHA256SUMS found" - echo "" -fi - -shopt -s nullglob -all_all=( "$OUTSIGDIR_BASE"/*/all.SHA256SUMS ) -shopt -u nullglob - -echo "--------------------" -echo "" -if (( ${#all_all[@]} )); then - compare_all="${all_all[0]}" - if [[ -n "$SIGNER" ]]; then - signer_all="$OUTSIGDIR_BASE/$SIGNER/all.SHA256SUMS" - if [[ -f "$signer_all" ]]; then - echo "Using $SIGNER's manifest as the base to compare against" - compare_all="$signer_all" - else - echo "Unable to find $SIGNER's manifest, using the first one found" - fi - else - echo "No SIGNER provided, using the first manifest found" - fi - - for current_manifest in "${all_all[@]}"; do - verify "$compare_all" "$current_manifest" - done - - # Sanity check: there should be no entries that exist in - # noncodesigned.SHA256SUMS that doesn't exist in all.SHA256SUMS - if [[ "$(comm -23 <(sort "$compare_noncodesigned") <(sort "$compare_all") | wc -c)" -ne 0 ]]; then - echo "ERR: There are unique lines in noncodesigned.SHA256SUMS which" - echo " do not exist in all.SHA256SUMS, something went very wrong." - exit 1 - fi - - echo "DONE: Checking output signatures for all.SHA256SUMS" - echo "" -else - echo "WARN: No signature directories with all.SHA256SUMS found" - echo "" -fi - -echo "====================" -echo "" -if (( ${#all_noncodesigned[@]} + ${#all_all[@]} == 0 )); then - echo "ERR: Unable to perform any verifications as no signature directories" - echo " were found" - echo "" - exit 1 -fi - -if [ -n "$failure" ]; then - exit 1 -fi diff --git a/bench-ci/guix/libexec/build.sh b/bench-ci/guix/libexec/build.sh deleted file mode 100755 index 87ed1996cf38..000000000000 --- a/bench-ci/guix/libexec/build.sh +++ /dev/null @@ -1,416 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2019-2022 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -export LC_ALL=C -set -e -o pipefail -export TZ=UTC - -# Although Guix _does_ set umask when building its own packages (in our case, -# this is all packages in manifest.scm), it does not set it for `guix -# shell`. It does make sense for at least `guix shell --container` -# to set umask, so if that change gets merged upstream and we bump the -# time-machine to a commit which includes the aforementioned change, we can -# remove this line. -# -# This line should be placed before any commands which creates files. -umask 0022 - -if [ -n "$V" ]; then - # Print both unexpanded (-v) and expanded (-x) forms of commands as they are - # read from this file. - set -vx - # Set VERBOSE for CMake-based builds - export VERBOSE="$V" -fi - -# Check that required environment variables are set -cat << EOF -Required environment variables as seen inside the container: - DIST_ARCHIVE_BASE: ${DIST_ARCHIVE_BASE:?not set} - DISTNAME: ${DISTNAME:?not set} - HOST: ${HOST:?not set} - SOURCE_DATE_EPOCH: ${SOURCE_DATE_EPOCH:?not set} - JOBS: ${JOBS:?not set} - DISTSRC: ${DISTSRC:?not set} - OUTDIR: ${OUTDIR:?not set} -EOF - -ACTUAL_OUTDIR="${OUTDIR}" -OUTDIR="${DISTSRC}/output" - -##################### -# Environment Setup # -##################### - -# The depends folder also serves as a base-prefix for depends packages for -# $HOSTs after successfully building. -BASEPREFIX="${PWD}/depends" - -# Given a package name and an output name, return the path of that output in our -# current guix environment -store_path() { - grep --extended-regexp "/[^-]{32}-${1}-[^-]+${2:+-${2}}" "${GUIX_ENVIRONMENT}/manifest" \ - | head --lines=1 \ - | sed --expression='s|\x29*$||' \ - --expression='s|^[[:space:]]*"||' \ - --expression='s|"[[:space:]]*$||' -} - - -# Set environment variables to point the NATIVE toolchain to the right -# includes/libs -NATIVE_GCC="$(store_path gcc-toolchain)" - -unset LIBRARY_PATH -unset CPATH -unset C_INCLUDE_PATH -unset CPLUS_INCLUDE_PATH -unset OBJC_INCLUDE_PATH -unset OBJCPLUS_INCLUDE_PATH - -# Set native toolchain -build_CC="${NATIVE_GCC}/bin/gcc -isystem ${NATIVE_GCC}/include" -build_CXX="${NATIVE_GCC}/bin/g++ -isystem ${NATIVE_GCC}/include/c++ -isystem ${NATIVE_GCC}/include" -export C_INCLUDE_PATH="${NATIVE_GCC}/include" -export CPLUS_INCLUDE_PATH="${NATIVE_GCC}/include/c++:${NATIVE_GCC}/include" - -case "$HOST" in - *darwin*) export LIBRARY_PATH="${NATIVE_GCC}/lib" ;; # Required for native packages - *mingw*) export LIBRARY_PATH="${NATIVE_GCC}/lib" ;; - *) - NATIVE_GCC_STATIC="$(store_path gcc-toolchain static)" - export LIBRARY_PATH="${NATIVE_GCC}/lib:${NATIVE_GCC_STATIC}/lib" - ;; -esac - -# Set environment variables to point the CROSS toolchain to the right -# includes/libs for $HOST -case "$HOST" in - *mingw*) - # Determine output paths to use in CROSS_* environment variables - CROSS_GLIBC="$(store_path "mingw-w64-x86_64-winpthreads")" - CROSS_GCC="$(store_path "gcc-cross-${HOST}")" - CROSS_GCC_LIB_STORE="$(store_path "gcc-cross-${HOST}" lib)" - CROSS_GCC_LIBS=( "${CROSS_GCC_LIB_STORE}/lib/gcc/${HOST}"/* ) # This expands to an array of directories... - CROSS_GCC_LIB="${CROSS_GCC_LIBS[0]}" # ...we just want the first one (there should only be one) - - # The search path ordering is generally: - # 1. gcc-related search paths - # 2. libc-related search paths - # 2. kernel-header-related search paths (not applicable to mingw-w64 hosts) - export CROSS_C_INCLUDE_PATH="${CROSS_GCC_LIB}/include:${CROSS_GCC_LIB}/include-fixed:${CROSS_GLIBC}/include" - export CROSS_CPLUS_INCLUDE_PATH="${CROSS_GCC}/include/c++:${CROSS_GCC}/include/c++/${HOST}:${CROSS_GCC}/include/c++/backward:${CROSS_C_INCLUDE_PATH}" - export CROSS_LIBRARY_PATH="${CROSS_GCC_LIB_STORE}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib" - ;; - *darwin*) - # The CROSS toolchain for darwin uses the SDK and ignores environment variables. - # See depends/hosts/darwin.mk for more details. - ;; - *linux*) - CROSS_GLIBC="$(store_path "glibc-cross-${HOST}")" - CROSS_GLIBC_STATIC="$(store_path "glibc-cross-${HOST}" static)" - CROSS_KERNEL="$(store_path "linux-libre-headers-cross-${HOST}")" - CROSS_GCC="$(store_path "gcc-cross-${HOST}")" - CROSS_GCC_LIB_STORE="$(store_path "gcc-cross-${HOST}" lib)" - CROSS_GCC_LIBS=( "${CROSS_GCC_LIB_STORE}/lib/gcc/${HOST}"/* ) # This expands to an array of directories... - CROSS_GCC_LIB="${CROSS_GCC_LIBS[0]}" # ...we just want the first one (there should only be one) - - export CROSS_C_INCLUDE_PATH="${CROSS_GCC_LIB}/include:${CROSS_GCC_LIB}/include-fixed:${CROSS_GLIBC}/include:${CROSS_KERNEL}/include" - export CROSS_CPLUS_INCLUDE_PATH="${CROSS_GCC}/include/c++:${CROSS_GCC}/include/c++/${HOST}:${CROSS_GCC}/include/c++/backward:${CROSS_C_INCLUDE_PATH}" - export CROSS_LIBRARY_PATH="${CROSS_GCC_LIB_STORE}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib:${CROSS_GLIBC_STATIC}/lib" - ;; - *) - exit 1 ;; -esac - -# Sanity check CROSS_*_PATH directories -IFS=':' read -ra PATHS <<< "${CROSS_C_INCLUDE_PATH}:${CROSS_CPLUS_INCLUDE_PATH}:${CROSS_LIBRARY_PATH}" -for p in "${PATHS[@]}"; do - if [ -n "$p" ] && [ ! -d "$p" ]; then - echo "'$p' doesn't exist or isn't a directory... Aborting..." - exit 1 - fi -done - -# Disable Guix ld auto-rpath behavior -export GUIX_LD_WRAPPER_DISABLE_RPATH=yes - -# Make /usr/bin if it doesn't exist -[ -e /usr/bin ] || mkdir -p /usr/bin - -# Symlink env to a conventional path -[ -e /usr/bin/env ] || ln -s --no-dereference "$(command -v env)" /usr/bin/env - -# Determine the correct value for -Wl,--dynamic-linker for the current $HOST -case "$HOST" in - x86_64-linux-gnu) ;; - *linux*) - glibc_dynamic_linker=$( - case "$HOST" in - arm-linux-gnueabihf) echo /lib/ld-linux-armhf.so.3 ;; - aarch64-linux-gnu) echo /lib/ld-linux-aarch64.so.1 ;; - riscv64-linux-gnu) echo /lib/ld-linux-riscv64-lp64d.so.1 ;; - powerpc64-linux-gnu) echo /lib64/ld64.so.1;; - powerpc64le-linux-gnu) echo /lib64/ld64.so.2;; - *) exit 1 ;; - esac - ) - ;; -esac - -# Environment variables for determinism -export TAR_OPTIONS="--owner=0 --group=0 --numeric-owner --mtime='@${SOURCE_DATE_EPOCH}' --sort=name" -export TZ="UTC" - -#################### -# Depends Building # -#################### - -# Build the depends tree, overriding variables that assume multilib gcc -make -C depends --jobs="$JOBS" HOST="$HOST" \ - ${V:+V=1} \ - ${SOURCES_PATH+SOURCES_PATH="$SOURCES_PATH"} \ - ${BASE_CACHE+BASE_CACHE="$BASE_CACHE"} \ - ${SDK_PATH+SDK_PATH="$SDK_PATH"} \ - ${build_CC+build_CC="$build_CC"} \ - ${build_CXX+build_CXX="$build_CXX"} \ - x86_64_linux_CC=x86_64-linux-gnu-gcc \ - x86_64_linux_CXX=x86_64-linux-gnu-g++ \ - x86_64_linux_AR=x86_64-linux-gnu-gcc-ar \ - x86_64_linux_RANLIB=x86_64-linux-gnu-gcc-ranlib \ - x86_64_linux_NM=x86_64-linux-gnu-gcc-nm \ - x86_64_linux_STRIP=x86_64-linux-gnu-strip \ - NO_QT=1 \ - NO_QR=1 \ - NO_ZMQ=1 \ - NO_WALLET=1 \ - NO_BDB=1 \ - NO_USDT=1 - -case "$HOST" in - *darwin*) - # Unset now that Qt is built - unset LIBRARY_PATH - ;; -esac - -########################### -# Source Tarball Building # -########################### - -GIT_ARCHIVE="${DIST_ARCHIVE_BASE}/${DISTNAME}.tar.gz" - -# Create the source tarball if not already there -if [ ! -e "$GIT_ARCHIVE" ]; then - mkdir -p "$(dirname "$GIT_ARCHIVE")" - git archive --prefix="${DISTNAME}/" --output="$GIT_ARCHIVE" HEAD -fi - -mkdir -p "$OUTDIR" - -########################### -# Binary Tarball Building # -########################### - -# CONFIGFLAGS -CONFIGFLAGS="-DREDUCE_EXPORTS=ON -DBUILD_BENCH=OFF -DBUILD_GUI_TESTS=OFF -DBUILD_FUZZ_BINARY=OFF -DCMAKE_SKIP_RPATH=TRUE" - -# BENCHCOINFLAGS -BENCHCOINFLAGS="-DBUILD_CLI=OFF -DBUILD_TESTS=OFF -DCMAKE_CXX_FLAGS=-fno-omit-frame-pointer" - -# CFLAGS -HOST_CFLAGS="-O2 -g" -HOST_CFLAGS+=$(find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;) -case "$HOST" in - *mingw*) HOST_CFLAGS+=" -fno-ident" ;; - *darwin*) unset HOST_CFLAGS ;; -esac - -# CXXFLAGS -HOST_CXXFLAGS="$HOST_CFLAGS" - -case "$HOST" in - arm-linux-gnueabihf) HOST_CXXFLAGS="${HOST_CXXFLAGS} -Wno-psabi" ;; -esac - -# LDFLAGS -case "$HOST" in - x86_64-linux-gnu) HOST_LDFLAGS=" -static-pie -static-libgcc -Wl,-O2" ;; - *linux*) HOST_LDFLAGS="-Wl,--as-needed -Wl,--dynamic-linker=$glibc_dynamic_linker -static-libstdc++ -Wl,-O2" ;; - *mingw*) HOST_LDFLAGS="-Wl,--no-insert-timestamp" ;; -esac - -mkdir -p "$DISTSRC" -( - cd "$DISTSRC" - - # Extract the source tarball - tar --strip-components=1 -xf "${GIT_ARCHIVE}" - - # Configure this DISTSRC for $HOST - # shellcheck disable=SC2086 - env CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" LDFLAGS="${HOST_LDFLAGS}" \ - cmake -S . -B build \ - --toolchain "${BASEPREFIX}/${HOST}/toolchain.cmake" \ - -DWITH_CCACHE=OFF \ - -Werror=dev \ - ${CONFIGFLAGS} \ - ${BENCHCOINFLAGS} - - # Build Bitcoin Core - cmake --build build -j "$JOBS" ${V:+--verbose} - - # Perform basic security checks on a series of executables. - # cmake --build build -j 1 --target check-security ${V:+--verbose} - # Check that executables only contain allowed version symbols. - # cmake --build build -j 1 --target check-symbols ${V:+--verbose} - - mkdir -p "$OUTDIR" - - # Make the os-specific installers - case "$HOST" in - *mingw*) - cmake --build build -j "$JOBS" -t deploy ${V:+--verbose} - mv build/bitcoin-win64-setup.exe "${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe" - ;; - esac - - # Setup the directory where our Bitcoin Core build for HOST will be - # installed. This directory will also later serve as the input for our - # binary tarballs. - INSTALLPATH="${PWD}/installed/${DISTNAME}" - mkdir -p "${INSTALLPATH}" - # Install built Bitcoin Core to $INSTALLPATH - case "$HOST" in - *darwin*) - # This workaround can be dropped for CMake >= 3.27. - # See the upstream commit 689616785f76acd844fd448c51c5b2a0711aafa2. - find build -name 'cmake_install.cmake' -exec sed -i 's| -u -r | |g' {} + - - cmake --install build --strip --prefix "${INSTALLPATH}" ${V:+--verbose} - ;; - *) - cmake --install build --prefix "${INSTALLPATH}" ${V:+--verbose} - ;; - esac - - ( - cd installed - - case "$HOST" in - *darwin*) ;; - *) - # Split binaries from their debug symbols - { - find "${DISTNAME}/bin" "${DISTNAME}/libexec" -type f -executable -print0 - } | xargs -0 -P"$JOBS" -I{} "${DISTSRC}/build/split-debug.sh" {} {} {}.dbg - ;; - esac - - case "$HOST" in - *mingw*) - cp "${DISTSRC}/doc/README_windows.txt" "${DISTNAME}/readme.txt" - ;; - *linux*) - cp "${DISTSRC}/README.md" "${DISTNAME}/" - ;; - esac - - # copy over the example bitcoin.conf file. if contrib/devtools/gen-bitcoin-conf.sh - # has not been run before buildling, this file will be a stub - cp "${DISTSRC}/share/examples/bitcoin.conf" "${DISTNAME}/" - - cp -r "${DISTSRC}/share/rpcauth" "${DISTNAME}/share/" - - # Deterministically produce {non-,}debug binary tarballs ready - # for release - case "$HOST" in - *mingw*) - find "${DISTNAME}" -not -name "*.dbg" -print0 \ - | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" - find "${DISTNAME}" -not -name "*.dbg" \ - | sort \ - | zip -X@ "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}-unsigned.zip" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}-unsigned.zip" && exit 1 ) - find "${DISTNAME}" -name "*.dbg" -print0 \ - | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" - find "${DISTNAME}" -name "*.dbg" \ - | sort \ - | zip -X@ "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}-debug.zip" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}-debug.zip" && exit 1 ) - ;; - *linux*) - find "${DISTNAME}" -not -name "*.dbg" -print0 \ - | sort --zero-terminated \ - | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ - | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" && exit 1 ) - find "${DISTNAME}" -name "*.dbg" -print0 \ - | sort --zero-terminated \ - | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ - | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}-debug.tar.gz" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}-debug.tar.gz" && exit 1 ) - ;; - *darwin*) - find "${DISTNAME}" -print0 \ - | sort --zero-terminated \ - | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ - | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}-unsigned.tar.gz" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}-unsigned.tar.gz" && exit 1 ) - ;; - esac - ) # $DISTSRC/installed - - # Finally make tarballs for codesigning - case "$HOST" in - *mingw*) - cp -rf --target-directory=. contrib/windeploy - ( - cd ./windeploy - mkdir -p unsigned - cp --target-directory=unsigned/ "${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe" - cp -r --target-directory=unsigned/ "${INSTALLPATH}" - find unsigned/ -name "*.dbg" -print0 \ - | xargs -0r rm - find . -print0 \ - | sort --zero-terminated \ - | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ - | gzip -9n > "${OUTDIR}/${DISTNAME}-win64-codesigning.tar.gz" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-win64-codesigning.tar.gz" && exit 1 ) - ) - ;; - *darwin*) - cmake --build build --target deploy ${V:+--verbose} - mv build/dist/bitcoin-macos-app.zip "${OUTDIR}/${DISTNAME}-${HOST}-unsigned.zip" - mkdir -p "unsigned-app-${HOST}" - cp --target-directory="unsigned-app-${HOST}" \ - contrib/macdeploy/detached-sig-create.sh - mv --target-directory="unsigned-app-${HOST}" build/dist - cp -r --target-directory="unsigned-app-${HOST}" "${INSTALLPATH}" - ( - cd "unsigned-app-${HOST}" - find . -print0 \ - | sort --zero-terminated \ - | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ - | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}-codesigning.tar.gz" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}-codesigning.tar.gz" && exit 1 ) - ) - ;; - esac -) # $DISTSRC - -rm -rf "$ACTUAL_OUTDIR" -mv --no-target-directory "$OUTDIR" "$ACTUAL_OUTDIR" \ - || ( rm -rf "$ACTUAL_OUTDIR" && exit 1 ) - -( - cd /outdir-base - { - echo "$GIT_ARCHIVE" - find "$ACTUAL_OUTDIR" -type f - } | xargs realpath --relative-base="$PWD" \ - | xargs sha256sum \ - | sort -k2 \ - | sponge "$ACTUAL_OUTDIR"/SHA256SUMS.part -) diff --git a/bench-ci/guix/libexec/codesign.sh b/bench-ci/guix/libexec/codesign.sh deleted file mode 100755 index fe86065350e9..000000000000 --- a/bench-ci/guix/libexec/codesign.sh +++ /dev/null @@ -1,153 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2021-2022 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -export LC_ALL=C -set -e -o pipefail - -# Environment variables for determinism -export TAR_OPTIONS="--owner=0 --group=0 --numeric-owner --mtime='@${SOURCE_DATE_EPOCH}' --sort=name" -export TZ=UTC - -# Although Guix _does_ set umask when building its own packages (in our case, -# this is all packages in manifest.scm), it does not set it for `guix -# shell`. It does make sense for at least `guix shell --container` -# to set umask, so if that change gets merged upstream and we bump the -# time-machine to a commit which includes the aforementioned change, we can -# remove this line. -# -# This line should be placed before any commands which creates files. -umask 0022 - -if [ -n "$V" ]; then - # Print both unexpanded (-v) and expanded (-x) forms of commands as they are - # read from this file. - set -vx - # Set VERBOSE for CMake-based builds - export VERBOSE="$V" -fi - -# Check that required environment variables are set -cat << EOF -Required environment variables as seen inside the container: - CODESIGNING_TARBALL: ${CODESIGNING_TARBALL:?not set} - DETACHED_SIGS_REPO: ${DETACHED_SIGS_REPO:?not set} - DIST_ARCHIVE_BASE: ${DIST_ARCHIVE_BASE:?not set} - DISTNAME: ${DISTNAME:?not set} - HOST: ${HOST:?not set} - SOURCE_DATE_EPOCH: ${SOURCE_DATE_EPOCH:?not set} - DISTSRC: ${DISTSRC:?not set} - OUTDIR: ${OUTDIR:?not set} -EOF - -ACTUAL_OUTDIR="${OUTDIR}" -OUTDIR="${DISTSRC}/output" - -git_head_version() { - local recent_tag - if recent_tag="$(git -C "$1" describe --exact-match HEAD 2> /dev/null)"; then - echo "${recent_tag#v}" - else - git -C "$1" rev-parse --short=12 HEAD - fi -} - -CODESIGNATURE_GIT_ARCHIVE="${DIST_ARCHIVE_BASE}/${DISTNAME}-codesignatures-$(git_head_version "$DETACHED_SIGS_REPO").tar.gz" - -# Create the codesignature tarball if not already there -if [ ! -e "$CODESIGNATURE_GIT_ARCHIVE" ]; then - mkdir -p "$(dirname "$CODESIGNATURE_GIT_ARCHIVE")" - git -C "$DETACHED_SIGS_REPO" archive --output="$CODESIGNATURE_GIT_ARCHIVE" HEAD -fi - -mkdir -p "$OUTDIR" - -mkdir -p "$DISTSRC" -( - cd "$DISTSRC" - - tar -xf "$CODESIGNING_TARBALL" - - mkdir -p codesignatures - tar -C codesignatures -xf "$CODESIGNATURE_GIT_ARCHIVE" - - case "$HOST" in - *mingw*) - # Apply detached codesignatures - WORKDIR=".tmp" - mkdir -p ${WORKDIR} - cp -r --target-directory="${WORKDIR}" "unsigned/${DISTNAME}" - find "${WORKDIR}/${DISTNAME}" -name "*.exe" -type f -exec rm {} \; - find unsigned/ -name "*.exe" -type f | while read -r bin - do - bin_base="$(realpath --relative-to=unsigned/ "${bin}")" - mkdir -p "${WORKDIR}/$(dirname "${bin_base}")" - osslsigncode attach-signature \ - -in "${bin}" \ - -out "${WORKDIR}/${bin_base/-unsigned}" \ - -CAfile "$GUIX_ENVIRONMENT/etc/ssl/certs/ca-certificates.crt" \ - -sigin codesignatures/win/"${bin_base}".pem - done - - # Move installer to outdir - cd "${WORKDIR}" - find . -name "*setup.exe" -print0 \ - | xargs -0r mv --target-directory="${OUTDIR}" - - # Make .zip from binaries - find "${DISTNAME}" -print0 \ - | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" - find "${DISTNAME}" \ - | sort \ - | zip -X@ "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}.zip" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}.zip" && exit 1 ) - ;; - *darwin*) - case "$HOST" in - arm64*) ARCH="arm64" ;; - x86_64*) ARCH="x86_64" ;; - esac - - # Apply detached codesignatures (in-place) - signapple apply dist/Bitcoin-Qt.app codesignatures/osx/"${HOST}"/dist/Bitcoin-Qt.app - find "${DISTNAME}" \( -wholename "*/bin/*" -o -wholename "*/libexec/*" \) -type f | while read -r bin - do - signapple apply "${bin}" "codesignatures/osx/${HOST}/${bin}.${ARCH}sign" - done - - # Make a .zip from dist/ - cd dist/ - find . -print0 \ - | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" - find . | sort \ - | zip -X@ "${OUTDIR}/${DISTNAME}-${HOST}.zip" - cd .. - - # Make a .tar.gz from bins - find "${DISTNAME}" -print0 \ - | sort --zero-terminated \ - | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ - | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" && exit 1 ) - ;; - *) - exit 1 - ;; - esac -) # $DISTSRC - -rm -rf "$ACTUAL_OUTDIR" -mv --no-target-directory "$OUTDIR" "$ACTUAL_OUTDIR" \ - || ( rm -rf "$ACTUAL_OUTDIR" && exit 1 ) - -( - cd /outdir-base - { - echo "$CODESIGNING_TARBALL" - echo "$CODESIGNATURE_GIT_ARCHIVE" - find "$ACTUAL_OUTDIR" -type f - } | xargs realpath --relative-base="$PWD" \ - | xargs sha256sum \ - | sort -k2 \ - | sponge "$ACTUAL_OUTDIR"/SHA256SUMS.part -) diff --git a/bench-ci/guix/libexec/prelude.bash b/bench-ci/guix/libexec/prelude.bash deleted file mode 100644 index 3cf568279f70..000000000000 --- a/bench-ci/guix/libexec/prelude.bash +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env bash -export LC_ALL=C -set -e -o pipefail - -# shellcheck source=contrib/shell/realpath.bash -source contrib/shell/realpath.bash - -# shellcheck source=contrib/shell/git-utils.bash -source contrib/shell/git-utils.bash - -# Source guix profile from the runner home directory -GUIX_PROFILE=/home/github-runner/.config/guix/current -. "$GUIX_PROFILE/etc/profile" || true -echo "Using the following guix command:" -command -v guix -echo "Guix command symlink points to:" -readlink -f "$(command -v guix)" -echo "Current Guix profile:" -echo "$GUIX_PROFILE" -echo "Profile generation info:" -guix describe - -################ -# Required non-builtin commands should be invocable -################ - -check_tools() { - for cmd in "$@"; do - if ! command -v "$cmd" > /dev/null 2>&1; then - echo "ERR: This script requires that '$cmd' is installed and available in your \$PATH" - exit 1 - fi - done -} - -################ -# SOURCE_DATE_EPOCH should not unintentionally be set -################ - -check_source_date_epoch() { - if [ -n "$SOURCE_DATE_EPOCH" ] && [ -z "$FORCE_SOURCE_DATE_EPOCH" ]; then - cat << EOF -ERR: Environment variable SOURCE_DATE_EPOCH is set which may break reproducibility. - - Aborting... - -Hint: You may want to: - 1. Unset this variable: \`unset SOURCE_DATE_EPOCH\` before rebuilding - 2. Set the 'FORCE_SOURCE_DATE_EPOCH' environment variable if you insist on - using your own epoch -EOF - exit 1 - fi -} - -check_tools cat env readlink dirname basename git - -################ -# We should be at the top directory of the repository -################ - -same_dir() { - local resolved1 resolved2 - resolved1="$(bash_realpath "${1}")" - resolved2="$(bash_realpath "${2}")" - [ "$resolved1" = "$resolved2" ] -} - -if ! same_dir "${PWD}" "$(git_root)"; then -cat << EOF -ERR: This script must be invoked from the top level of the git repository - -Hint: This may look something like: - env FOO=BAR ./contrib/guix/guix- - -EOF -exit 1 -fi - -################ -# Execute "$@" in a pinned, possibly older version of Guix, for reproducibility -# across time. -time-machine() { - # shellcheck disable=SC2086 - guix time-machine --url=https://github.com/fanquake/guix.git \ - --commit=5cb84f2013c5b1e48a7d0e617032266f1e6059e2 \ - --cores="$JOBS" \ - --keep-failed \ - --fallback \ - ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ - ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS} \ - -- "$@" -} - - -################ -# Set common variables -################ - -VERSION="${FORCE_VERSION:-$(git_head_version)}" -DISTNAME="${DISTNAME:-bitcoin-${VERSION}}" - -version_base_prefix="${PWD}/guix-build-" -VERSION_BASE="${version_base_prefix}${VERSION}" # TOP - -DISTSRC_BASE="${DISTSRC_BASE:-${VERSION_BASE}}" - -OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}" - -var_base_basename="var" -VAR_BASE="${VAR_BASE:-${VERSION_BASE}/${var_base_basename}}" - -profiles_base_basename="profiles" -PROFILES_BASE="${PROFILES_BASE:-${VAR_BASE}/${profiles_base_basename}}" diff --git a/bench-ci/guix/manifest.scm b/bench-ci/guix/manifest.scm deleted file mode 100644 index f89eccc85360..000000000000 --- a/bench-ci/guix/manifest.scm +++ /dev/null @@ -1,610 +0,0 @@ -(use-modules (gnu packages) - ((gnu packages bash) #:select (bash-minimal)) - (gnu packages bison) - ((gnu packages certs) #:select (nss-certs)) - ((gnu packages cmake) #:select (cmake-minimal)) - (gnu packages commencement) - (gnu packages compression) - (gnu packages cross-base) - (gnu packages gawk) - (gnu packages gcc) - ((gnu packages installers) #:select (nsis-x86_64)) - ((gnu packages linux) #:select (linux-libre-headers-6.1)) - (gnu packages llvm) - (gnu packages mingw) - (gnu packages ninja) - (gnu packages pkg-config) - ((gnu packages python) #:select (python-minimal)) - ((gnu packages python-build) #:select (python-poetry-core)) - ((gnu packages python-crypto) #:select (python-asn1crypto)) - ((gnu packages python-science) #:select (python-scikit-build-core)) - ((gnu packages python-xyz) #:select (python-pydantic-2)) - ((gnu packages tls) #:select (openssl)) - ((gnu packages version-control) #:select (git-minimal)) - (guix build-system cmake) - (guix build-system gnu) - (guix build-system python) - (guix build-system pyproject) - (guix build-system trivial) - (guix download) - (guix gexp) - (guix git-download) - ((guix licenses) #:prefix license:) - (guix packages) - ((guix utils) #:select (cc-for-target substitute-keyword-arguments))) - -(define-syntax-rule (search-our-patches file-name ...) - "Return the list of absolute file names corresponding to each -FILE-NAME found in ./patches relative to the current file." - (parameterize - ((%patch-path (list (string-append (dirname (current-filename)) "/patches")))) - (list (search-patch file-name) ...))) - -(define building-on (string-append "--build=" (list-ref (string-split (%current-system) #\-) 0) "-guix-linux-gnu")) - -(define (make-cross-toolchain target - base-gcc-for-libc - base-kernel-headers - base-libc - base-gcc) - "Create a cross-compilation toolchain package for TARGET" - (let* ((xbinutils (cross-binutils target)) - ;; 1. Build a cross-compiling gcc without targeting any libc, derived - ;; from BASE-GCC-FOR-LIBC - (xgcc-sans-libc (cross-gcc target - #:xgcc base-gcc-for-libc - #:xbinutils xbinutils)) - ;; 2. Build cross-compiled kernel headers with XGCC-SANS-LIBC, derived - ;; from BASE-KERNEL-HEADERS - (xkernel (cross-kernel-headers target - #:linux-headers base-kernel-headers - #:xgcc xgcc-sans-libc - #:xbinutils xbinutils)) - ;; 3. Build a cross-compiled libc with XGCC-SANS-LIBC and XKERNEL, - ;; derived from BASE-LIBC - (xlibc (cross-libc target - #:libc base-libc - #:xgcc xgcc-sans-libc - #:xbinutils xbinutils - #:xheaders xkernel)) - ;; 4. Build a cross-compiling gcc targeting XLIBC, derived from - ;; BASE-GCC - (xgcc (cross-gcc target - #:xgcc base-gcc - #:xbinutils xbinutils - #:libc xlibc))) - ;; Define a meta-package that propagates the resulting XBINUTILS, XLIBC, and - ;; XGCC - (package - (name (string-append target "-toolchain")) - (version (package-version xgcc)) - (source #f) - (build-system trivial-build-system) - (arguments '(#:builder (begin (mkdir %output) #t))) - (propagated-inputs - (list xbinutils - xlibc - xgcc - `(,xlibc "static") - `(,xgcc "lib"))) - (synopsis (string-append "Complete GCC tool chain for " target)) - (description (string-append "This package provides a complete GCC tool -chain for " target " development.")) - (home-page (package-home-page xgcc)) - (license (package-license xgcc))))) - -(define base-gcc gcc-13) ;; 13.3.0 - -(define base-linux-kernel-headers linux-libre-headers-6.1) - -(define* (make-bitcoin-cross-toolchain target - #:key - (base-gcc-for-libc linux-base-gcc) - (base-kernel-headers base-linux-kernel-headers) - (base-libc glibc-2.31) - (base-gcc linux-base-gcc)) - "Convenience wrapper around MAKE-CROSS-TOOLCHAIN with default values -desirable for building Bitcoin Core release binaries." - (make-cross-toolchain target - base-gcc-for-libc - base-kernel-headers - base-libc - base-gcc)) - -(define (gcc-mingw-patches gcc) - (package-with-extra-patches gcc - (search-our-patches "gcc-remap-guix-store.patch"))) - -(define (binutils-mingw-patches binutils) - (package-with-extra-patches binutils - (search-our-patches "binutils-unaligned-default.patch"))) - -(define (winpthreads-patches mingw-w64-x86_64-winpthreads) - (package-with-extra-patches mingw-w64-x86_64-winpthreads - (search-our-patches "winpthreads-remap-guix-store.patch"))) - -(define (make-mingw-pthreads-cross-toolchain target) - "Create a cross-compilation toolchain package for TARGET" - (let* ((xbinutils (binutils-mingw-patches (cross-binutils target))) - (machine (substring target 0 (string-index target #\-))) - (pthreads-xlibc (winpthreads-patches (make-mingw-w64 machine - #:xgcc (cross-gcc target #:xgcc (gcc-mingw-patches base-gcc)) - #:with-winpthreads? #t))) - (pthreads-xgcc (cross-gcc target - #:xgcc (gcc-mingw-patches mingw-w64-base-gcc) - #:xbinutils xbinutils - #:libc pthreads-xlibc))) - ;; Define a meta-package that propagates the resulting XBINUTILS, XLIBC, and - ;; XGCC - (package - (name (string-append target "-posix-toolchain")) - (version (package-version pthreads-xgcc)) - (source #f) - (build-system trivial-build-system) - (arguments '(#:builder (begin (mkdir %output) #t))) - (propagated-inputs - (list xbinutils - pthreads-xlibc - pthreads-xgcc - `(,pthreads-xgcc "lib"))) - (synopsis (string-append "Complete GCC tool chain for " target)) - (description (string-append "This package provides a complete GCC tool -chain for " target " development.")) - (home-page (package-home-page pthreads-xgcc)) - (license (package-license pthreads-xgcc))))) - -;; While LIEF is packaged in Guix, we maintain our own package, -;; to simplify building, and more easily apply updates. -;; Moreover, the Guix's package uses cmake, which caused build -;; failure; see https://github.com/bitcoin/bitcoin/pull/27296. -(define-public python-lief - (package - (name "python-lief") - (version "0.16.6") - (source (origin - (method git-fetch) - (uri (git-reference - (url "https://github.com/lief-project/LIEF") - (commit version))) - (file-name (git-file-name name version)) - (sha256 - (base32 - "1pq9nagrnkl1x943bqnpiyxmkd9vk99znfxiwqp6vf012b50bz2a")) - (patches (search-our-patches "lief-scikit-0-9.patch")))) - (build-system pyproject-build-system) - (native-inputs (list cmake-minimal - ninja - python-scikit-build-core - python-pydantic-2)) - (arguments - (list - #:tests? #f ;needs network - #:phases #~(modify-phases %standard-phases - (add-before 'build 'set-pythonpath - (lambda _ - (setenv "PYTHONPATH" - (string-append (string-append (getcwd) "/api/python/backend") - ":" (or (getenv "PYTHONPATH") ""))))) - (add-after 'set-pythonpath 'change-directory - (lambda _ - (chdir "api/python")))))) - (home-page "https://github.com/lief-project/LIEF") - (synopsis "Library to instrument executable formats") - (description - "@code{python-lief} is a cross platform library which can parse, modify -and abstract ELF, PE and MachO formats.") - (license license:asl2.0))) - -(define osslsigncode - (package - (name "osslsigncode") - (version "2.5") - (source (origin - (method git-fetch) - (uri (git-reference - (url "https://github.com/mtrojnar/osslsigncode") - (commit version))) - (sha256 - (base32 - "1j47vwq4caxfv0xw68kw5yh00qcpbd56d7rq6c483ma3y7s96yyz")))) - (build-system cmake-build-system) - (inputs (list openssl)) - (home-page "https://github.com/mtrojnar/osslsigncode") - (synopsis "Authenticode signing and timestamping tool") - (description "osslsigncode is a small tool that implements part of the -functionality of the Microsoft tool signtool.exe - more exactly the Authenticode -signing and timestamping. But osslsigncode is based on OpenSSL and cURL, and -thus should be able to compile on most platforms where these exist.") - (license license:gpl3+))) ; license is with openssl exception - -(define-public python-elfesteem - (let ((commit "2eb1e5384ff7a220fd1afacd4a0170acff54fe56")) - (package - (name "python-elfesteem") - (version (git-version "0.1" "1" commit)) - (source - (origin - (method git-fetch) - (uri (git-reference - (url "https://github.com/LRGH/elfesteem") - (commit commit))) - (file-name (git-file-name name commit)) - (sha256 - (base32 - "07x6p8clh11z8s1n2kdxrqwqm2almgc5qpkcr9ckb6y5ivjdr5r6")))) - (build-system python-build-system) - ;; There are no tests, but attempting to run python setup.py test leads to - ;; PYTHONPATH problems, just disable the test - (arguments '(#:tests? #f)) - (home-page "https://github.com/LRGH/elfesteem") - (synopsis "ELF/PE/Mach-O parsing library") - (description "elfesteem parses ELF, PE and Mach-O files.") - (license license:lgpl2.1)))) - -(define-public python-oscrypto - (package - (name "python-oscrypto") - (version "1.3.0") - (source - (origin - (method git-fetch) - (uri (git-reference - (url "https://github.com/wbond/oscrypto") - (commit version))) - (file-name (git-file-name name version)) - (sha256 - (base32 - "1v5wkmzcyiqy39db8j2dvkdrv2nlsc48556h73x4dzjwd6kg4q0a")) - (patches (search-our-patches "oscrypto-hard-code-openssl.patch")))) - (build-system python-build-system) - (native-search-paths - (list (search-path-specification - (variable "SSL_CERT_FILE") - (file-type 'regular) - (separator #f) ;single entry - (files '("etc/ssl/certs/ca-certificates.crt"))))) - - (propagated-inputs - (list python-asn1crypto openssl)) - (arguments - `(#:phases - (modify-phases %standard-phases - (add-after 'unpack 'hard-code-path-to-libscrypt - (lambda* (#:key inputs #:allow-other-keys) - (let ((openssl (assoc-ref inputs "openssl"))) - (substitute* "oscrypto/__init__.py" - (("@GUIX_OSCRYPTO_USE_OPENSSL@") - (string-append openssl "/lib/libcrypto.so" "," openssl "/lib/libssl.so"))) - #t))) - (add-after 'unpack 'disable-broken-tests - (lambda _ - ;; This test is broken as there is no keyboard interrupt. - (substitute* "tests/test_trust_list.py" - (("^(.*)class TrustListTests" line indent) - (string-append indent - "@unittest.skip(\"Disabled by Guix\")\n" - line))) - (substitute* "tests/test_tls.py" - (("^(.*)class TLSTests" line indent) - (string-append indent - "@unittest.skip(\"Disabled by Guix\")\n" - line))) - #t)) - (replace 'check - (lambda _ - (invoke "python" "run.py" "tests") - #t))))) - (home-page "https://github.com/wbond/oscrypto") - (synopsis "Compiler-free Python crypto library backed by the OS") - (description "oscrypto is a compilation-free, always up-to-date encryption library for Python.") - (license license:expat))) - -(define-public python-oscryptotests - (package (inherit python-oscrypto) - (name "python-oscryptotests") - (propagated-inputs - (list python-oscrypto)) - (arguments - `(#:tests? #f - #:phases - (modify-phases %standard-phases - (add-after 'unpack 'hard-code-path-to-libscrypt - (lambda* (#:key inputs #:allow-other-keys) - (chdir "tests") - #t))))))) - -(define-public python-certvalidator - (let ((commit "a145bf25eb75a9f014b3e7678826132efbba6213")) - (package - (name "python-certvalidator") - (version (git-version "0.1" "1" commit)) - (source - (origin - (method git-fetch) - (uri (git-reference - (url "https://github.com/achow101/certvalidator") - (commit commit))) - (file-name (git-file-name name commit)) - (sha256 - (base32 - "1qw2k7xis53179lpqdqyylbcmp76lj7sagp883wmxg5i7chhc96k")))) - (build-system python-build-system) - (propagated-inputs - (list python-asn1crypto - python-oscrypto - python-oscryptotests)) ;; certvalidator tests import oscryptotests - (arguments - `(#:phases - (modify-phases %standard-phases - (add-after 'unpack 'disable-broken-tests - (lambda _ - (substitute* "tests/test_certificate_validator.py" - (("^(.*)class CertificateValidatorTests" line indent) - (string-append indent - "@unittest.skip(\"Disabled by Guix\")\n" - line))) - (substitute* "tests/test_crl_client.py" - (("^(.*)def test_fetch_crl" line indent) - (string-append indent - "@unittest.skip(\"Disabled by Guix\")\n" - line))) - (substitute* "tests/test_ocsp_client.py" - (("^(.*)def test_fetch_ocsp" line indent) - (string-append indent - "@unittest.skip(\"Disabled by Guix\")\n" - line))) - (substitute* "tests/test_registry.py" - (("^(.*)def test_build_paths" line indent) - (string-append indent - "@unittest.skip(\"Disabled by Guix\")\n" - line))) - (substitute* "tests/test_validate.py" - (("^(.*)def test_revocation_mode_hard" line indent) - (string-append indent - "@unittest.skip(\"Disabled by Guix\")\n" - line))) - (substitute* "tests/test_validate.py" - (("^(.*)def test_revocation_mode_soft" line indent) - (string-append indent - "@unittest.skip(\"Disabled by Guix\")\n" - line))) - #t)) - (replace 'check - (lambda _ - (invoke "python" "run.py" "tests") - #t))))) - (home-page "https://github.com/wbond/certvalidator") - (synopsis "Python library for validating X.509 certificates and paths") - (description "certvalidator is a Python library for validating X.509 -certificates or paths. Supports various options, including: validation at a -specific moment in time, whitelisting and revocation checks.") - (license license:expat)))) - -(define-public python-signapple - (let ((commit "85bfcecc33d2773bc09bc318cec0614af2c8e287")) - (package - (name "python-signapple") - (version (git-version "0.2.0" "1" commit)) - (source - (origin - (method git-fetch) - (uri (git-reference - (url "https://github.com/achow101/signapple") - (commit commit))) - (file-name (git-file-name name commit)) - (sha256 - (base32 - "17yqjll8nw83q6dhgqhkl7w502z5vy9sln8m6mlx0f1c10isg8yg")))) - (build-system pyproject-build-system) - (propagated-inputs - (list python-asn1crypto - python-oscrypto - python-certvalidator - python-elfesteem)) - (native-inputs (list python-poetry-core)) - ;; There are no tests, but attempting to run python setup.py test leads to - ;; problems, just disable the test - (arguments '(#:tests? #f)) - (home-page "https://github.com/achow101/signapple") - (synopsis "Mach-O binary signature tool") - (description "signapple is a Python tool for creating, verifying, and -inspecting signatures in Mach-O binaries.") - (license license:expat)))) - -(define-public mingw-w64-base-gcc - (package - (inherit base-gcc) - (arguments - (substitute-keyword-arguments (package-arguments base-gcc) - ((#:configure-flags flags) - `(append ,flags - ;; https://gcc.gnu.org/install/configure.html - (list "--enable-threads=posix", - "--enable-default-ssp=yes", - "--disable-gcov", - building-on))))))) - -(define-public linux-base-gcc - (package - (inherit base-gcc) - (arguments - (substitute-keyword-arguments (package-arguments base-gcc) - ((#:configure-flags flags) - `(append ,flags - ;; https://gcc.gnu.org/install/configure.html - (list "--enable-initfini-array=yes", - "--enable-default-ssp=yes", - "--enable-default-pie=yes", - "--enable-standard-branch-protection=yes", - "--enable-cet=yes", - "--disable-gcov", - "--disable-libsanitizer", - building-on))) - ((#:phases phases) - `(modify-phases ,phases - ;; Given a XGCC package, return a modified package that replace each instance of - ;; -rpath in the default system spec that's inserted by Guix with -rpath-link - (add-after 'pre-configure 'replace-rpath-with-rpath-link - (lambda _ - (substitute* (cons "gcc/config/rs6000/sysv4.h" - (find-files "gcc/config" - "^gnu-user.*\\.h$")) - (("-rpath=") "-rpath-link=")) - #t)))))))) - -(define-public glibc-2.31 - (let ((commit "7b27c450c34563a28e634cccb399cd415e71ebfe")) - (package - (inherit glibc) ;; 2.39 - (version "2.31") - (source (origin - (method git-fetch) - (uri (git-reference - (url "https://sourceware.org/git/glibc.git") - (commit commit))) - (file-name (git-file-name "glibc" commit)) - (sha256 - (base32 - "017qdpr5id7ddb4lpkzj2li1abvw916m3fc6n7nw28z4h5qbv2n0")) - (patches (search-our-patches "glibc-guix-prefix.patch" - "glibc-riscv-jumptarget.patch")))) - (arguments - (substitute-keyword-arguments (package-arguments glibc) - ((#:configure-flags flags) - `(append ,flags - ;; https://www.gnu.org/software/libc/manual/html_node/Configuring-and-compiling.html - (list "--enable-stack-protector=all", - "--enable-cet", - "--enable-bind-now", - "--disable-werror", - "--disable-timezone-tools", - "--disable-profile", - building-on))) - ((#:phases phases) - `(modify-phases ,phases - (add-before 'configure 'set-etc-rpc-installation-directory - (lambda* (#:key outputs #:allow-other-keys) - ;; Install the rpc data base file under `$out/etc/rpc'. - ;; Otherwise build will fail with "Permission denied." - ;; Can be removed when we are building 2.32 or later. - (let ((out (assoc-ref outputs "out"))) - (substitute* "sunrpc/Makefile" - (("^\\$\\(inst_sysconfdir\\)/rpc(.*)$" _ suffix) - (string-append out "/etc/rpc" suffix "\n")) - (("^install-others =.*$") - (string-append "install-others = " out "/etc/rpc\n"))))))))))))) - -(define-public glibc-2.42 - (let ((commit "71874f167aa5bb1538ff7e394beaacee28ebe65f")) - (package - (inherit glibc) ;; 2.39 - (version "2.42") - (source (origin - (method git-fetch) - (uri (git-reference - (url "https://sourceware.org/git/glibc.git") - (commit commit))) - (file-name (git-file-name "glibc" commit)) - (sha256 - (base32 - "1pfbk907fkbavg7grbvb5zlhd3y47f8jj3d2v1s5w7xjnn0ypigq")) - (patches (search-our-patches "glibc-2.42-guix-prefix.patch")))) - (arguments - (substitute-keyword-arguments (package-arguments glibc) - ((#:configure-flags flags) - `(append ,flags - ;; https://www.gnu.org/software/libc/manual/html_node/Configuring-and-compiling.html - (list "--enable-stack-protector=all", - "--enable-bind-now", - "--enable-fortify-source", - "--enable-cet=yes", - "--enable-nscd=no", - "--enable-static-nss=yes", - "--disable-timezone-tools", - "--disable-profile", - "--disable-werror", - building-on)))))))) - -;; The sponge tool from moreutils. -(define-public sponge - (package - (name "sponge") - (version "0.69") - (source (origin - (method url-fetch) - (uri (string-append - "https://git.joeyh.name/index.cgi/moreutils.git/snapshot/ - moreutils-" version ".tar.gz")) - (file-name (string-append "moreutils-" version ".tar.gz")) - (sha256 - (base32 - "1l859qnzccslvxlh5ghn863bkq2vgmqgnik6jr21b9kc6ljmsy8g")))) - (build-system gnu-build-system) - (arguments - (list #:phases - #~(modify-phases %standard-phases - (delete 'configure) - (replace 'install - (lambda* (#:key outputs #:allow-other-keys) - (let ((bin (string-append (assoc-ref outputs "out") "/bin"))) - (install-file "sponge" bin))))) - #:make-flags - #~(list "sponge" (string-append "CC=" #$(cc-for-target))))) - (home-page "https://joeyh.name/code/moreutils/") - (synopsis "Miscellaneous general-purpose command-line tools") - (description "Just sponge") - (license license:gpl2+))) - -(packages->manifest - (append - (list ;; The Basics - bash-minimal - which - coreutils-minimal - ;; File(system) inspection - grep - diffutils - findutils - ;; File transformation - patch - gawk - sed - sponge - ;; Compression and archiving - tar - gzip - xz - ;; Build tools - gcc-toolchain-13 - cmake-minimal - gnu-make - ninja - ;; Scripting - python-minimal ;; (3.10) - ;; Git - git-minimal - ;; Tests - python-lief) - (let ((target (getenv "HOST"))) - (cond ((string-suffix? "-mingw32" target) - (list zip - (make-mingw-pthreads-cross-toolchain "x86_64-w64-mingw32") - nsis-x86_64 - nss-certs - osslsigncode)) - ((string-contains target "x86_64-linux-") - (list (list gcc-toolchain-13 "static") - (make-bitcoin-cross-toolchain target - #:base-libc glibc-2.42))) - ((string-contains target "-linux-") - (list bison - pkg-config - (list gcc-toolchain-13 "static") - (make-bitcoin-cross-toolchain target))) - ((string-contains target "darwin") - (list clang-toolchain-19 - lld-19 - (make-lld-wrapper lld-19 #:lld-as-ld? #t) - python-signapple - zip)) - (else '()))))) diff --git a/bench-ci/guix/patches/binutils-unaligned-default.patch b/bench-ci/guix/patches/binutils-unaligned-default.patch deleted file mode 100644 index d1bc71aee142..000000000000 --- a/bench-ci/guix/patches/binutils-unaligned-default.patch +++ /dev/null @@ -1,22 +0,0 @@ -commit 6537181f59ed186a341db621812a6bc35e22eaf6 -Author: fanquake -Date: Wed Apr 10 12:15:52 2024 +0200 - - build: turn on -muse-unaligned-vector-move by default - - This allows us to avoid (more invasively) patching GCC, to avoid - unaligned instruction use. - -diff --git a/gas/config/tc-i386.c b/gas/config/tc-i386.c -index e0632681477..14a9653abdf 100644 ---- a/gas/config/tc-i386.c -+++ b/gas/config/tc-i386.c -@@ -801,7 +801,7 @@ static unsigned int no_cond_jump_promotion = 0; - static unsigned int sse2avx; - - /* Encode aligned vector move as unaligned vector move. */ --static unsigned int use_unaligned_vector_move; -+static unsigned int use_unaligned_vector_move = 1; - - /* Encode scalar AVX instructions with specific vector length. */ - static enum diff --git a/bench-ci/guix/patches/gcc-remap-guix-store.patch b/bench-ci/guix/patches/gcc-remap-guix-store.patch deleted file mode 100644 index a8b41d485b04..000000000000 --- a/bench-ci/guix/patches/gcc-remap-guix-store.patch +++ /dev/null @@ -1,20 +0,0 @@ -Without ffile-prefix-map, the debug symbols will contain paths for the -guix store which will include the hashes of each package. However, the -hash for the same package will differ when on different architectures. -In order to be reproducible regardless of the architecture used to build -the package, map all guix store prefixes to something fixed, e.g. /usr. - ---- a/libgcc/Makefile.in -+++ b/libgcc/Makefile.in -@@ -854,7 +854,7 @@ endif - # libgcc_eh.a, only LIB2ADDEH matters. If we do, only LIB2ADDEHSTATIC and - # LIB2ADDEHSHARED matter. (Usually all three are identical.) - --c_flags := -fexceptions -+c_flags := -fexceptions $(shell find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;) - - ifeq ($(enable_shared),yes) - --- -2.37.0 - diff --git a/bench-ci/guix/patches/glibc-2.42-guix-prefix.patch b/bench-ci/guix/patches/glibc-2.42-guix-prefix.patch deleted file mode 100644 index f2fc1b90f183..000000000000 --- a/bench-ci/guix/patches/glibc-2.42-guix-prefix.patch +++ /dev/null @@ -1,47 +0,0 @@ -Without -ffile-prefix-map, the debug symbols will contain paths for the -guix store which will include the hashes of each package. However, the -hash for the same package will differ when on different architectures. -In order to be reproducible regardless of the architecture used to build -the package, map all guix store prefixes to something fixed, e.g. /usr. - ---- a/Makeconfig -+++ b/Makeconfig -@@ -1074,6 +1074,10 @@ CPPFLAGS-.o = $(pic-default) - CFLAGS-.o = $(filter %frame-pointer,$(+cflags)) $(pie-default) - CFLAGS-.o += $(call elide-fortify-source,.o,$(routines_no_fortify)) - CFLAGS-.o += $(call elide-fortify-source,_chk.o,$(routines_no_fortify)) -+ -+# Map Guix store paths to /usr -+CFLAGS-.o += `find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;` -+ - libtype.o := lib%.a - object-suffixes += .o - ifeq (yes,$(build-shared)) -diff --git a/iconv/Makefile b/iconv/Makefile -index afb3fb7bdb..5acee345e0 100644 ---- a/iconv/Makefile -+++ b/iconv/Makefile -@@ -65,6 +65,9 @@ CFLAGS-gconv_cache.c += -DGCONV_DIR='"$(gconvdir)"' - CFLAGS-gconv_conf.c += -DGCONV_PATH='"$(gconvdir)"' - CFLAGS-iconvconfig.c += -DGCONV_PATH='"$(gconvdir)"' -DGCONV_DIR='"$(gconvdir)"' - -+# Map Guix store paths to /usr -+CFLAGS-.c += `find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;` -+ - # Set libof-* for each routine. - cpp-srcs-left := $(iconv_prog-modules) $(iconvconfig-modules) - lib := iconvprogs -diff --git a/posix/Makefile b/posix/Makefile -index 3d368b91f6..d79d8fb648 100644 ---- a/posix/Makefile -+++ b/posix/Makefile -@@ -590,6 +590,9 @@ CFLAGS-execlp.os = -fomit-frame-pointer - CFLAGS-nanosleep.c += -fexceptions -fasynchronous-unwind-tables - CFLAGS-fork.c = $(libio-mtsafe) $(config-cflags-wno-ignored-attributes) - -+# Map Guix store paths to /usr -+CFLAGS-.c += `find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;` -+ - tstgetopt-ARGS = -a -b -cfoobar --required foobar --optional=bazbug \ - --none random --col --color --colour - diff --git a/bench-ci/guix/patches/glibc-guix-prefix.patch b/bench-ci/guix/patches/glibc-guix-prefix.patch deleted file mode 100644 index 60e12ca52546..000000000000 --- a/bench-ci/guix/patches/glibc-guix-prefix.patch +++ /dev/null @@ -1,16 +0,0 @@ -Without ffile-prefix-map, the debug symbols will contain paths for the -guix store which will include the hashes of each package. However, the -hash for the same package will differ when on different architectures. -In order to be reproducible regardless of the architecture used to build -the package, map all guix store prefixes to something fixed, e.g. /usr. - ---- a/Makeconfig -+++ b/Makeconfig -@@ -1007,6 +1007,7 @@ object-suffixes := - CPPFLAGS-.o = $(pic-default) - # libc.a must be compiled with -fPIE/-fpie for static PIE. - CFLAGS-.o = $(filter %frame-pointer,$(+cflags)) $(pie-default) -+CFLAGS-.o += `find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;` - libtype.o := lib%.a - object-suffixes += .o - ifeq (yes,$(build-shared)) diff --git a/bench-ci/guix/patches/glibc-riscv-jumptarget.patch b/bench-ci/guix/patches/glibc-riscv-jumptarget.patch deleted file mode 100644 index 702959433d86..000000000000 --- a/bench-ci/guix/patches/glibc-riscv-jumptarget.patch +++ /dev/null @@ -1,57 +0,0 @@ -commit 68389203832ab39dd0dbaabbc4059e7fff51c29b -Author: Fangrui Song -Date: Thu Oct 28 11:39:49 2021 -0700 - - riscv: Fix incorrect jal with HIDDEN_JUMPTARGET - - A non-local STV_DEFAULT defined symbol is by default preemptible in a - shared object. j/jal cannot target a preemptible symbol. On other - architectures, such a jump instruction either causes PLT [BZ #18822], or - if short-ranged, sometimes rejected by the linker (but not by GNU ld's - riscv port [ld PR/28509]). - - Use HIDDEN_JUMPTARGET to target a non-preemptible symbol instead. - - With this patch, ld.so and libc.so can be linked with LLD if source - files are compiled/assembled with -mno-relax/-Wa,-mno-relax. - - Acked-by: Palmer Dabbelt - Reviewed-by: Adhemerval Zanella - -Can be dropped when we are using glibc 2.35 or later. - -diff --git a/sysdeps/riscv/setjmp.S b/sysdeps/riscv/setjmp.S -index 0b92016b31..bec7ff80f4 100644 ---- a/sysdeps/riscv/setjmp.S -+++ b/sysdeps/riscv/setjmp.S -@@ -21,7 +21,7 @@ - - ENTRY (_setjmp) - li a1, 0 -- j __sigsetjmp -+ j HIDDEN_JUMPTARGET (__sigsetjmp) - END (_setjmp) - ENTRY (setjmp) - li a1, 1 -diff --git a/sysdeps/unix/sysv/linux/riscv/setcontext.S b/sysdeps/unix/sysv/linux/riscv/setcontext.S -index 9510518750..e44a68aad4 100644 ---- a/sysdeps/unix/sysv/linux/riscv/setcontext.S -+++ b/sysdeps/unix/sysv/linux/riscv/setcontext.S -@@ -95,6 +95,7 @@ LEAF (__setcontext) - 99: j __syscall_error - - END (__setcontext) -+libc_hidden_def (__setcontext) - weak_alias (__setcontext, setcontext) - - LEAF (__start_context) -@@ -108,7 +109,7 @@ LEAF (__start_context) - /* Invoke subsequent context if present, else exit(0). */ - mv a0, s2 - beqz s2, 1f -- jal __setcontext --1: j exit -+ jal HIDDEN_JUMPTARGET (__setcontext) -+1: j HIDDEN_JUMPTARGET (exit) - - END (__start_context) diff --git a/bench-ci/guix/patches/lief-scikit-0-9.patch b/bench-ci/guix/patches/lief-scikit-0-9.patch deleted file mode 100644 index 71e617834f07..000000000000 --- a/bench-ci/guix/patches/lief-scikit-0-9.patch +++ /dev/null @@ -1,21 +0,0 @@ -Partially revert f23ced2f4ffc170d0a6f40ff4a1bee575e3447cf - -Restore compat with python-scikit-build-core 0.9.x -Can be dropped when using python-scikit-build-core >= 0.10.x - ---- a/api/python/backend/setup.py -+++ b/api/python/backend/setup.py -@@ -101,12 +101,12 @@ def _get_hooked_config(is_editable: bool) -> Optional[dict[str, Union[str, List[ - config_settings = { - "logging.level": "DEBUG", - "build-dir": config.build_dir, -- "build.targets": config.build.targets, - "install.strip": config.strip, - "backport.find-python": "0", - "wheel.py-api": config.build.py_api, - "cmake.source-dir": SRC_DIR.as_posix(), - "cmake.build-type": config.build.build_type, -+ "cmake.targets": config.build.targets, - "cmake.args": [ - *config.cmake_generator, - *config.get_cmake_args(is_editable), diff --git a/bench-ci/guix/patches/oscrypto-hard-code-openssl.patch b/bench-ci/guix/patches/oscrypto-hard-code-openssl.patch deleted file mode 100644 index 32027f2d09af..000000000000 --- a/bench-ci/guix/patches/oscrypto-hard-code-openssl.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/oscrypto/__init__.py b/oscrypto/__init__.py -index eb27313..371ab24 100644 ---- a/oscrypto/__init__.py -+++ b/oscrypto/__init__.py -@@ -302,3 +302,8 @@ def load_order(): - 'oscrypto._win.tls', - 'oscrypto.tls', - ] -+ -+ -+paths = '@GUIX_OSCRYPTO_USE_OPENSSL@'.split(',') -+assert len(paths) == 2, 'Value for OSCRYPTO_USE_OPENSSL env var must be two paths separated by a comma' -+use_openssl(*paths) diff --git a/bench-ci/guix/patches/winpthreads-remap-guix-store.patch b/bench-ci/guix/patches/winpthreads-remap-guix-store.patch deleted file mode 100644 index e1f1a6eba531..000000000000 --- a/bench-ci/guix/patches/winpthreads-remap-guix-store.patch +++ /dev/null @@ -1,17 +0,0 @@ -Without ffile-prefix-map, the debug symbols will contain paths for the -guix store which will include the hashes of each package. However, the -hash for the same package will differ when on different architectures. -In order to be reproducible regardless of the architecture used to build -the package, map all guix store prefixes to something fixed, e.g. /usr. - ---- a/mingw-w64-libraries/winpthreads/Makefile.in -+++ b/mingw-w64-libraries/winpthreads/Makefile.in -@@ -478,7 +478,7 @@ top_build_prefix = @top_build_prefix@ - top_builddir = @top_builddir@ - top_srcdir = @top_srcdir@ - SUBDIRS = . tests --AM_CFLAGS = -Wall -DWIN32_LEAN_AND_MEAN $(am__append_1) -+AM_CFLAGS = -Wall -DWIN32_LEAN_AND_MEAN $(am__append_1) $(shell find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;) - ACLOCAL_AMFLAGS = -I m4 - lib_LTLIBRARIES = libwinpthread.la - include_HEADERS = include/pthread.h include/sched.h include/semaphore.h include/pthread_unistd.h include/pthread_time.h include/pthread_compat.h include/pthread_signal.h diff --git a/bench-ci/guix/security-check.py b/bench-ci/guix/security-check.py deleted file mode 100755 index ac943e33aabd..000000000000 --- a/bench-ci/guix/security-check.py +++ /dev/null @@ -1,297 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2015-2022 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -''' -Perform basic security checks on a series of executables. -Exit status will be 0 if successful, and the program will be silent. -Otherwise the exit status will be 1 and it will log which executables failed which checks. - -Example usage: - - find ../path/to/guix/binaries -type f -executable | xargs python3 contrib/guix/security-check.py -''' -import re -import sys - -import lief - -def check_ELF_RELRO(binary) -> bool: - ''' - Check for read-only relocations. - GNU_RELRO program header must exist - Dynamic section must have BIND_NOW flag - ''' - have_gnu_relro = False - for segment in binary.segments: - # Note: not checking p_flags == PF_R: here as linkers set the permission differently - # This does not affect security: the permission flags of the GNU_RELRO program - # header are ignored, the PT_LOAD header determines the effective permissions. - # However, the dynamic linker need to write to this area so these are RW. - # Glibc itself takes care of mprotecting this area R after relocations are finished. - # See also https://marc.info/?l=binutils&m=1498883354122353 - if segment.type == lief.ELF.Segment.TYPE.GNU_RELRO: - have_gnu_relro = True - - have_bindnow = False - try: - flags = binary.get(lief.ELF.DynamicEntry.TAG.FLAGS) - if flags.has(lief.ELF.DynamicEntryFlags.FLAG.BIND_NOW): - have_bindnow = True - except Exception: - have_bindnow = False - - return have_gnu_relro and have_bindnow - -def check_ELF_CANARY(binary) -> bool: - ''' - Check for use of stack canary - ''' - return binary.has_symbol('__stack_chk_fail') - -def check_ELF_SEPARATE_CODE(binary): - ''' - Check that sections are appropriately separated in virtual memory, - based on their permissions. This checks for missing -Wl,-z,separate-code - and potentially other problems. - ''' - R = lief.ELF.Segment.FLAGS.R - W = lief.ELF.Segment.FLAGS.W - E = lief.ELF.Segment.FLAGS.X - EXPECTED_FLAGS = { - # Read + execute - '.init': R | E, - '.plt': R | E, - '.plt.got': R | E, - '.plt.sec': R | E, - '.text': R | E, - '.fini': R | E, - # Read-only data - '.interp': R, - '.note.gnu.property': R, - '.note.gnu.build-id': R, - '.note.ABI-tag': R, - '.gnu.hash': R, - '.dynsym': R, - '.dynstr': R, - '.gnu.version': R, - '.gnu.version_r': R, - '.rela.dyn': R, - '.rela.plt': R, - '.rodata': R, - '.eh_frame_hdr': R, - '.eh_frame': R, - '.qtmetadata': R, - '.gcc_except_table': R, - '.stapsdt.base': R, - # Writable data - '.init_array': R | W, - '.fini_array': R | W, - '.dynamic': R | W, - '.got': R | W, - '.data': R | W, - '.bss': R | W, - } - if binary.header.machine_type == lief.ELF.ARCH.PPC64: - # .plt is RW on ppc64 even with separate-code - EXPECTED_FLAGS['.plt'] = R | W - # For all LOAD program headers get mapping to the list of sections, - # and for each section, remember the flags of the associated program header. - flags_per_section = {} - for segment in binary.segments: - if segment.type == lief.ELF.Segment.TYPE.LOAD: - for section in segment.sections: - flags_per_section[section.name] = segment.flags - # Spot-check ELF LOAD program header flags per section - # If these sections exist, check them against the expected R/W/E flags - for (section, flags) in flags_per_section.items(): - if section in EXPECTED_FLAGS: - if int(EXPECTED_FLAGS[section]) != int(flags): - return False - return True - -def check_ELF_CONTROL_FLOW(binary) -> bool: - ''' - Check for control flow instrumentation - ''' - main = binary.get_function_address('main') - content = binary.get_content_from_virtual_address(main, 4, lief.Binary.VA_TYPES.AUTO) - - if content.tolist() == [243, 15, 30, 250]: # endbr64 - return True - return False - -def check_ELF_FORTIFY(binary) -> bool: - # no imported fortified funcs if we are fully static - # check could be changed to include all symbols - if binary.header.machine_type == lief.ELF.ARCH.X86_64: - return True - - # bitcoin wrapper does not currently contain any fortified functions - if '--monolithic' in binary.strings: - return True - - chk_funcs = set() - - for sym in binary.imported_symbols: - match = re.search(r'__[a-z]*_chk', sym.name) - if match: - chk_funcs.add(match.group(0)) - - # ignore stack-protector - chk_funcs.discard('__stack_chk') - - return len(chk_funcs) >= 1 - -def check_PE_DYNAMIC_BASE(binary) -> bool: - '''PIE: DllCharacteristics bit 0x40 signifies dynamicbase (ASLR)''' - return lief.PE.OptionalHeader.DLL_CHARACTERISTICS.DYNAMIC_BASE in binary.optional_header.dll_characteristics_lists - -# Must support high-entropy 64-bit address space layout randomization -# in addition to DYNAMIC_BASE to have secure ASLR. -def check_PE_HIGH_ENTROPY_VA(binary) -> bool: - '''PIE: DllCharacteristics bit 0x20 signifies high-entropy ASLR''' - return lief.PE.OptionalHeader.DLL_CHARACTERISTICS.HIGH_ENTROPY_VA in binary.optional_header.dll_characteristics_lists - -def check_PE_RELOC_SECTION(binary) -> bool: - '''Check for a reloc section. This is required for functional ASLR.''' - return binary.has_relocations - -def check_PE_CONTROL_FLOW(binary) -> bool: - ''' - Check for control flow instrumentation - ''' - main = binary.get_symbol('main').value - - section_addr = binary.section_from_rva(main).virtual_address - virtual_address = binary.optional_header.imagebase + section_addr + main - - content = binary.get_content_from_virtual_address(virtual_address, 4, lief.Binary.VA_TYPES.VA) - - if content.tolist() == [243, 15, 30, 250]: # endbr64 - return True - return False - -def check_PE_CANARY(binary) -> bool: - ''' - Check for use of stack canary - ''' - return binary.has_symbol('__stack_chk_fail') - -def check_MACHO_NOUNDEFS(binary) -> bool: - ''' - Check for no undefined references. - ''' - return binary.header.has(lief.MachO.Header.FLAGS.NOUNDEFS) - -def check_MACHO_FIXUP_CHAINS(binary) -> bool: - ''' - Check for use of chained fixups. - ''' - return binary.has_dyld_chained_fixups - -def check_MACHO_CANARY(binary) -> bool: - ''' - Check for use of stack canary - ''' - return binary.has_symbol('___stack_chk_fail') - -def check_PIE(binary) -> bool: - ''' - Check for position independent executable (PIE), - allowing for address space randomization. - ''' - return binary.is_pie - -def check_NX(binary) -> bool: - ''' - Check for no stack execution - ''' - - # binary.has_nx checks are only for the stack, but MachO binaries might - # have executable heaps. - if binary.format == lief.Binary.FORMATS.MACHO: - return binary.concrete.has_nx_stack and binary.concrete.has_nx_heap - else: - return binary.has_nx - -def check_MACHO_CONTROL_FLOW(binary) -> bool: - ''' - Check for control flow instrumentation - ''' - content = binary.get_content_from_virtual_address(binary.entrypoint, 4, lief.Binary.VA_TYPES.AUTO) - - if content.tolist() == [243, 15, 30, 250]: # endbr64 - return True - return False - -def check_MACHO_BRANCH_PROTECTION(binary) -> bool: - ''' - Check for branch protection instrumentation - ''' - content = binary.get_content_from_virtual_address(binary.entrypoint, 4, lief.Binary.VA_TYPES.AUTO) - - if content.tolist() == [95, 36, 3, 213]: # bti - return True - return False - -BASE_ELF = [ - ('FORTIFY', check_ELF_FORTIFY), - ('PIE', check_PIE), - ('NX', check_NX), - ('RELRO', check_ELF_RELRO), - ('CANARY', check_ELF_CANARY), - ('SEPARATE_CODE', check_ELF_SEPARATE_CODE), -] - -BASE_PE = [ - ('PIE', check_PIE), - ('DYNAMIC_BASE', check_PE_DYNAMIC_BASE), - ('HIGH_ENTROPY_VA', check_PE_HIGH_ENTROPY_VA), - ('NX', check_NX), - ('RELOC_SECTION', check_PE_RELOC_SECTION), - ('CONTROL_FLOW', check_PE_CONTROL_FLOW), - ('CANARY', check_PE_CANARY), -] - -BASE_MACHO = [ - ('NOUNDEFS', check_MACHO_NOUNDEFS), - ('CANARY', check_MACHO_CANARY), - ('FIXUP_CHAINS', check_MACHO_FIXUP_CHAINS), -] - -CHECKS = { - lief.Binary.FORMATS.ELF: { - lief.Header.ARCHITECTURES.X86_64: BASE_ELF + [('CONTROL_FLOW', check_ELF_CONTROL_FLOW)], - lief.Header.ARCHITECTURES.ARM: BASE_ELF, - lief.Header.ARCHITECTURES.ARM64: BASE_ELF, - lief.Header.ARCHITECTURES.PPC64: BASE_ELF, - lief.Header.ARCHITECTURES.RISCV: BASE_ELF, - }, - lief.Binary.FORMATS.PE: { - lief.Header.ARCHITECTURES.X86_64: BASE_PE, - }, - lief.Binary.FORMATS.MACHO: { - lief.Header.ARCHITECTURES.X86_64: BASE_MACHO + [('PIE', check_PIE), - ('NX', check_NX), - ('CONTROL_FLOW', check_MACHO_CONTROL_FLOW)], - lief.Header.ARCHITECTURES.ARM64: BASE_MACHO + [('BRANCH_PROTECTION', check_MACHO_BRANCH_PROTECTION)], - } -} - -if __name__ == '__main__': - retval: int = 0 - for filename in sys.argv[1:]: - binary = lief.parse(filename) - - etype = binary.format - arch = binary.abstract.header.architecture - - failed: list[str] = [] - for (name, func) in CHECKS[etype][arch]: - if not func(binary): - failed.append(name) - if failed: - print(f'{filename}: failed {" ".join(failed)}') - retval = 1 - sys.exit(retval) diff --git a/bench-ci/guix/symbol-check.py b/bench-ci/guix/symbol-check.py deleted file mode 100755 index 3d7a654c8589..000000000000 --- a/bench-ci/guix/symbol-check.py +++ /dev/null @@ -1,338 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2014 Wladimir J. van der Laan -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -''' -A script to check that release executables only contain certain symbols -and are only linked against allowed libraries. - -Example usage: - - find ../path/to/guix/binaries -type f -executable | xargs python3 contrib/guix/symbol-check.py -''' -import sys - -import lief - -# Debian 11 (Bullseye) EOL: 2026. https://wiki.debian.org/LTS -# -# - libgcc version 10.2.1 (https://packages.debian.org/bullseye/libgcc-s1) -# - libc version 2.31 (https://packages.debian.org/source/bullseye/glibc) -# -# Ubuntu 20.04 (Focal) EOL: 2030. https://wiki.ubuntu.com/ReleaseTeam -# -# - libgcc version 10.5.0 (https://packages.ubuntu.com/focal/libgcc1) -# - libc version 2.31 (https://packages.ubuntu.com/focal/libc6) -# -# CentOS Stream 9 EOL: 2027. https://www.centos.org/cl-vs-cs/#end-of-life -# -# - libgcc version 12.2.1 (https://mirror.stream.centos.org/9-stream/AppStream/x86_64/os/Packages/) -# - libc version 2.34 (https://mirror.stream.centos.org/9-stream/AppStream/x86_64/os/Packages/) -# -# See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html for more info. - -MAX_VERSIONS = { -'GCC': (7,0,0), -'GLIBC': { - lief.ELF.ARCH.X86_64: (0,0), - lief.ELF.ARCH.ARM: (2,31), - lief.ELF.ARCH.AARCH64:(2,31), - lief.ELF.ARCH.PPC64: (2,31), - lief.ELF.ARCH.RISCV: (2,31), -}, -'LIBATOMIC': (1,0), -'V': (0,5,0), # xkb (bitcoin-qt only) -} - -# Ignore symbols that are exported as part of every executable -IGNORE_EXPORTS = { -'environ', '_environ', '__environ', '_fini', '_init', 'stdin', -'stdout', 'stderr', '__libc_single_threaded', -} - -# Expected linker-loader names can be found here: -# https://sourceware.org/glibc/wiki/ABIList?action=recall&rev=16 -ELF_INTERPRETER_NAMES: dict[lief.ELF.ARCH, dict[lief.Header.ENDIANNESS, str]] = { - lief.ELF.ARCH.X86_64: { - lief.Header.ENDIANNESS.LITTLE: "", - }, - lief.ELF.ARCH.ARM: { - lief.Header.ENDIANNESS.LITTLE: "/lib/ld-linux-armhf.so.3", - }, - lief.ELF.ARCH.AARCH64: { - lief.Header.ENDIANNESS.LITTLE: "/lib/ld-linux-aarch64.so.1", - }, - lief.ELF.ARCH.PPC64: { - lief.Header.ENDIANNESS.BIG: "/lib64/ld64.so.1", - lief.Header.ENDIANNESS.LITTLE: "/lib64/ld64.so.2", - }, - lief.ELF.ARCH.RISCV: { - lief.Header.ENDIANNESS.LITTLE: "/lib/ld-linux-riscv64-lp64d.so.1", - }, -} - -ELF_ABIS: dict[lief.ELF.ARCH, dict[lief.Header.ENDIANNESS, list[int]]] = { - lief.ELF.ARCH.X86_64: { - lief.Header.ENDIANNESS.LITTLE: [3,2,0], - }, - lief.ELF.ARCH.ARM: { - lief.Header.ENDIANNESS.LITTLE: [3,2,0], - }, - lief.ELF.ARCH.AARCH64: { - lief.Header.ENDIANNESS.LITTLE: [3,7,0], - }, - lief.ELF.ARCH.PPC64: { - lief.Header.ENDIANNESS.LITTLE: [3,10,0], - lief.Header.ENDIANNESS.BIG: [3,2,0], - }, - lief.ELF.ARCH.RISCV: { - lief.Header.ENDIANNESS.LITTLE: [4,15,0], - }, -} - -# Allowed NEEDED libraries -ELF_ALLOWED_LIBRARIES = { -# bitcoind and bitcoin-qt -'libgcc_s.so.1', # GCC base support -'libc.so.6', # C library -'libpthread.so.0', # threading -'libm.so.6', # math library -'libatomic.so.1', -'ld-linux.so.2', # 32-bit dynamic linker -'ld-linux-aarch64.so.1', # 64-bit ARM dynamic linker -'ld-linux-armhf.so.3', # 32-bit ARM dynamic linker -'ld64.so.1', # POWER64 ABIv1 dynamic linker -'ld64.so.2', # POWER64 ABIv2 dynamic linker -'ld-linux-riscv64-lp64d.so.1', # 64-bit RISC-V dynamic linker -# bitcoin-qt only -'libxcb.so.1', # part of X11 -'libxkbcommon.so.0', # keyboard keymapping -'libxkbcommon-x11.so.0', # keyboard keymapping -'libfontconfig.so.1', # font support -'libfreetype.so.6', # font parsing -'libdl.so.2', # programming interface to dynamic linker -'libxcb-icccm.so.4', -'libxcb-image.so.0', -'libxcb-shm.so.0', -'libxcb-keysyms.so.1', -'libxcb-randr.so.0', -'libxcb-render-util.so.0', -'libxcb-render.so.0', -'libxcb-shape.so.0', -'libxcb-sync.so.1', -'libxcb-xfixes.so.0', -'libxcb-xkb.so.1', -} - -MACHO_ALLOWED_LIBRARIES = { -# bitcoind and bitcoin-qt -'libc++.1.dylib', # C++ Standard Library -'libSystem.B.dylib', # libc, libm, libpthread, libinfo -# bitcoin-qt only -'AppKit', # user interface -'ApplicationServices', # common application tasks. -'Carbon', # deprecated c back-compat API -'ColorSync', -'CoreFoundation', # low level func, data types -'CoreGraphics', # 2D rendering -'CoreServices', # operating system services -'CoreText', # interface for laying out text and handling fonts. -'CoreVideo', # video processing -'Foundation', # base layer functionality for apps/frameworks -'ImageIO', # read and write image file formats. -'IOKit', # user-space access to hardware devices and drivers. -'IOSurface', # cross process image/drawing buffers -'libobjc.A.dylib', # Objective-C runtime library -'Metal', # 3D graphics -'QuartzCore', # animation -'Security', # access control and authentication -'UniformTypeIdentifiers', # collection of types that map to MIME and file types -} - -PE_ALLOWED_LIBRARIES = { -'ADVAPI32.dll', # legacy security & registry -'bcrypt.dll', # newer security and identity API -'IPHLPAPI.DLL', # IP helper API -'KERNEL32.dll', # win32 base APIs -'msvcrt.dll', # C standard library for MSVC -'SHELL32.dll', # shell API -'WS2_32.dll', # sockets -# bitcoin-qt only -'api-ms-win-core-synch-l1-2-0.dll', # Synchronization Primitives API -'api-ms-win-core-winrt-l1-1-0.dll', # Windows Runtime API -'api-ms-win-core-winrt-string-l1-1-0.dll', # WinRT String API -'AUTHZ.dll', # Windows Authorization Framework -'comdlg32.dll', # Common Dialog Box Library -'d3d11.dll', # Direct3D 11 API -'d3d12.dll', # Direct3D 12 API -'d3d9.dll', # Direct3D 9 API -'dwmapi.dll', # desktop window manager -'DWrite.dll', # DirectX Typography Services -'dxgi.dll', # DirectX Graphics Infrastructure -'GDI32.dll', # graphics device interface -'IMM32.dll', # input method editor -'NETAPI32.dll', # network management -'ole32.dll', # component object model -'OLEAUT32.dll', # OLE Automation API -'SHLWAPI.dll', # light weight shell API -'USER32.dll', # user interface -'USERENV.dll', # user management -'UxTheme.dll', # visual style -'VERSION.dll', # version checking -'WINMM.dll', # WinMM audio API -'WTSAPI32.dll', # Remote Desktop -'SETUPAPI.dll', # Windows Setup API -'SHCORE.dll', # Stream Handler Core -} - -def check_version(max_versions, version, arch) -> bool: - (lib, _, ver) = version.rpartition('_') - ver = tuple([int(x) for x in ver.split('.')]) - if not lib in max_versions: - return False - if isinstance(max_versions[lib], tuple): - return ver <= max_versions[lib] - else: - return ver <= max_versions[lib][arch] - -def check_imported_symbols(binary) -> bool: - ok: bool = True - - for symbol in binary.imported_symbols: - if not symbol.imported: - continue - - version = symbol.symbol_version if symbol.has_version else None - - if version: - aux_version = version.symbol_version_auxiliary.name if version.has_auxiliary_version else None - if aux_version and not check_version(MAX_VERSIONS, aux_version, binary.header.machine_type): - print(f'{filename}: symbol {symbol.name} from unsupported version {version}') - ok = False - return ok - -def check_exported_symbols(binary) -> bool: - ok: bool = True - - for symbol in binary.dynamic_symbols: - if not symbol.exported: - continue - name = symbol.name - if binary.header.machine_type == lief.ELF.ARCH.RISCV or name in IGNORE_EXPORTS: - continue - print(f'{filename}: export of symbol {name} not allowed!') - ok = False - return ok - -def check_RUNPATH(binary) -> bool: - assert binary.get(lief.ELF.DynamicEntry.TAG.RUNPATH) is None - assert binary.get(lief.ELF.DynamicEntry.TAG.RPATH) is None - return True - -def check_ELF_libraries(binary) -> bool: - ok: bool = True - - if binary.header.machine_type == lief.ELF.ARCH.X86_64: - return len(binary.libraries) == 0 - - for library in binary.libraries: - if library not in ELF_ALLOWED_LIBRARIES: - print(f'{filename}: {library} is not in ALLOWED_LIBRARIES!') - ok = False - return ok - -def check_MACHO_libraries(binary) -> bool: - ok: bool = True - for dylib in binary.libraries: - split = dylib.name.split('/') - if split[-1] not in MACHO_ALLOWED_LIBRARIES: - print(f'{split[-1]} is not in ALLOWED_LIBRARIES!') - ok = False - return ok - -def check_MACHO_min_os(binary) -> bool: - if binary.build_version.minos == [14,0,0]: - return True - return False - -def check_MACHO_sdk(binary) -> bool: - if binary.build_version.sdk == [14, 0, 0]: - return True - return False - -def check_MACHO_lld(binary) -> bool: - if binary.build_version.tools[0].version == [19, 1, 4]: - return True - return False - -def check_PE_libraries(binary) -> bool: - ok: bool = True - for dylib in binary.libraries: - if dylib not in PE_ALLOWED_LIBRARIES: - print(f'{dylib} is not in ALLOWED_LIBRARIES!') - ok = False - return ok - -def check_PE_subsystem_version(binary) -> bool: - major: int = binary.optional_header.major_subsystem_version - minor: int = binary.optional_header.minor_subsystem_version - if major == 6 and minor == 2: - return True - return False - -def check_PE_application_manifest(binary) -> bool: - if not binary.has_resources: - # No resources at all. - return False - - rm = binary.resources_manager - return rm.has_manifest - -def check_ELF_interpreter(binary) -> bool: - expected_interpreter = ELF_INTERPRETER_NAMES[binary.header.machine_type][binary.abstract.header.endianness] - - return binary.concrete.interpreter == expected_interpreter - -def check_ELF_ABI(binary) -> bool: - expected_abi = ELF_ABIS[binary.header.machine_type][binary.abstract.header.endianness] - note = binary.concrete.get(lief.ELF.Note.TYPE.GNU_ABI_TAG) - assert note.abi == lief.ELF.NoteAbi.ABI.LINUX - return note.version == expected_abi - -CHECKS = { -lief.Binary.FORMATS.ELF: [ - ('IMPORTED_SYMBOLS', check_imported_symbols), - ('EXPORTED_SYMBOLS', check_exported_symbols), - ('LIBRARY_DEPENDENCIES', check_ELF_libraries), - ('INTERPRETER_NAME', check_ELF_interpreter), - ('ABI', check_ELF_ABI), - ('RUNPATH', check_RUNPATH), -], -lief.Binary.FORMATS.MACHO: [ - ('DYNAMIC_LIBRARIES', check_MACHO_libraries), - ('MIN_OS', check_MACHO_min_os), - ('SDK', check_MACHO_sdk), - ('LLD', check_MACHO_lld), -], -lief.Binary.FORMATS.PE: [ - ('DYNAMIC_LIBRARIES', check_PE_libraries), - ('SUBSYSTEM_VERSION', check_PE_subsystem_version), - ('APPLICATION_MANIFEST', check_PE_application_manifest), -] -} - -if __name__ == '__main__': - retval: int = 0 - for filename in sys.argv[1:]: - binary = lief.parse(filename) - - etype = binary.format - - failed: list[str] = [] - for (name, func) in CHECKS[etype]: - if not func(binary): - failed.append(name) - if failed: - print(f'{filename}: failed {" ".join(failed)}') - retval = 1 - sys.exit(retval) diff --git a/flake.nix b/flake.nix index f769290ebca5..e06fb057b38d 100644 --- a/flake.nix +++ b/flake.nix @@ -62,7 +62,7 @@ "-DWITH_ZMQ=OFF" ]; in - pkgs.stdenv.mkDerivation { + pkgs.ccacheStdenv.mkDerivation { inherit pname version @@ -149,10 +149,11 @@ default = pkgs.mkShell { buildInputs = [ # Benchmarking - pkgs.cargo-flamegraph + cargo-flamegraph pkgs.flamegraph pkgs.hyperfine pkgs.jq + pkgs.just pkgs.perf pkgs.perf-tools pkgs.util-linux diff --git a/justfile b/justfile index 86dbbab91115..5b32a5d7bf85 100644 --- a/justfile +++ b/justfile @@ -7,10 +7,9 @@ default: # Build base and head binaries for CI [group('ci')] -build-assumeutxo-binaries-guix base_commit head_commit: +build-binaries base_commit head_commit: #!/usr/bin/env bash set -euxo pipefail - unset SOURCE_DATE_EPOCH # needed to run on NixOS ./bench-ci/build_binaries.sh {{ base_commit }} {{ head_commit }} # Run uninstrumented benchmarks on mainnet diff --git a/shell.nix b/shell.nix deleted file mode 100644 index 35a9fdbf49c2..000000000000 --- a/shell.nix +++ /dev/null @@ -1,101 +0,0 @@ -# Copyright 0xB10C, willcl-ark -{ pkgs ? import - (fetchTarball "https://github.com/nixos/nixpkgs/archive/nixos-25.11.tar.gz") - { }, }: -let - inherit (pkgs.lib) optionals strings; - inherit (pkgs) stdenv; - - # Override the default cargo-flamegraph with a custom fork - cargo-flamegraph = pkgs.rustPlatform.buildRustPackage rec { - pname = - "flamegraph"; # Match the name in Cargo.toml, doesn't seem to work otherwise - version = "bitcoin-core"; - - src = pkgs.fetchFromGitHub { - owner = "willcl-ark"; - repo = "flamegraph"; - rev = "bitcoin-core"; - sha256 = "sha256-tQbr3MYfAiOxeT12V9au5KQK5X5JeGuV6p8GR/Sgen4="; - }; - - doCheck = false; - cargoHash = "sha256-QWPqTyTFSZNJNayNqLmsQSu0rX26XBKfdLROZ9tRjrg="; - - useFetchCargoVendor = true; - - nativeBuildInputs = - pkgs.lib.optionals stdenv.hostPlatform.isLinux [ pkgs.makeWrapper ]; - buildInputs = pkgs.lib.optionals stdenv.hostPlatform.isDarwin - [ pkgs.darwin.apple_sdk.frameworks.Security ]; - - postFixup = pkgs.lib.optionalString stdenv.hostPlatform.isLinux '' - wrapProgram $out/bin/cargo-flamegraph \ - --set-default PERF ${pkgs.linuxPackages.perf}/bin/perf - wrapProgram $out/bin/flamegraph \ - --set-default PERF ${pkgs.linuxPackages.perf}/bin/perf - ''; - }; - -in pkgs.mkShell { - nativeBuildInputs = with pkgs; [ - autoconf - automake - boost - ccache - clang_18 - cmake - libevent - libtool - pkg-config - sqlite - zeromq - ]; - buildInputs = with pkgs; [ - just - bash - git - shellcheck - python310 - uv - - # Benchmarking - cargo-flamegraph - flamegraph - hyperfine - jq - linuxKernel.packages.linux_6_6.perf - perf-tools - util-linux - - # Binary patching - patchelf - - # Guix - curl - getent - ]; - - shellHook = '' - echo "Bitcoin Core build nix-shell" - echo "" - echo "Setting up python venv" - - # fixes libstdc++ issues and libgl.so issues - export LD_LIBRARY_PATH=${stdenv.cc.cc.lib}/lib/:$LD_LIBRARY_PATH - - uv venv --python 3.10 - source .venv/bin/activate - uv pip install -r pyproject.toml - - patch-binary() { - if [ -z "$1" ]; then - echo "Usage: patch-binary " - return 1 - fi - patchelf --set-interpreter "$(cat $NIX_CC/nix-support/dynamic-linker)" "$1" - } - echo "Added patch-binary command" - echo " Usage: 'patch-binary '" - ''; -} From c3f79d8edbb18be214ce5ca59681f2e0f03be911 Mon Sep 17 00:00:00 2001 From: will Date: Mon, 8 Dec 2025 10:58:35 +0000 Subject: [PATCH 28/48] disable ccache for now --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index e06fb057b38d..3d0c2fa657b4 100644 --- a/flake.nix +++ b/flake.nix @@ -62,7 +62,7 @@ "-DWITH_ZMQ=OFF" ]; in - pkgs.ccacheStdenv.mkDerivation { + pkgs.stdenv.mkDerivation { inherit pname version From 4d998b7fb78853d89519c8d64d56807959fb66ed Mon Sep 17 00:00:00 2001 From: will Date: Mon, 8 Dec 2025 11:19:19 +0000 Subject: [PATCH 29/48] increase build priority --- bench-ci/build_binaries.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench-ci/build_binaries.sh b/bench-ci/build_binaries.sh index 242456250d8d..4c42e5157a7e 100755 --- a/bench-ci/build_binaries.sh +++ b/bench-ci/build_binaries.sh @@ -25,7 +25,7 @@ for build in "base:${base_commit}" "head:${head_commit}"; do name="${build%%:*}" commit="${build#*:}" git checkout "$commit" - taskset -c 2-15 chrt -f 1 nix build -L + taskset -c 0-15 nix build -L cp "./result/bin/bitcoind" "./binaries/${name}/bitcoind" rm -rf "./result" done From 62a5eab3b8b03d6da20422e406bb18a810b1b320 Mon Sep 17 00:00:00 2001 From: will Date: Mon, 8 Dec 2025 12:04:31 +0000 Subject: [PATCH 30/48] use python runner --- .github/workflows/benchmark.yml | 141 ++++--- bench-ci/build_binaries.sh | 38 -- bench-ci/parse_and_plot.py | 247 ----------- bench-ci/prelude.sh | 78 ---- bench-ci/run-benchmark-instrumented.sh | 91 ---- bench-ci/run-benchmark.sh | 64 --- bench.py | 559 +++++++++++++++++++++++++ bench.toml | 30 ++ bench/README.md | 237 +++++++++++ bench/__init__.py | 3 + bench/analyze.py | 535 +++++++++++++++++++++++ bench/benchmark.py | 351 ++++++++++++++++ bench/build.py | 172 ++++++++ bench/capabilities.py | 162 +++++++ bench/config.py | 208 +++++++++ bench/report.py | 453 ++++++++++++++++++++ bench/utils.py | 259 ++++++++++++ flake.nix | 18 +- justfile | 93 +++- 19 files changed, 3130 insertions(+), 609 deletions(-) delete mode 100755 bench-ci/build_binaries.sh delete mode 100755 bench-ci/parse_and_plot.py delete mode 100644 bench-ci/prelude.sh delete mode 100755 bench-ci/run-benchmark-instrumented.sh delete mode 100755 bench-ci/run-benchmark.sh create mode 100755 bench.py create mode 100644 bench.toml create mode 100644 bench/README.md create mode 100644 bench/__init__.py create mode 100644 bench/analyze.py create mode 100644 bench/benchmark.py create mode 100644 bench/build.py create mode 100644 bench/capabilities.py create mode 100644 bench/config.py create mode 100644 bench/report.py create mode 100644 bench/utils.py diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 65e667e88fe1..46aa545e8cf5 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -3,6 +3,7 @@ on: pull_request: branches: - master + jobs: build-binaries: runs-on: [self-hosted, linux, x64] @@ -13,159 +14,175 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 1 + - name: Fetch base commit run: | - echo "CHECKOUT_COMMIT=$(git rev-parse HEAD)" >> "$GITHUB_ENV" + echo "HEAD_SHA=$(git rev-parse HEAD)" >> "$GITHUB_ENV" git fetch --depth=1 origin ${{ github.event.pull_request.base.sha }} + - name: Build both binaries - env: - CCACHE_DIR: /nix/var/cache/ccache run: | - mkdir -p ${{ runner.temp }}/binaries/base - mkdir -p ${{ runner.temp }}/binaries/head - nix develop --command bash -c ' - just build-binaries $BASE_SHA $CHECKOUT_COMMIT - cp binaries/base/bitcoind ${{ runner.temp }}/binaries/base/bitcoind - cp binaries/head/bitcoind ${{ runner.temp }}/binaries/head/bitcoind - ' + nix develop --command python3 bench.py build \ + --binaries-dir ${{ runner.temp }}/binaries \ + $BASE_SHA $HEAD_SHA + - name: Upload binaries uses: actions/upload-artifact@v4 with: name: bitcoind-binaries path: ${{ runner.temp }}/binaries/ + uninstrumented: needs: build-binaries strategy: matrix: include: - - network: mainnet - name: mainnet-default-uninstrumented + - name: mainnet-default-uninstrumented timeout: 600 - datadir_path: /data/pruned-840k dbcache: 450 - - network: mainnet - name: mainnet-large-uninstrumented + - name: mainnet-large-uninstrumented timeout: 600 - datadir_path: /data/pruned-840k dbcache: 32000 runs-on: [self-hosted, linux, x64] timeout-minutes: ${{ matrix.timeout }} env: - ORIGINAL_DATADIR: ${{ matrix.datadir_path }} + ORIGINAL_DATADIR: /data/pruned-840k BASE_SHA: ${{ github.event.pull_request.base.sha }} steps: - name: Checkout repo uses: actions/checkout@v4 with: fetch-depth: 1 + - name: Download binaries uses: actions/download-artifact@v4 with: name: bitcoind-binaries path: ${{ runner.temp }}/binaries + - name: Set binary permissions run: | chmod +x ${{ runner.temp }}/binaries/base/bitcoind chmod +x ${{ runner.temp }}/binaries/head/bitcoind + - name: Fetch base commit run: | - echo "CHECKOUT_COMMIT=$(git rev-parse HEAD)" >> "$GITHUB_ENV" + echo "HEAD_SHA=$(git rev-parse HEAD)" >> "$GITHUB_ENV" git fetch --depth=1 origin ${{ github.event.pull_request.base.sha }} - - name: Run AssumeUTXO ${{ matrix.network }} - env: - TMP_DATADIR: "${{ runner.temp }}/base_datadir" - BINARIES_DIR: "${{ runner.temp }}/binaries" + + - name: Run benchmark run: | - mkdir -p "$TMP_DATADIR" - nix develop --command just run-${{ matrix.network }}-ci $BASE_SHA $CHECKOUT_COMMIT $TMP_DATADIR $ORIGINAL_DATADIR ${{ runner.temp }}/results.json ${{ matrix.dbcache }} $BINARIES_DIR - - uses: actions/upload-artifact@v4 + nix develop --command python3 bench.py run \ + --profile ci \ + --binaries-dir ${{ runner.temp }}/binaries \ + --datadir $ORIGINAL_DATADIR \ + --tmp-datadir ${{ runner.temp }}/datadir \ + --output-dir ${{ runner.temp }}/output \ + --dbcache ${{ matrix.dbcache }} \ + $BASE_SHA $HEAD_SHA + + - name: Upload results + uses: actions/upload-artifact@v4 with: name: result-${{ matrix.name }} - path: "${{ runner.temp }}/results.json" - - name: Write GitHub and runner context files + path: ${{ runner.temp }}/output/results.json + + - name: Write context metadata env: GITHUB_CONTEXT: ${{ toJSON(github) }} RUNNER_CONTEXT: ${{ toJSON(runner) }} run: | - mkdir -p contexts - nix develop --command bash -c ' - echo "$GITHUB_CONTEXT" | jq "del(.token)" > contexts/github.json - echo "$RUNNER_CONTEXT" > contexts/runner.json - ' - - name: Upload context metadata as artifact + mkdir -p ${{ runner.temp }}/contexts + echo "$GITHUB_CONTEXT" | jq "del(.token)" > ${{ runner.temp }}/contexts/github.json + echo "$RUNNER_CONTEXT" > ${{ runner.temp }}/contexts/runner.json + + - name: Upload context metadata uses: actions/upload-artifact@v4 with: name: run-metadata-${{ matrix.name }} - path: ./contexts/ + path: ${{ runner.temp }}/contexts/ + instrumented: needs: build-binaries strategy: matrix: include: - - network: mainnet - name: mainnet-default-instrumented + - name: mainnet-default-instrumented timeout: 600 - datadir_path: /data/pruned-840k dbcache: 450 - - network: mainnet - name: mainnet-large-instrumented + - name: mainnet-large-instrumented timeout: 600 - datadir_path: /data/pruned-840k dbcache: 32000 runs-on: [self-hosted, linux, x64] timeout-minutes: ${{ matrix.timeout }} env: - ORIGINAL_DATADIR: ${{ matrix.datadir_path }} + ORIGINAL_DATADIR: /data/pruned-840k BASE_SHA: ${{ github.event.pull_request.base.sha }} steps: - name: Checkout repo uses: actions/checkout@v4 with: fetch-depth: 1 + - name: Download binaries uses: actions/download-artifact@v4 with: name: bitcoind-binaries path: ${{ runner.temp }}/binaries + - name: Set binary permissions run: | chmod +x ${{ runner.temp }}/binaries/base/bitcoind chmod +x ${{ runner.temp }}/binaries/head/bitcoind + - name: Fetch base commit run: | - echo "CHECKOUT_COMMIT=$(git rev-parse HEAD)" >> "$GITHUB_ENV" + echo "HEAD_SHA=$(git rev-parse HEAD)" >> "$GITHUB_ENV" git fetch --depth=1 origin ${{ github.event.pull_request.base.sha }} - - name: Run AssumeUTXO ${{ matrix.network }} - env: - TMP_DATADIR: "${{ runner.temp }}/base_datadir" - BINARIES_DIR: "${{ runner.temp }}/binaries" + + - name: Run instrumented benchmark run: | - mkdir -p "$TMP_DATADIR" - nix develop --command just run-${{ matrix.network }}-ci-instrumented $BASE_SHA $CHECKOUT_COMMIT $TMP_DATADIR $ORIGINAL_DATADIR ${{ runner.temp }}/results.json ${{ matrix.dbcache }} ${{ runner.temp }}/pngs $BINARIES_DIR - - uses: actions/upload-artifact@v4 + nix develop --command python3 bench.py run \ + --profile ci \ + --instrumented \ + --binaries-dir ${{ runner.temp }}/binaries \ + --datadir $ORIGINAL_DATADIR \ + --tmp-datadir ${{ runner.temp }}/datadir \ + --output-dir ${{ runner.temp }}/output \ + --dbcache ${{ matrix.dbcache }} \ + $BASE_SHA $HEAD_SHA + + - name: Upload results + uses: actions/upload-artifact@v4 with: name: result-${{ matrix.name }} - path: "${{ runner.temp }}/results.json" - - uses: actions/upload-artifact@v4 + path: ${{ runner.temp }}/output/results.json + + - name: Upload plots + uses: actions/upload-artifact@v4 with: name: pngs-${{ matrix.name }} - path: "${{ runner.temp }}/pngs/*.png" - - uses: actions/upload-artifact@v4 + path: ${{ runner.temp }}/output/plots/*.png + if-no-files-found: ignore + + - name: Upload flamegraphs + uses: actions/upload-artifact@v4 with: name: flamegraph-${{ matrix.name }} - path: "**/*-flamegraph.svg" - - name: Write GitHub and runner context files + path: ${{ runner.temp }}/output/*-flamegraph.svg + if-no-files-found: ignore + + - name: Write context metadata env: GITHUB_CONTEXT: ${{ toJSON(github) }} RUNNER_CONTEXT: ${{ toJSON(runner) }} run: | - mkdir -p contexts - nix develop --command bash -c ' - echo "$GITHUB_CONTEXT" | jq "del(.token)" > contexts/github.json - echo "$RUNNER_CONTEXT" > contexts/runner.json - ' - - name: Upload context metadata as artifact + mkdir -p ${{ runner.temp }}/contexts + echo "$GITHUB_CONTEXT" | jq "del(.token)" > ${{ runner.temp }}/contexts/github.json + echo "$RUNNER_CONTEXT" > ${{ runner.temp }}/contexts/runner.json + + - name: Upload context metadata uses: actions/upload-artifact@v4 with: name: run-metadata-${{ matrix.name }} - path: ./contexts/ + path: ${{ runner.temp }}/contexts/ diff --git a/bench-ci/build_binaries.sh b/bench-ci/build_binaries.sh deleted file mode 100755 index 4c42e5157a7e..000000000000 --- a/bench-ci/build_binaries.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env bash -set -euxo pipefail - -if [ $# -ne 2 ]; then - echo "Usage: $0 " - exit 1 -fi - -# Save current state of git -initial_ref=$(git symbolic-ref -q HEAD || git rev-parse HEAD) -if git symbolic-ref -q HEAD >/dev/null; then - initial_state="branch" - initial_branch=${initial_ref#refs/heads/} -else - initial_state="detached" -fi - -base_commit="$1" -head_commit="$2" - -mkdir -p binaries/base -mkdir -p binaries/head - -for build in "base:${base_commit}" "head:${head_commit}"; do - name="${build%%:*}" - commit="${build#*:}" - git checkout "$commit" - taskset -c 0-15 nix build -L - cp "./result/bin/bitcoind" "./binaries/${name}/bitcoind" - rm -rf "./result" -done - -# Restore initial git state -if [ "$initial_state" = "branch" ]; then - git checkout "$initial_branch" -else - git checkout "$initial_ref" -fi diff --git a/bench-ci/parse_and_plot.py b/bench-ci/parse_and_plot.py deleted file mode 100755 index 2a8a112cc4cf..000000000000 --- a/bench-ci/parse_and_plot.py +++ /dev/null @@ -1,247 +0,0 @@ -#!/usr/bin/env python3 -import sys -import os -import re -import datetime -import matplotlib.pyplot as plt -from collections import OrderedDict - - -def parse_updatetip_line(line): - match = re.match( - r'^([\d\-:TZ]+) UpdateTip: new best.+height=(\d+).+tx=(\d+).+cache=([\d.]+)MiB\((\d+)txo\)', - line - ) - if not match: - return None - iso_str, height_str, tx_str, cache_size_mb_str, cache_coins_count_str = match.groups() - parsed_datetime = datetime.datetime.strptime(iso_str, "%Y-%m-%dT%H:%M:%SZ") - return parsed_datetime, int(height_str), int(tx_str), float(cache_size_mb_str), int(cache_coins_count_str) - - -def parse_leveldb_compact_line(line): - match = re.match(r'^([\d\-:TZ]+) \[leveldb] Compacting.*files', line) - if not match: - return None - iso_str = match.groups()[0] - parsed_datetime = datetime.datetime.strptime(iso_str, "%Y-%m-%dT%H:%M:%SZ") - return parsed_datetime - - -def parse_leveldb_generated_table_line(line): - match = re.match(r'^([\d\-:TZ]+) \[leveldb] Generated table.*: (\d+) keys, (\d+) bytes', line) - if not match: - return None - iso_str, keys_count_str, bytes_count_str = match.groups() - parsed_datetime = datetime.datetime.strptime(iso_str, "%Y-%m-%dT%H:%M:%SZ") - return parsed_datetime, int(keys_count_str), int(bytes_count_str) - - -def parse_validation_txadd_line(line): - match = re.match(r'^([\d\-:TZ]+) \[validation] TransactionAddedToMempool: txid=.+wtxid=.+', line) - if not match: - return None - iso_str = match.groups()[0] - parsed_datetime = datetime.datetime.strptime(iso_str, "%Y-%m-%dT%H:%M:%SZ") - return parsed_datetime - - -def parse_coindb_write_batch_line(line): - match = re.match(r'^([\d\-:TZ]+) \[coindb] Writing (partial|final) batch of ([\d.]+) MiB', line) - if not match: - return None - iso_str, is_partial_str, size_mb_str = match.groups() - parsed_datetime = datetime.datetime.strptime(iso_str, "%Y-%m-%dT%H:%M:%SZ") - return parsed_datetime, is_partial_str, float(size_mb_str) - - -def parse_coindb_commit_line(line): - match = re.match(r'^([\d\-:TZ]+) \[coindb] Committed (\d+) changed transaction outputs', line) - if not match: - return None - iso_str, txout_count_str = match.groups() - parsed_datetime = datetime.datetime.strptime(iso_str, "%Y-%m-%dT%H:%M:%SZ") - return parsed_datetime, int(txout_count_str) - - -def parse_log_file(log_file): - with open(log_file, 'r', encoding='utf-8') as f: - update_tip_data = [] - leveldb_compact_data = [] - leveldb_gen_table_data = [] - validation_txadd_data = [] - coindb_write_batch_data = [] - coindb_commit_data = [] - - for line in f: - if result := parse_updatetip_line(line): - update_tip_data.append(result) - elif result := parse_leveldb_compact_line(line): - leveldb_compact_data.append(result) - elif result := parse_leveldb_generated_table_line(line): - leveldb_gen_table_data.append(result) - elif result := parse_validation_txadd_line(line): - validation_txadd_data.append(result) - elif result := parse_coindb_write_batch_line(line): - coindb_write_batch_data.append(result) - elif result := parse_coindb_commit_line(line): - coindb_commit_data.append(result) - - if not update_tip_data: - print("No UpdateTip entries found.") - sys.exit(0) - - assert all(update_tip_data[i][0] <= update_tip_data[i + 1][0] for i in - range(len(update_tip_data) - 1)), "UpdateTip entries are not sorted by time" - - return update_tip_data, leveldb_compact_data, leveldb_gen_table_data, validation_txadd_data, coindb_write_batch_data, coindb_commit_data - - -def generate_plot(x, y, x_label, y_label, title, output_file, is_height_based=False): - if not x or not y: - print(f"Skipping plot '{title}' as there is no data.") - return - - plt.figure(figsize=(30, 10)) - plt.plot(x, y) - plt.title(title, fontsize=20) - plt.xlabel(x_label, fontsize=16) - plt.ylabel(y_label, fontsize=16) - plt.grid(True) - - # Make sure the x-axis covers the full data range - min_x, max_x = min(x), max(x) - plt.xlim(min_x, max_x) - - # Add vertical lines for major protocol upgrades if this is a height-based plot - if is_height_based: - # Define all notable heights from the chainparams file - fork_heights = OrderedDict([ - ('BIP34', 227931), # Block v2, coinbase includes height - ('BIP66', 363725), # Strict DER signatures - ('BIP65', 388381), # OP_CHECKLOCKTIMEVERIFY - ('CSV', 419328), # BIP68, 112, 113 - OP_CHECKSEQUENCEVERIFY - ('Segwit', 481824), # BIP141, 143, 144, 145 - Segregated Witness - ('Taproot', 709632), # BIP341, 342 - Schnorr signatures & Taproot - ('Halving 1', 210000), # First halving - ('Halving 2', 420000), # Second halving - ('Halving 3', 630000), # Third halving - ('Halving 4', 840000), # Fourth halving - ]) - - # Colors for the different types of events - fork_colors = { - 'BIP34': 'blue', - 'BIP66': 'blue', - 'BIP65': 'blue', - 'CSV': 'blue', - 'Segwit': 'green', - 'Taproot': 'red', - 'Halving 1': 'purple', - 'Halving 2': 'purple', - 'Halving 3': 'purple', - 'Halving 4': 'purple', - } - - # Line styles for different types of events - fork_styles = { - 'BIP34': '--', - 'BIP66': '--', - 'BIP65': '--', - 'CSV': '--', - 'Segwit': '--', - 'Taproot': '--', - 'Halving 1': ':', - 'Halving 2': ':', - 'Halving 3': ':', - 'Halving 4': ':', - } - - max_y = max(y) - - # Position text labels at different heights to avoid overlap - text_positions = {} - position_increment = max_y * 0.05 - current_position = max_y * 0.9 - - # Add lines for forks that are in range - for fork_name, height in fork_heights.items(): - if min_x <= height <= max_x: - plt.axvline(x=height, color=fork_colors[fork_name], - linestyle=fork_styles[fork_name]) - - # Avoid label overlaps by staggering vertical positions - if height in text_positions: - # If this x position already has a label, adjust position - text_positions[height] -= position_increment - else: - text_positions[height] = current_position - current_position -= position_increment - if current_position < max_y * 0.1: - current_position = max_y * 0.9 # Reset if we're too low - - plt.text(height, text_positions[height], f'{fork_name} ({height})', - rotation=90, verticalalignment='top', - color=fork_colors[fork_name]) - - plt.xticks(rotation=90, fontsize=12) - plt.yticks(fontsize=12) - plt.tight_layout() - plt.savefig(output_file) - plt.close() - print(f"Saved plot to {output_file}") - - -if __name__ == "__main__": - if len(sys.argv) != 4: - print(f"Usage: {sys.argv[0]} ") - sys.exit(1) - - commit = sys.argv[1] - - log_file = sys.argv[2] - if not os.path.isfile(log_file): - print(f"File not found: {log_file}") - sys.exit(1) - - png_dir = sys.argv[3] - os.makedirs(png_dir, exist_ok=True) - - update_tip_data, leveldb_compact_data, leveldb_gen_table_data, validation_txadd_data, coindb_write_batch_data, coindb_commit_data = parse_log_file(log_file) - times, heights, tx_counts, cache_size, cache_count = zip(*update_tip_data) - float_minutes = [(t - times[0]).total_seconds() / 60 for t in times] - - generate_plot(float_minutes, heights, "Elapsed minutes", "Block Height", "Block Height vs Time", os.path.join(png_dir, f"{commit}-height_vs_time.png")) - generate_plot(heights, cache_size, "Block Height", "Cache Size (MiB)", "Cache Size vs Block Height", os.path.join(png_dir, f"{commit}-cache_vs_height.png"), is_height_based=True) - generate_plot(float_minutes, cache_size, "Elapsed minutes", "Cache Size (MiB)", "Cache Size vs Time", os.path.join(png_dir, f"{commit}-cache_vs_time.png")) - generate_plot(heights, tx_counts, "Block Height", "Transaction Count", "Transactions vs Block Height", os.path.join(png_dir, f"{commit}-tx_vs_height.png"), is_height_based=True) - generate_plot(heights, cache_count, "Block Height", "Coins Cache Size", "Coins Cache Size vs Height", os.path.join(png_dir, f"{commit}-coins_cache_vs_height.png"), is_height_based=True) - - # LevelDB Compaction and Generated Tables - if leveldb_compact_data: - leveldb_compact_times = [(t - times[0]).total_seconds() / 60 for t in leveldb_compact_data] - leveldb_compact_y = [1 for _ in leveldb_compact_times] # dummy y axis to mark compactions - generate_plot(leveldb_compact_times, leveldb_compact_y, "Elapsed minutes", "LevelDB Compaction", "LevelDB Compaction Events vs Time", os.path.join(png_dir, f"{commit}-leveldb_compact_vs_time.png")) - if leveldb_gen_table_data: - leveldb_gen_table_times, leveldb_gen_table_keys, leveldb_gen_table_bytes = zip(*leveldb_gen_table_data) - leveldb_gen_table_float_minutes = [(t - times[0]).total_seconds() / 60 for t in leveldb_gen_table_times] - generate_plot(leveldb_gen_table_float_minutes, leveldb_gen_table_keys, "Elapsed minutes", "Number of keys", "LevelDB Keys Generated vs Time", os.path.join(png_dir, f"{commit}-leveldb_gen_keys_vs_time.png")) - generate_plot(leveldb_gen_table_float_minutes, leveldb_gen_table_bytes, "Elapsed minutes", "Number of bytes", "LevelDB Bytes Generated vs Time", os.path.join(png_dir, f"{commit}-leveldb_gen_bytes_vs_time.png")) - - # validation mempool add transaction lines - if validation_txadd_data: - validation_txadd_times = [(t - times[0]).total_seconds() / 60 for t in validation_txadd_data] - validation_txadd_y = [1 for _ in validation_txadd_times] # dummy y axis to mark transaction additions - generate_plot(validation_txadd_times, validation_txadd_y, "Elapsed minutes", "Transaction Additions", "Transaction Additions to Mempool vs Time", os.path.join(png_dir, f"{commit}-validation_txadd_vs_time.png")) - - # coindb write batch lines - if coindb_write_batch_data: - coindb_write_batch_times, is_partial_strs, sizes_mb = zip(*coindb_write_batch_data) - coindb_write_batch_float_minutes = [(t - times[0]).total_seconds() / 60 for t in coindb_write_batch_times] - generate_plot(coindb_write_batch_float_minutes, sizes_mb, "Elapsed minutes", "Batch Size MiB", "Coin Database Partial/Final Write Batch Size vs Time", os.path.join(png_dir, f"{commit}-coindb_write_batch_size_vs_time.png")) - if coindb_commit_data: - coindb_commit_times, txout_counts = zip(*coindb_commit_data) - coindb_commit_float_minutes = [(t - times[0]).total_seconds() / 60 for t in coindb_commit_times] - generate_plot(coindb_commit_float_minutes, txout_counts, "Elapsed minutes", "Transaction Output Count", "Coin Database Transaction Output Committed vs Time", os.path.join(png_dir, f"{commit}-coindb_commit_txout_vs_time.png")) - - print("Plots saved!") diff --git a/bench-ci/prelude.sh b/bench-ci/prelude.sh deleted file mode 100644 index 98a5232b40fb..000000000000 --- a/bench-ci/prelude.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash -# Shared functions - -set -euxo pipefail - -clean_datadir() { - set -euxo pipefail - - local TMP_DATADIR="$1" - - mkdir -p "${TMP_DATADIR}" - - # If we're in CI, clean without confirmation - if [ -n "${CI:-}" ]; then - rm -Rf "${TMP_DATADIR:?}"/* - else - read -rp "Are you sure you want to delete everything in ${TMP_DATADIR}? [y/N] " response - if [[ "$response" =~ ^[Yy]$ ]]; then - rm -Rf "${TMP_DATADIR:?}"/* - else - echo "Aborting..." - exit 1 - fi - fi -} - -clean_logs() { - set -euxo pipefail - - local TMP_DATADIR="$1" - local logfile="${TMP_DATADIR}/debug.log" - - echo "Checking for ${logfile}" - if [ -e "${logfile}" ]; then - echo "Removing ${logfile}" - rm "${logfile}" - fi -} - -# Executes once before each *set* of timing runs. -setup_run() { - set -euxo pipefail - - local TMP_DATADIR="$1" - clean_datadir "${TMP_DATADIR}" -} - -# Executes before each timing run. -prepare_run() { - set -euxo pipefail - - local TMP_DATADIR="$1" - local ORIGINAL_DATADIR="$2" - - clean_datadir "${TMP_DATADIR}" - # Don't copy hidden files so use * - taskset -c 0-15 cp -r "$ORIGINAL_DATADIR"/* "$TMP_DATADIR" - # Clear page caches - /run/wrappers/bin/drop-caches - clean_logs "${TMP_DATADIR}" -} - -# Executes after the completion of all benchmarking runs for each individual -# command to be benchmarked. -cleanup_run() { - set -euxo pipefail - local TMP_DATADIR="$1" - clean_datadir "${TMP_DATADIR}" -} - -# Export all shared functions for use by hyperfine subshells -export_shared_functions() { - export -f clean_datadir - export -f clean_logs - export -f setup_run - export -f prepare_run - export -f cleanup_run -} diff --git a/bench-ci/run-benchmark-instrumented.sh b/bench-ci/run-benchmark-instrumented.sh deleted file mode 100755 index 1f4be0b0b2dd..000000000000 --- a/bench-ci/run-benchmark-instrumented.sh +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env bash - -set -euxo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "${SCRIPT_DIR}/prelude.sh" - -# Executed after each timing run - generates plots and handles flamegraph -conclude_run() { - set -euxo pipefail - - local commit="$1" - local TMP_DATADIR="$2" - local PNG_DIR="$3" - - # Search in subdirs e.g. $datadir/signet - debug_log=$(find "${TMP_DATADIR}" -name debug.log -print -quit) - if [ -n "${debug_log}" ]; then - echo "Generating plots from ${debug_log}" - if [ -x "bench-ci/parse_and_plot.py" ]; then - bench-ci/parse_and_plot.py "${commit}" "${debug_log}" "${PNG_DIR}" - else - ls -al "bench-ci/" - echo "parse_and_plot.py not found or not executable, skipping plot generation" - fi - else - ls -al "${TMP_DATADIR}/" - echo "debug.log not found, skipping plot generation" - fi - - # Move flamegraph if exists - if [ -e flamegraph.svg ]; then - mv flamegraph.svg "${commit}"-flamegraph.svg - fi -} - -run_benchmark() { - local base_commit="$1" - local head_commit="$2" - local TMP_DATADIR="$3" - local ORIGINAL_DATADIR="$4" - local results_file="$5" - local png_dir="$6" - local chain="$7" - local stop_at_height="$8" - local connect_address="$9" - local dbcache="${10}" - local BINARIES_DIR="${11}" - - # Export functions so they can be used by hyperfine - export_shared_functions - export -f conclude_run - - # Debug: Print all variables being used - echo "=== Debug Information ===" - echo "TMP_DATADIR: ${TMP_DATADIR}" - echo "ORIGINAL_DATADIR: ${ORIGINAL_DATADIR}" - echo "BINARIES_DIR: ${BINARIES_DIR}" - echo "base_commit: ${base_commit}" - echo "head_commit: ${head_commit}" - echo "results_file: ${results_file}" - echo "png_dir: ${png_dir}" - echo "chain: ${chain}" - echo "stop_at_height: ${stop_at_height}" - echo "connect_address: ${connect_address}" - echo "dbcache: ${dbcache}" - printf '\n' - - # Run hyperfine - hyperfine \ - --shell=bash \ - --setup "setup_run ${TMP_DATADIR}" \ - --prepare "prepare_run ${TMP_DATADIR} ${ORIGINAL_DATADIR}" \ - --conclude "conclude_run {commit} ${TMP_DATADIR} ${png_dir}" \ - --cleanup "cleanup_run ${TMP_DATADIR}" \ - --runs 1 \ - --export-json "${results_file}" \ - --show-output \ - --command-name "base (${base_commit})" \ - --command-name "head (${head_commit})" \ - "taskset -c 1 flamegraph --palette bitcoin --title 'bitcoind IBD@{commit}' -c 'record -F 101 --call-graph fp' -- taskset -c 2-15 chrt -o 0 ${BINARIES_DIR}/{commit}/bitcoind -datadir=${TMP_DATADIR} -connect=${connect_address} -daemon=0 -prune=10000 -chain=${chain} -stopatheight=${stop_at_height} -dbcache=${dbcache} -printtoconsole=0 -debug=coindb -debug=leveldb -debug=bench -debug=validation" \ - -L commit "base,head" -} - -# Main execution -if [ "$#" -ne 11 ]; then - echo "Usage: $0 base_commit head_commit TMP_DATADIR ORIGINAL_DATADIR results_dir png_dir chain stop_at_height connect_address dbcache BINARIES_DIR" - exit 1 -fi - -run_benchmark "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" "$9" "${10}" "${11}" diff --git a/bench-ci/run-benchmark.sh b/bench-ci/run-benchmark.sh deleted file mode 100755 index bce7857919ac..000000000000 --- a/bench-ci/run-benchmark.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env bash - -set -euxo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "${SCRIPT_DIR}/prelude.sh" - -# Executed after each timing run (no-op for uninstrumented) -conclude_run() { - set -euxo pipefail - return 0 -} - -run_benchmark() { - local base_commit="$1" - local head_commit="$2" - local TMP_DATADIR="$3" - local ORIGINAL_DATADIR="$4" - local results_file="$5" - local chain="$6" - local stop_at_height="$7" - local connect_address="$8" - local dbcache="${9}" - local BINARIES_DIR="${10}" - - # Export functions so they can be used by hyperfine - export_shared_functions - - # Debug: Print all variables being used - echo "=== Debug Information ===" - echo "TMP_DATADIR: ${TMP_DATADIR}" - echo "ORIGINAL_DATADIR: ${ORIGINAL_DATADIR}" - echo "BINARIES_DIR: ${BINARIES_DIR}" - echo "base_commit: ${base_commit}" - echo "head_commit: ${head_commit}" - echo "results_file: ${results_file}" - echo "chain: ${chain}" - echo "stop_at_height: ${stop_at_height}" - echo "connect_address: ${connect_address}" - echo "dbcache: ${dbcache}" - printf '\n' - - # Run hyperfine - hyperfine \ - --shell=bash \ - --setup "setup_run ${TMP_DATADIR}" \ - --prepare "prepare_run ${TMP_DATADIR} ${ORIGINAL_DATADIR}" \ - --cleanup "cleanup_run ${TMP_DATADIR}" \ - --runs 3 \ - --export-json "${results_file}" \ - --show-output \ - --command-name "base (${base_commit})" \ - --command-name "head (${head_commit})" \ - "taskset -c 2-15 chrt -o 0 ${BINARIES_DIR}/{commit}/bitcoind -datadir=${TMP_DATADIR} -connect=${connect_address} -daemon=0 -prune=10000 -chain=${chain} -stopatheight=${stop_at_height} -dbcache=${dbcache} -printtoconsole=0" \ - -L commit "base,head" -} - -# Main execution -if [ "$#" -ne 10 ]; then - echo "Usage: $0 base_commit head_commit TMP_DATADIR ORIGINAL_DATADIR results_dir chain stop_at_height connect_address dbcache BINARIES_DIR" - exit 1 -fi - -run_benchmark "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" "${9}" "${10}" diff --git a/bench.py b/bench.py new file mode 100755 index 000000000000..072f4ef95fc4 --- /dev/null +++ b/bench.py @@ -0,0 +1,559 @@ +#!/usr/bin/env python3 +"""Benchcoin - Bitcoin Core benchmarking toolkit. + +A unified CLI for building, benchmarking, analyzing, and reporting +on Bitcoin Core performance. + +Usage: + bench.py build BASE HEAD Build bitcoind at two commits + bench.py run BASE HEAD Run benchmark + bench.py analyze LOGFILE Generate plots from debug.log + bench.py report INPUT OUTPUT Generate HTML report + bench.py full BASE HEAD Complete pipeline: build → run → analyze +""" + +from __future__ import annotations + +import argparse +import logging +import sys +from pathlib import Path + +from bench.capabilities import detect_capabilities +from bench.config import build_config + +# Setup logging +logging.basicConfig( + level=logging.INFO, + format="%(levelname)s: %(message)s", +) +logger = logging.getLogger(__name__) + + +def cmd_build(args: argparse.Namespace) -> int: + """Build bitcoind at two commits.""" + from bench.build import BuildPhase + + capabilities = detect_capabilities() + config = build_config( + cli_args={ + "binaries_dir": args.binaries_dir, + "skip_existing": args.skip_existing, + "no_cpu_pinning": args.no_cpu_pinning, + "dry_run": args.dry_run, + "verbose": args.verbose, + }, + config_file=Path(args.config) if args.config else None, + profile=args.profile, + ) + + if args.verbose: + logging.getLogger().setLevel(logging.DEBUG) + + phase = BuildPhase(config, capabilities) + + try: + result = phase.run( + args.base_commit, + args.head_commit, + binaries_dir=Path(args.binaries_dir) if args.binaries_dir else None, + ) + logger.info(f"Built base binary: {result.base_binary}") + logger.info(f"Built head binary: {result.head_binary}") + return 0 + except Exception as e: + logger.error(f"Build failed: {e}") + return 1 + + +def cmd_run(args: argparse.Namespace) -> int: + """Run benchmark comparing two commits.""" + from bench.benchmark import BenchmarkPhase + + capabilities = detect_capabilities() + config = build_config( + cli_args={ + "datadir": args.datadir, + "tmp_datadir": args.tmp_datadir, + "binaries_dir": args.binaries_dir, + "output_dir": args.output_dir, + "stop_height": args.stop_height, + "dbcache": args.dbcache, + "runs": args.runs, + "connect": args.connect, + "chain": args.chain, + "instrumented": args.instrumented, + "no_cpu_pinning": args.no_cpu_pinning, + "no_cache_drop": args.no_cache_drop, + "dry_run": args.dry_run, + "verbose": args.verbose, + }, + config_file=Path(args.config) if args.config else None, + profile=args.profile, + ) + + if args.verbose: + logging.getLogger().setLevel(logging.DEBUG) + + # Validate config + errors = config.validate() + if errors: + for error in errors: + logger.error(error) + return 1 + + # Check binaries exist + binaries_dir = ( + Path(args.binaries_dir) if args.binaries_dir else Path(config.binaries_dir) + ) + base_binary = binaries_dir / "base" / "bitcoind" + head_binary = binaries_dir / "head" / "bitcoind" + + if not base_binary.exists(): + logger.error(f"Base binary not found: {base_binary}") + logger.error("Run 'bench.py build' first") + return 1 + + if not head_binary.exists(): + logger.error(f"Head binary not found: {head_binary}") + logger.error("Run 'bench.py build' first") + return 1 + + phase = BenchmarkPhase(config, capabilities) + + try: + result = phase.run( + base_commit=args.base_commit, + head_commit=args.head_commit, + base_binary=base_binary, + head_binary=head_binary, + datadir=Path(config.datadir), + output_dir=Path(config.output_dir), + ) + logger.info(f"Results saved to: {result.results_file}") + return 0 + except Exception as e: + logger.error(f"Benchmark failed: {e}") + if args.verbose: + import traceback + + traceback.print_exc() + return 1 + + +def cmd_analyze(args: argparse.Namespace) -> int: + """Generate plots from debug.log.""" + from bench.analyze import AnalyzePhase + + if args.verbose: + logging.getLogger().setLevel(logging.DEBUG) + + log_file = Path(args.log_file) + output_dir = Path(args.output_dir) + + if not log_file.exists(): + logger.error(f"Log file not found: {log_file}") + return 1 + + phase = AnalyzePhase() + + try: + result = phase.run( + commit=args.commit, + log_file=log_file, + output_dir=output_dir, + ) + logger.info(f"Generated {len(result.plots)} plots in {result.output_dir}") + return 0 + except Exception as e: + logger.error(f"Analysis failed: {e}") + if args.verbose: + import traceback + + traceback.print_exc() + return 1 + + +def cmd_report(args: argparse.Namespace) -> int: + """Generate HTML report from benchmark results.""" + from bench.report import ReportPhase + + if args.verbose: + logging.getLogger().setLevel(logging.DEBUG) + + input_dir = Path(args.input_dir) + output_dir = Path(args.output_dir) + + if not input_dir.exists(): + logger.error(f"Input directory not found: {input_dir}") + return 1 + + phase = ReportPhase() + + try: + result = phase.run( + input_dir=input_dir, + output_dir=output_dir, + title=args.title or "Benchmark Results", + ) + + # Print speedups + if result.speedups: + logger.info("Speedups:") + for network, speedup in result.speedups.items(): + sign = "+" if speedup > 0 else "" + logger.info(f" {network}: {sign}{speedup}%") + + return 0 + except Exception as e: + logger.error(f"Report generation failed: {e}") + if args.verbose: + import traceback + + traceback.print_exc() + return 1 + + +def cmd_full(args: argparse.Namespace) -> int: + """Run full pipeline: build → run → analyze.""" + from bench.analyze import AnalyzePhase + from bench.benchmark import BenchmarkPhase + from bench.build import BuildPhase + from bench.utils import find_debug_log + + capabilities = detect_capabilities() + config = build_config( + cli_args={ + "datadir": args.datadir, + "tmp_datadir": args.tmp_datadir, + "binaries_dir": args.binaries_dir, + "output_dir": args.output_dir, + "stop_height": args.stop_height, + "dbcache": args.dbcache, + "runs": args.runs, + "connect": args.connect, + "chain": args.chain, + "instrumented": args.instrumented, + "skip_existing": args.skip_existing, + "no_cpu_pinning": args.no_cpu_pinning, + "no_cache_drop": args.no_cache_drop, + "dry_run": args.dry_run, + "verbose": args.verbose, + }, + config_file=Path(args.config) if args.config else None, + profile=args.profile, + ) + + if args.verbose: + logging.getLogger().setLevel(logging.DEBUG) + + # Validate config + errors = config.validate() + if errors: + for error in errors: + logger.error(error) + return 1 + + output_dir = Path(config.output_dir) + binaries_dir = Path(config.binaries_dir) + + # Phase 1: Build + logger.info("=== Phase 1: Build ===") + build_phase = BuildPhase(config, capabilities) + + try: + build_result = build_phase.run( + args.base_commit, + args.head_commit, + binaries_dir=binaries_dir, + ) + except Exception as e: + logger.error(f"Build failed: {e}") + return 1 + + # Phase 2: Benchmark + logger.info("=== Phase 2: Benchmark ===") + benchmark_phase = BenchmarkPhase(config, capabilities) + + try: + benchmark_result = benchmark_phase.run( + base_commit=build_result.base_commit, + head_commit=build_result.head_commit, + base_binary=build_result.base_binary, + head_binary=build_result.head_binary, + datadir=Path(config.datadir), + output_dir=output_dir, + ) + except Exception as e: + logger.error(f"Benchmark failed: {e}") + return 1 + + # Phase 3: Analyze (for instrumented runs) + if config.instrumented: + logger.info("=== Phase 3: Analyze ===") + analyze_phase = AnalyzePhase(config) + + # Analyze base debug log + if benchmark_result.debug_log_base: + try: + analyze_phase.run( + commit=build_result.base_commit, + log_file=benchmark_result.debug_log_base, + output_dir=output_dir / "plots", + ) + except Exception as e: + logger.warning(f"Analysis for base failed: {e}") + + # Analyze head debug log + if benchmark_result.debug_log_head: + try: + analyze_phase.run( + commit=build_result.head_commit, + log_file=benchmark_result.debug_log_head, + output_dir=output_dir / "plots", + ) + except Exception as e: + logger.warning(f"Analysis for head failed: {e}") + + logger.info("=== Complete ===") + logger.info(f"Results: {benchmark_result.results_file}") + return 0 + + +def main() -> int: + """Main entry point.""" + parser = argparse.ArgumentParser( + description="Benchcoin - Bitcoin Core benchmarking toolkit", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, + ) + + # Global options + parser.add_argument( + "--config", + metavar="PATH", + help="Config file (default: bench.toml)", + ) + parser.add_argument( + "--profile", + choices=["quick", "full", "ci"], + default="full", + help="Configuration profile (default: full)", + ) + parser.add_argument( + "-v", + "--verbose", + action="store_true", + help="Verbose output", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Show what would be done without executing", + ) + + subparsers = parser.add_subparsers(dest="command", help="Commands") + + # Build command + build_parser = subparsers.add_parser("build", help="Build bitcoind at two commits") + build_parser.add_argument("base_commit", help="Base commit (for comparison)") + build_parser.add_argument("head_commit", help="Head commit (new code)") + build_parser.add_argument( + "--binaries-dir", + metavar="PATH", + help="Where to store binaries (default: ./binaries)", + ) + build_parser.add_argument( + "--skip-existing", + action="store_true", + help="Skip build if binary already exists", + ) + build_parser.add_argument( + "--no-cpu-pinning", + action="store_true", + help="Disable CPU affinity", + ) + build_parser.set_defaults(func=cmd_build) + + # Run command + run_parser = subparsers.add_parser("run", help="Run benchmark") + run_parser.add_argument("base_commit", help="Base commit hash") + run_parser.add_argument("head_commit", help="Head commit hash") + run_parser.add_argument( + "--datadir", + required=True, + metavar="PATH", + help="Source datadir with blockchain snapshot", + ) + run_parser.add_argument( + "--tmp-datadir", + metavar="PATH", + help="Temp datadir for benchmark runs", + ) + run_parser.add_argument( + "--binaries-dir", + metavar="PATH", + help="Location of pre-built binaries", + ) + run_parser.add_argument( + "--output-dir", + metavar="PATH", + help="Output directory for results", + ) + run_parser.add_argument( + "--stop-height", + type=int, + metavar="N", + help="Block height to stop at", + ) + run_parser.add_argument( + "--dbcache", + type=int, + metavar="N", + help="Database cache size in MB", + ) + run_parser.add_argument( + "--runs", + type=int, + metavar="N", + help="Number of benchmark iterations", + ) + run_parser.add_argument( + "--connect", + metavar="ADDR", + help="Connect address for sync", + ) + run_parser.add_argument( + "--chain", + choices=["main", "testnet", "signet", "regtest"], + help="Chain to use", + ) + run_parser.add_argument( + "--instrumented", + action="store_true", + help="Enable profiling (flamegraph + debug logging)", + ) + run_parser.add_argument( + "--no-cpu-pinning", + action="store_true", + help="Disable CPU affinity and scheduler priority", + ) + run_parser.add_argument( + "--no-cache-drop", + action="store_true", + help="Skip cache dropping between runs", + ) + run_parser.set_defaults(func=cmd_run) + + # Analyze command + analyze_parser = subparsers.add_parser( + "analyze", help="Generate plots from debug.log" + ) + analyze_parser.add_argument("commit", help="Commit hash (for naming)") + analyze_parser.add_argument("log_file", help="Path to debug.log") + analyze_parser.add_argument( + "--output-dir", + default="./plots", + metavar="PATH", + help="Output directory for plots", + ) + analyze_parser.set_defaults(func=cmd_analyze) + + # Report command + report_parser = subparsers.add_parser("report", help="Generate HTML report") + report_parser.add_argument("input_dir", help="Directory with results.json") + report_parser.add_argument("output_dir", help="Output directory for report") + report_parser.add_argument( + "--title", + help="Report title", + ) + report_parser.set_defaults(func=cmd_report) + + # Full command + full_parser = subparsers.add_parser( + "full", help="Full pipeline: build → run → analyze" + ) + full_parser.add_argument("base_commit", help="Base commit (for comparison)") + full_parser.add_argument("head_commit", help="Head commit (new code)") + full_parser.add_argument( + "--datadir", + required=True, + metavar="PATH", + help="Source datadir with blockchain snapshot", + ) + full_parser.add_argument( + "--tmp-datadir", + metavar="PATH", + help="Temp datadir for benchmark runs", + ) + full_parser.add_argument( + "--binaries-dir", + metavar="PATH", + help="Where to store binaries", + ) + full_parser.add_argument( + "--output-dir", + metavar="PATH", + help="Output directory for results", + ) + full_parser.add_argument( + "--stop-height", + type=int, + metavar="N", + help="Block height to stop at", + ) + full_parser.add_argument( + "--dbcache", + type=int, + metavar="N", + help="Database cache size in MB", + ) + full_parser.add_argument( + "--runs", + type=int, + metavar="N", + help="Number of benchmark iterations", + ) + full_parser.add_argument( + "--connect", + metavar="ADDR", + help="Connect address for sync", + ) + full_parser.add_argument( + "--chain", + choices=["main", "testnet", "signet", "regtest"], + help="Chain to use", + ) + full_parser.add_argument( + "--instrumented", + action="store_true", + help="Enable profiling (flamegraph + debug logging)", + ) + full_parser.add_argument( + "--skip-existing", + action="store_true", + help="Skip build if binary already exists", + ) + full_parser.add_argument( + "--no-cpu-pinning", + action="store_true", + help="Disable CPU affinity and scheduler priority", + ) + full_parser.add_argument( + "--no-cache-drop", + action="store_true", + help="Skip cache dropping between runs", + ) + full_parser.set_defaults(func=cmd_full) + + args = parser.parse_args() + + if not args.command: + parser.print_help() + return 1 + + return args.func(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bench.toml b/bench.toml new file mode 100644 index 000000000000..0513f4a0abf5 --- /dev/null +++ b/bench.toml @@ -0,0 +1,30 @@ +# Benchcoin configuration +# Values here override built-in defaults but are overridden by environment +# variables (BENCH_*) and CLI arguments. + +[defaults] +chain = "main" +dbcache = 450 +stop_height = 855000 +runs = 3 +# connect = "" # Empty or omit to use public P2P network + +[paths] +binaries_dir = "./binaries" +output_dir = "./bench-output" + +# Profiles override specific defaults +# Usage: bench.py --profile quick full HEAD~1 HEAD + +[profiles.quick] +stop_height = 1100 +runs = 3 + +[profiles.full] +stop_height = 855000 +runs = 3 + +[profiles.ci] +stop_height = 855000 +runs = 3 +connect = "148.251.128.115:33333" diff --git a/bench/README.md b/bench/README.md new file mode 100644 index 000000000000..9cf11ac9df4b --- /dev/null +++ b/bench/README.md @@ -0,0 +1,237 @@ +# Benchcoin + +A CLI for benchmarking Bitcoin Core IBD. + +## Quick Start + +```bash +# Quick smoke test on signet (requires nix) +nix develop --command python3 bench.py --profile quick full \ + --chain signet --datadir /path/to/signet/datadir HEAD~1 HEAD + +# Or use just (wraps nix develop) +just quick HEAD~1 HEAD /path/to/signet/datadir +``` + +## Requirements + +- **Nix** with flakes enabled (provides hyperfine, flamegraph, etc.) +- A blockchain datadir snapshot to benchmark against +- Two git commits to compare + +Optional (auto-detected, gracefully degrades without): +- `/run/wrappers/bin/drop-caches` (NixOS) - clears page cache between runs +- `taskset` / `chrt` - CPU affinity and scheduler priority + +## Commands + +``` +bench.py [GLOBAL_OPTIONS] COMMAND [OPTIONS] ARGS + +Global Options: + --profile {quick,full,ci} Configuration profile + --config PATH Custom config file + -v, --verbose Verbose output + --dry-run Show what would run + +Commands: + build Build bitcoind at two commits + run Run benchmark (requires pre-built binaries) + analyze Generate plots from debug.log + report Generate HTML report + full Complete pipeline: build → run → analyze +``` + +### build + +Build bitcoind binaries at two commits for comparison: + +```bash +python3 bench.py build HEAD~1 HEAD +python3 bench.py build --binaries-dir /tmp/bins abc123 def456 +python3 bench.py build --skip-existing HEAD~1 HEAD # reuse existing +``` + +### run + +Run hyperfine benchmark comparing two pre-built binaries: + +```bash +python3 bench.py run --datadir /data/snapshot HEAD~1 HEAD +python3 bench.py run --instrumented --datadir /data/snapshot HEAD~1 HEAD +``` + +Options: +- `--datadir PATH` - Source blockchain snapshot (required) +- `--tmp-datadir PATH` - Working directory (default: ./bench-output/tmp-datadir) +- `--stop-height N` - Block height to sync to +- `--dbcache N` - Database cache in MB +- `--runs N` - Number of iterations (default: 3, forced to 1 if instrumented) +- `--instrumented` - Enable flamegraph profiling and debug logging +- `--connect ADDR` - P2P node to sync from (empty = public network) +- `--chain {main,signet,testnet,regtest}` - Which chain +- `--no-cpu-pinning` - Disable taskset/chrt +- `--no-cache-drop` - Don't clear page cache between runs + +### analyze + +Generate plots from a debug.log file: + +```bash +python3 bench.py analyze abc123 /path/to/debug.log --output-dir ./plots +``` + +Generates PNG plots for: +- Block height vs time +- Cache size vs height/time +- Transaction count vs height +- LevelDB compaction events +- CoinDB write batches + +### report + +Generate HTML report from benchmark results: + +```bash +python3 bench.py report ./bench-output ./report +``` + +### full + +Run complete pipeline (build + run + analyze if instrumented): + +```bash +python3 bench.py --profile quick full --chain signet --datadir /tmp/signet HEAD~1 HEAD +python3 bench.py --profile full full --datadir /data/mainnet HEAD~1 HEAD +``` + +## Profiles + +Profiles set sensible defaults for common scenarios: + +| Profile | stop_height | runs | dbcache | connect | +|---------|-------------|------|---------|---------| +| quick | 1,500 | 1 | 450 | (public network) | +| full | 855,000 | 3 | 450 | (public network) | +| ci | 855,000 | 3 | 450 | 148.251.128.115:33333 | + +Override any profile setting with CLI flags: + +```bash +python3 bench.py --profile quick full --stop-height 5000 --datadir ... HEAD~1 HEAD +``` + +## Configuration + +Configuration is layered (lowest to highest priority): + +1. Built-in defaults +2. `bench.toml` (in repo root) +3. Environment variables (`BENCH_DATADIR`, `BENCH_DBCACHE`, etc.) +4. CLI arguments + +### bench.toml + +```toml +[defaults] +chain = "main" +dbcache = 450 +stop_height = 855000 +runs = 3 + +[paths] +binaries_dir = "./binaries" +output_dir = "./bench-output" + +[profiles.quick] +stop_height = 1500 +runs = 1 +dbcache = 450 + +[profiles.ci] +connect = "148.251.128.115:33333" +``` + +### Environment Variables + +```bash +export BENCH_DATADIR=/data/snapshot +export BENCH_DBCACHE=1000 +export BENCH_STOP_HEIGHT=100000 +``` + +## Justfile Recipes + +The justfile wraps common operations with `nix develop`: + +```bash +just quick HEAD~1 HEAD /path/to/datadir # Quick signet test +just full HEAD~1 HEAD /path/to/datadir # Full mainnet benchmark +just instrumented HEAD~1 HEAD /path/to/datadir # With flamegraphs +just build HEAD~1 HEAD # Build only +just run HEAD~1 HEAD /path/to/datadir # Run only (binaries must exist) +``` + +## Architecture + +``` +bench.py CLI entry point (argparse) +bench/ +├── config.py Layered configuration (TOML + env + CLI) +├── capabilities.py System capability detection +├── build.py Build phase (nix build) +├── benchmark.py Benchmark phase (hyperfine) +├── analyze.py Plot generation (matplotlib) +├── report.py HTML report generation +└── utils.py Git operations, datadir management +``` + +### Capability Detection + +The tool auto-detects system capabilities and gracefully degrades: + +```python +from bench.capabilities import detect_capabilities +caps = detect_capabilities() +# caps.has_hyperfine, caps.can_drop_caches, caps.can_pin_cpu, etc. +``` + +Missing optional features emit warnings but don't fail: + +``` +WARNING: drop-caches not available - cache won't be cleared between runs +WARNING: taskset not available - CPU affinity won't be set +``` + +Missing required features (hyperfine, flamegraph for instrumented) cause errors. + +### Hyperfine Integration + +The benchmark phase generates temporary shell scripts for hyperfine hooks: + +- `setup` - Clean tmp datadir (once before all runs) +- `prepare` - Copy snapshot, drop caches, clean logs (before each run) +- `cleanup` - Clean tmp datadir (after all runs per command) +- `conclude` - Collect flamegraph/logs (instrumented only, after each run) + +### Instrumented Mode + +When `--instrumented` is set: + +1. Wraps bitcoind in `flamegraph` for CPU profiling +2. Enables debug logging: `-debug=coindb -debug=leveldb -debug=bench -debug=validation` +3. Forces `runs=1` (profiling overhead makes multiple runs pointless) +4. Generates flamegraph SVGs and performance plots + +## CI Integration + +GitHub Actions workflows call bench.py directly (already in nix develop): + +```yaml +- run: | + nix develop --command python3 bench.py build \ + --binaries-dir ${{ runner.temp }}/binaries \ + $BASE_SHA $HEAD_SHA +``` + +CI-specific paths and the dedicated sync node are configured via `--profile ci`. diff --git a/bench/__init__.py b/bench/__init__.py new file mode 100644 index 000000000000..cb50424b155c --- /dev/null +++ b/bench/__init__.py @@ -0,0 +1,3 @@ +"""Benchcoin - Bitcoin Core benchmarking toolkit.""" + +__version__ = "0.1.0" diff --git a/bench/analyze.py b/bench/analyze.py new file mode 100644 index 000000000000..a31b807c422b --- /dev/null +++ b/bench/analyze.py @@ -0,0 +1,535 @@ +"""Analyze phase - parse debug.log and generate performance plots. + +Refactored from bench-ci/parse_and_plot.py for better structure and reusability. +""" + +from __future__ import annotations + +import datetime +import logging +import re +from collections import OrderedDict +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING + +# matplotlib is optional - gracefully handle if not installed +try: + import matplotlib.pyplot as plt + + HAS_MATPLOTLIB = True +except ImportError: + HAS_MATPLOTLIB = False + +if TYPE_CHECKING: + from .config import Config + +logger = logging.getLogger(__name__) + +# Bitcoin fork heights for plot annotations +FORK_HEIGHTS = OrderedDict( + [ + ("BIP34", 227931), # Block v2, coinbase includes height + ("BIP66", 363725), # Strict DER signatures + ("BIP65", 388381), # OP_CHECKLOCKTIMEVERIFY + ("CSV", 419328), # BIP68, 112, 113 - OP_CHECKSEQUENCEVERIFY + ("Segwit", 481824), # BIP141, 143, 144, 145 - Segregated Witness + ("Taproot", 709632), # BIP341, 342 - Schnorr signatures & Taproot + ("Halving 1", 210000), # First halving + ("Halving 2", 420000), # Second halving + ("Halving 3", 630000), # Third halving + ("Halving 4", 840000), # Fourth halving + ] +) + +FORK_COLORS = { + "BIP34": "blue", + "BIP66": "blue", + "BIP65": "blue", + "CSV": "blue", + "Segwit": "green", + "Taproot": "red", + "Halving 1": "purple", + "Halving 2": "purple", + "Halving 3": "purple", + "Halving 4": "purple", +} + +FORK_STYLES = { + "BIP34": "--", + "BIP66": "--", + "BIP65": "--", + "CSV": "--", + "Segwit": "--", + "Taproot": "--", + "Halving 1": ":", + "Halving 2": ":", + "Halving 3": ":", + "Halving 4": ":", +} + + +@dataclass +class UpdateTipEntry: + """Parsed UpdateTip log entry.""" + + timestamp: datetime.datetime + height: int + tx_count: int + cache_size_mb: float + cache_coins_count: int + + +@dataclass +class LevelDBCompactEntry: + """Parsed LevelDB compaction log entry.""" + + timestamp: datetime.datetime + + +@dataclass +class LevelDBGenTableEntry: + """Parsed LevelDB generated table log entry.""" + + timestamp: datetime.datetime + keys_count: int + bytes_count: int + + +@dataclass +class ValidationTxAddEntry: + """Parsed validation transaction added log entry.""" + + timestamp: datetime.datetime + + +@dataclass +class CoinDBWriteBatchEntry: + """Parsed coindb write batch log entry.""" + + timestamp: datetime.datetime + is_partial: bool + size_mb: float + + +@dataclass +class CoinDBCommitEntry: + """Parsed coindb commit log entry.""" + + timestamp: datetime.datetime + txout_count: int + + +@dataclass +class ParsedLog: + """All parsed data from a debug.log file.""" + + update_tip: list[UpdateTipEntry] + leveldb_compact: list[LevelDBCompactEntry] + leveldb_gen_table: list[LevelDBGenTableEntry] + validation_txadd: list[ValidationTxAddEntry] + coindb_write_batch: list[CoinDBWriteBatchEntry] + coindb_commit: list[CoinDBCommitEntry] + + +@dataclass +class AnalyzeResult: + """Result of the analyze phase.""" + + commit: str + output_dir: Path + plots: list[Path] + + +class LogParser: + """Parse bitcoind debug.log files.""" + + # Regex patterns + UPDATETIP_RE = re.compile( + r"^([\d\-:TZ]+) UpdateTip: new best.+height=(\d+).+tx=(\d+).+cache=([\d.]+)MiB\((\d+)txo\)" + ) + LEVELDB_COMPACT_RE = re.compile(r"^([\d\-:TZ]+) \[leveldb] Compacting.*files") + LEVELDB_GEN_TABLE_RE = re.compile( + r"^([\d\-:TZ]+) \[leveldb] Generated table.*: (\d+) keys, (\d+) bytes" + ) + VALIDATION_TXADD_RE = re.compile( + r"^([\d\-:TZ]+) \[validation] TransactionAddedToMempool: txid=.+wtxid=.+" + ) + COINDB_WRITE_BATCH_RE = re.compile( + r"^([\d\-:TZ]+) \[coindb] Writing (partial|final) batch of ([\d.]+) MiB" + ) + COINDB_COMMIT_RE = re.compile( + r"^([\d\-:TZ]+) \[coindb] Committed (\d+) changed transaction outputs" + ) + + @staticmethod + def parse_timestamp(iso_str: str) -> datetime.datetime: + """Parse ISO 8601 timestamp from log.""" + return datetime.datetime.strptime(iso_str, "%Y-%m-%dT%H:%M:%SZ") + + def parse_file(self, log_file: Path) -> ParsedLog: + """Parse a debug.log file and extract all relevant data.""" + update_tip: list[UpdateTipEntry] = [] + leveldb_compact: list[LevelDBCompactEntry] = [] + leveldb_gen_table: list[LevelDBGenTableEntry] = [] + validation_txadd: list[ValidationTxAddEntry] = [] + coindb_write_batch: list[CoinDBWriteBatchEntry] = [] + coindb_commit: list[CoinDBCommitEntry] = [] + + with open(log_file, "r", encoding="utf-8") as f: + for line in f: + if match := self.UPDATETIP_RE.match(line): + iso_str, height, tx, cache_mb, coins = match.groups() + update_tip.append( + UpdateTipEntry( + timestamp=self.parse_timestamp(iso_str), + height=int(height), + tx_count=int(tx), + cache_size_mb=float(cache_mb), + cache_coins_count=int(coins), + ) + ) + elif match := self.LEVELDB_COMPACT_RE.match(line): + leveldb_compact.append( + LevelDBCompactEntry( + timestamp=self.parse_timestamp(match.group(1)) + ) + ) + elif match := self.LEVELDB_GEN_TABLE_RE.match(line): + iso_str, keys, bytes_count = match.groups() + leveldb_gen_table.append( + LevelDBGenTableEntry( + timestamp=self.parse_timestamp(iso_str), + keys_count=int(keys), + bytes_count=int(bytes_count), + ) + ) + elif match := self.VALIDATION_TXADD_RE.match(line): + validation_txadd.append( + ValidationTxAddEntry( + timestamp=self.parse_timestamp(match.group(1)) + ) + ) + elif match := self.COINDB_WRITE_BATCH_RE.match(line): + iso_str, batch_type, size_mb = match.groups() + coindb_write_batch.append( + CoinDBWriteBatchEntry( + timestamp=self.parse_timestamp(iso_str), + is_partial=(batch_type == "partial"), + size_mb=float(size_mb), + ) + ) + elif match := self.COINDB_COMMIT_RE.match(line): + iso_str, txout_count = match.groups() + coindb_commit.append( + CoinDBCommitEntry( + timestamp=self.parse_timestamp(iso_str), + txout_count=int(txout_count), + ) + ) + + return ParsedLog( + update_tip=update_tip, + leveldb_compact=leveldb_compact, + leveldb_gen_table=leveldb_gen_table, + validation_txadd=validation_txadd, + coindb_write_batch=coindb_write_batch, + coindb_commit=coindb_commit, + ) + + +class PlotGenerator: + """Generate performance plots from parsed log data.""" + + def __init__(self, commit: str, output_dir: Path): + self.commit = commit + self.output_dir = output_dir + self.generated_plots: list[Path] = [] + + if not HAS_MATPLOTLIB: + raise RuntimeError( + "matplotlib is required for plot generation. " + "Install with: pip install matplotlib" + ) + + def generate_all(self, data: ParsedLog) -> list[Path]: + """Generate all plots from parsed data.""" + if not data.update_tip: + logger.warning("No UpdateTip entries found, skipping plot generation") + return [] + + # Verify entries are sorted by time + for i in range(len(data.update_tip) - 1): + if data.update_tip[i].timestamp > data.update_tip[i + 1].timestamp: + logger.warning("UpdateTip entries are not sorted by time") + break + + # Extract base time for elapsed calculations + base_time = data.update_tip[0].timestamp + + # Extract data series + times = [e.timestamp for e in data.update_tip] + heights = [e.height for e in data.update_tip] + tx_counts = [e.tx_count for e in data.update_tip] + cache_sizes = [e.cache_size_mb for e in data.update_tip] + cache_counts = [e.cache_coins_count for e in data.update_tip] + elapsed_minutes = [(t - base_time).total_seconds() / 60 for t in times] + + # Generate core plots + self._plot( + elapsed_minutes, + heights, + "Elapsed minutes", + "Block Height", + "Block Height vs Time", + f"{self.commit}-height_vs_time.png", + ) + + self._plot( + heights, + cache_sizes, + "Block Height", + "Cache Size (MiB)", + "Cache Size vs Block Height", + f"{self.commit}-cache_vs_height.png", + is_height_based=True, + ) + + self._plot( + elapsed_minutes, + cache_sizes, + "Elapsed minutes", + "Cache Size (MiB)", + "Cache Size vs Time", + f"{self.commit}-cache_vs_time.png", + ) + + self._plot( + heights, + tx_counts, + "Block Height", + "Transaction Count", + "Transactions vs Block Height", + f"{self.commit}-tx_vs_height.png", + is_height_based=True, + ) + + self._plot( + heights, + cache_counts, + "Block Height", + "Coins Cache Size", + "Coins Cache Size vs Height", + f"{self.commit}-coins_cache_vs_height.png", + is_height_based=True, + ) + + # LevelDB plots + if data.leveldb_compact: + compact_minutes = [ + (e.timestamp - base_time).total_seconds() / 60 + for e in data.leveldb_compact + ] + self._plot( + compact_minutes, + [1] * len(compact_minutes), + "Elapsed minutes", + "LevelDB Compaction", + "LevelDB Compaction Events vs Time", + f"{self.commit}-leveldb_compact_vs_time.png", + ) + + if data.leveldb_gen_table: + gen_minutes = [ + (e.timestamp - base_time).total_seconds() / 60 + for e in data.leveldb_gen_table + ] + gen_keys = [e.keys_count for e in data.leveldb_gen_table] + gen_bytes = [e.bytes_count for e in data.leveldb_gen_table] + + self._plot( + gen_minutes, + gen_keys, + "Elapsed minutes", + "Number of keys", + "LevelDB Keys Generated vs Time", + f"{self.commit}-leveldb_gen_keys_vs_time.png", + ) + + self._plot( + gen_minutes, + gen_bytes, + "Elapsed minutes", + "Number of bytes", + "LevelDB Bytes Generated vs Time", + f"{self.commit}-leveldb_gen_bytes_vs_time.png", + ) + + # Validation plots + if data.validation_txadd: + txadd_minutes = [ + (e.timestamp - base_time).total_seconds() / 60 + for e in data.validation_txadd + ] + self._plot( + txadd_minutes, + [1] * len(txadd_minutes), + "Elapsed minutes", + "Transaction Additions", + "Transaction Additions to Mempool vs Time", + f"{self.commit}-validation_txadd_vs_time.png", + ) + + # CoinDB plots + if data.coindb_write_batch: + batch_minutes = [ + (e.timestamp - base_time).total_seconds() / 60 + for e in data.coindb_write_batch + ] + batch_sizes = [e.size_mb for e in data.coindb_write_batch] + self._plot( + batch_minutes, + batch_sizes, + "Elapsed minutes", + "Batch Size MiB", + "Coin Database Partial/Final Write Batch Size vs Time", + f"{self.commit}-coindb_write_batch_size_vs_time.png", + ) + + if data.coindb_commit: + commit_minutes = [ + (e.timestamp - base_time).total_seconds() / 60 + for e in data.coindb_commit + ] + commit_txouts = [e.txout_count for e in data.coindb_commit] + self._plot( + commit_minutes, + commit_txouts, + "Elapsed minutes", + "Transaction Output Count", + "Coin Database Transaction Output Committed vs Time", + f"{self.commit}-coindb_commit_txout_vs_time.png", + ) + + return self.generated_plots + + def _plot( + self, + x: list, + y: list, + x_label: str, + y_label: str, + title: str, + filename: str, + is_height_based: bool = False, + ) -> None: + """Generate a single plot.""" + if not x or not y: + logger.debug(f"Skipping plot '{title}' - no data") + return + + plt.figure(figsize=(30, 10)) + plt.plot(x, y) + plt.title(title, fontsize=20) + plt.xlabel(x_label, fontsize=16) + plt.ylabel(y_label, fontsize=16) + plt.grid(True) + + min_x, max_x = min(x), max(x) + plt.xlim(min_x, max_x) + + # Add fork markers for height-based plots + if is_height_based: + self._add_fork_markers(min_x, max_x, max(y)) + + plt.xticks(rotation=90, fontsize=12) + plt.yticks(fontsize=12) + plt.tight_layout() + + output_path = self.output_dir / filename + plt.savefig(output_path) + plt.close() + + self.generated_plots.append(output_path) + logger.info(f"Saved plot: {output_path}") + + def _add_fork_markers(self, min_x: float, max_x: float, max_y: float) -> None: + """Add vertical lines for Bitcoin forks.""" + text_positions = {} + position_increment = max_y * 0.05 + current_position = max_y * 0.9 + + for fork_name, height in FORK_HEIGHTS.items(): + if min_x <= height <= max_x: + plt.axvline( + x=height, + color=FORK_COLORS[fork_name], + linestyle=FORK_STYLES[fork_name], + ) + + if height in text_positions: + text_positions[height] -= position_increment + else: + text_positions[height] = current_position + current_position -= position_increment + if current_position < max_y * 0.1: + current_position = max_y * 0.9 + + plt.text( + height, + text_positions[height], + f"{fork_name} ({height})", + rotation=90, + verticalalignment="top", + color=FORK_COLORS[fork_name], + ) + + +class AnalyzePhase: + """Analyze benchmark results and generate plots.""" + + def __init__(self, config: Config | None = None): + self.config = config + + def run( + self, + commit: str, + log_file: Path, + output_dir: Path, + ) -> AnalyzeResult: + """Analyze a debug.log and generate plots. + + Args: + commit: Commit hash (for naming) + log_file: Path to debug.log + output_dir: Where to save plots + + Returns: + AnalyzeResult with paths to generated plots + """ + if not HAS_MATPLOTLIB: + raise RuntimeError( + "matplotlib is required for plot generation. " + "Install with: pip install matplotlib" + ) + + if not log_file.exists(): + raise FileNotFoundError(f"Log file not found: {log_file}") + + output_dir.mkdir(parents=True, exist_ok=True) + + logger.info(f"Parsing log file: {log_file}") + parser = LogParser() + data = parser.parse_file(log_file) + + logger.info(f"Generating plots for {commit[:12]}") + generator = PlotGenerator(commit[:12], output_dir) + plots = generator.generate_all(data) + + logger.info(f"Generated {len(plots)} plots") + + return AnalyzeResult( + commit=commit, + output_dir=output_dir, + plots=plots, + ) diff --git a/bench/benchmark.py b/bench/benchmark.py new file mode 100644 index 000000000000..40022eeaee24 --- /dev/null +++ b/bench/benchmark.py @@ -0,0 +1,351 @@ +"""Benchmark phase - run hyperfine benchmarks comparing two bitcoind binaries.""" + +from __future__ import annotations + +import logging +import os +import shutil +import subprocess +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from .capabilities import Capabilities + from .config import Config + + +logger = logging.getLogger(__name__) + +# Debug flags for instrumented mode +INSTRUMENTED_DEBUG_FLAGS = ["coindb", "leveldb", "bench", "validation"] + + +@dataclass +class BenchmarkResult: + """Result of the benchmark phase.""" + + results_file: Path + base_commit: str + head_commit: str + instrumented: bool + flamegraph_base: Path | None = None + flamegraph_head: Path | None = None + debug_log_base: Path | None = None + debug_log_head: Path | None = None + + +class BenchmarkPhase: + """Run hyperfine benchmarks comparing two bitcoind binaries.""" + + def __init__( + self, + config: Config, + capabilities: Capabilities, + ): + self.config = config + self.capabilities = capabilities + self._temp_scripts: list[Path] = [] + + def run( + self, + base_commit: str, + head_commit: str, + base_binary: Path, + head_binary: Path, + datadir: Path, + output_dir: Path, + ) -> BenchmarkResult: + """Run benchmarks comparing base and head binaries. + + Args: + base_commit: Git hash of base commit + head_commit: Git hash of head commit + base_binary: Path to base bitcoind binary + head_binary: Path to head bitcoind binary + datadir: Source datadir with blockchain snapshot + output_dir: Where to store results + + Returns: + BenchmarkResult with paths to outputs + """ + # Check prerequisites + errors = self.capabilities.check_for_run(self.config.instrumented) + if errors: + raise RuntimeError("Benchmark prerequisites not met:\n" + "\n".join(errors)) + + # Log warnings about missing optional capabilities + for warning in self.capabilities.get_warnings(): + logger.warning(warning) + + # Setup directories + output_dir.mkdir(parents=True, exist_ok=True) + tmp_datadir = Path(self.config.tmp_datadir) + tmp_datadir.mkdir(parents=True, exist_ok=True) + + results_file = output_dir / "results.json" + + logger.info("Starting benchmark") + logger.info(f" Base: {base_commit[:12]}") + logger.info(f" Head: {head_commit[:12]}") + logger.info(f" Instrumented: {self.config.instrumented}") + logger.info(f" Runs: {self.config.runs}") + logger.info(f" Stop height: {self.config.stop_height}") + logger.info(f" dbcache: {self.config.dbcache}") + + try: + # Create hook scripts for hyperfine + setup_script = self._create_setup_script(tmp_datadir) + prepare_script = self._create_prepare_script(tmp_datadir, datadir) + cleanup_script = self._create_cleanup_script(tmp_datadir) + + # Build hyperfine command + cmd = self._build_hyperfine_cmd( + base_commit=base_commit, + head_commit=head_commit, + base_binary=base_binary, + head_binary=head_binary, + tmp_datadir=tmp_datadir, + results_file=results_file, + setup_script=setup_script, + prepare_script=prepare_script, + cleanup_script=cleanup_script, + output_dir=output_dir, + ) + + if self.config.dry_run: + logger.info(f"[DRY RUN] Would run: {' '.join(cmd)}") + return BenchmarkResult( + results_file=results_file, + base_commit=base_commit, + head_commit=head_commit, + instrumented=self.config.instrumented, + ) + + # Run hyperfine + logger.info("Running hyperfine...") + _result = subprocess.run(cmd, check=True) + + # Collect results + benchmark_result = BenchmarkResult( + results_file=results_file, + base_commit=base_commit, + head_commit=head_commit, + instrumented=self.config.instrumented, + ) + + # For instrumented runs, collect flamegraphs and debug logs + if self.config.instrumented: + base_fg = output_dir / f"{base_commit[:12]}-flamegraph.svg" + head_fg = output_dir / f"{head_commit[:12]}-flamegraph.svg" + base_log = output_dir / f"{base_commit[:12]}-debug.log" + head_log = output_dir / f"{head_commit[:12]}-debug.log" + + # Move flamegraphs from current directory if they exist + for src_name, dest in [ + ("base-flamegraph.svg", base_fg), + ("head-flamegraph.svg", head_fg), + ]: + src = Path(src_name) + if src.exists(): + shutil.move(str(src), str(dest)) + + if base_fg.exists(): + benchmark_result.flamegraph_base = base_fg + if head_fg.exists(): + benchmark_result.flamegraph_head = head_fg + if base_log.exists(): + benchmark_result.debug_log_base = base_log + if head_log.exists(): + benchmark_result.debug_log_head = head_log + + # Clean up tmp_datadir + if tmp_datadir.exists(): + logger.debug(f"Cleaning up tmp_datadir: {tmp_datadir}") + shutil.rmtree(tmp_datadir) + + return benchmark_result + + finally: + # Clean up temp scripts + for script in self._temp_scripts: + if script.exists(): + script.unlink() + self._temp_scripts.clear() + + def _create_temp_script(self, commands: list[str], name: str) -> Path: + """Create a temporary shell script.""" + content = "#!/usr/bin/env bash\nset -euxo pipefail\n" + content += "\n".join(commands) + "\n" + + fd, path = tempfile.mkstemp(suffix=".sh", prefix=f"bench_{name}_") + os.write(fd, content.encode()) + os.close(fd) + os.chmod(path, 0o755) + + script_path = Path(path) + self._temp_scripts.append(script_path) + return script_path + + def _create_setup_script(self, tmp_datadir: Path) -> Path: + """Create setup script (runs once before all timing runs).""" + commands = [ + f'mkdir -p "{tmp_datadir}"', + f'rm -rf "{tmp_datadir}"/*', + ] + return self._create_temp_script(commands, "setup") + + def _create_prepare_script(self, tmp_datadir: Path, original_datadir: Path) -> Path: + """Create prepare script (runs before each timing run).""" + commands = [ + f'rm -rf "{tmp_datadir}"/*', + ] + + # Copy datadir with optional CPU affinity + if self.capabilities.can_pin_cpu and not self.config.no_cpu_pinning: + commands.append( + f'taskset -c 0-15 cp -r "{original_datadir}"/* "{tmp_datadir}"' + ) + else: + commands.append(f'cp -r "{original_datadir}"/* "{tmp_datadir}"') + + # Drop caches if available + if self.capabilities.can_drop_caches and not self.config.no_cache_drop: + commands.append(self.capabilities.drop_caches_path) + + # Clean debug logs + commands.append( + f'find "{tmp_datadir}" -name debug.log -delete 2>/dev/null || true' + ) + + return self._create_temp_script(commands, "prepare") + + def _create_cleanup_script(self, tmp_datadir: Path) -> Path: + """Create cleanup script (runs after all timing runs for each command).""" + commands = [ + f'rm -rf "{tmp_datadir}"/*', + ] + return self._create_temp_script(commands, "cleanup") + + def _build_bitcoind_cmd( + self, + binary: Path, + tmp_datadir: Path, + ) -> str: + """Build the bitcoind command string for hyperfine.""" + parts = [] + + # Add flamegraph wrapper for instrumented mode + if self.config.instrumented: + # Flamegraph runs on core 1 + if self.capabilities.can_pin_cpu and not self.config.no_cpu_pinning: + parts.append("taskset -c 1") + parts.append("flamegraph") + parts.append("--palette bitcoin") + parts.append("--title 'bitcoind IBD'") + parts.append("-c 'record -F 101 --call-graph fp'") + parts.append("--") + + # Add CPU affinity for bitcoind (cores 2-15) + if self.capabilities.can_pin_cpu and not self.config.no_cpu_pinning: + parts.append("taskset -c 2-15") + + # Add scheduler priority + if self.capabilities.can_set_scheduler and not self.config.no_cpu_pinning: + parts.append("chrt -o 0") + + # Bitcoind command + parts.append(str(binary)) + parts.append(f"-datadir={tmp_datadir}") + parts.append(f"-dbcache={self.config.dbcache}") + parts.append(f"-stopatheight={self.config.stop_height}") + parts.append("-prune=10000") + parts.append(f"-chain={self.config.chain}") + parts.append("-daemon=0") + parts.append("-printtoconsole=0") + + if self.config.connect: + parts.append(f"-connect={self.config.connect}") + + # Debug flags for instrumented mode + if self.config.instrumented: + for flag in INSTRUMENTED_DEBUG_FLAGS: + parts.append(f"-debug={flag}") + + return " ".join(parts) + + def _build_hyperfine_cmd( + self, + base_commit: str, + head_commit: str, + base_binary: Path, + head_binary: Path, + tmp_datadir: Path, + results_file: Path, + setup_script: Path, + prepare_script: Path, + cleanup_script: Path, + output_dir: Path, + ) -> list[str]: + """Build the hyperfine command.""" + cmd = [ + "hyperfine", + "--shell=bash", + f"--setup={setup_script}", + f"--prepare={prepare_script}", + f"--cleanup={cleanup_script}", + f"--runs={self.config.runs}", + f"--export-json={results_file}", + "--show-output", + ] + + # For instrumented runs, we need separate conclude scripts per commit + # since hyperfine's parameter substitution doesn't work with --conclude + if self.config.instrumented: + base_conclude = self._create_conclude_script_for_commit( + base_commit[:12], tmp_datadir, output_dir + ) + head_conclude = self._create_conclude_script_for_commit( + head_commit[:12], tmp_datadir, output_dir + ) + # We'll handle conclude differently - see below + + # Command names + cmd.append(f"--command-name=base ({base_commit[:12]})") + cmd.append(f"--command-name=head ({head_commit[:12]})") + + # Build the actual commands to benchmark + base_cmd = self._build_bitcoind_cmd(base_binary, tmp_datadir) + head_cmd = self._build_bitcoind_cmd(head_binary, tmp_datadir) + + # For instrumented runs, append the conclude logic to each command + if self.config.instrumented: + base_cmd += f" && {base_conclude}" + head_cmd += f" && {head_conclude}" + + cmd.append(base_cmd) + cmd.append(head_cmd) + + return cmd + + def _create_conclude_script_for_commit( + self, + commit: str, + tmp_datadir: Path, + output_dir: Path, + ) -> str: + """Create inline conclude commands for a specific commit.""" + # Return shell commands to run after each benchmark + commands = [] + + # Move flamegraph if exists + commands.append(f'if [ -e flamegraph.svg ]; then mv flamegraph.svg "{output_dir}/{commit}-flamegraph.svg"; fi') + + # Copy debug log if exists + commands.append( + f'debug_log=$(find "{tmp_datadir}" -name debug.log -print -quit); ' + f'if [ -n "$debug_log" ]; then cp "$debug_log" "{output_dir}/{commit}-debug.log"; fi' + ) + + return " && ".join(commands) diff --git a/bench/build.py b/bench/build.py new file mode 100644 index 000000000000..feac1b0d5875 --- /dev/null +++ b/bench/build.py @@ -0,0 +1,172 @@ +"""Build phase - compile bitcoind at specified commits.""" + +from __future__ import annotations + +import logging +import shutil +import subprocess +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from .capabilities import Capabilities + from .config import Config + +from .utils import GitState, git_checkout, git_rev_parse + +logger = logging.getLogger(__name__) + + +@dataclass +class BuildResult: + """Result of the build phase.""" + + base_binary: Path + head_binary: Path + base_commit: str + head_commit: str + + +class BuildPhase: + """Build bitcoind binaries at two commits for comparison.""" + + def __init__( + self, + config: Config, + capabilities: Capabilities, + repo_path: Path | None = None, + ): + self.config = config + self.capabilities = capabilities + self.repo_path = repo_path or Path.cwd() + + def run( + self, + base_commit: str, + head_commit: str, + binaries_dir: Path | None = None, + ) -> BuildResult: + """Build bitcoind at both commits. + + Args: + base_commit: Git ref for base (comparison) commit + head_commit: Git ref for head (new) commit + binaries_dir: Where to store binaries (default: ./binaries) + + Returns: + BuildResult with paths to built binaries + """ + # Check prerequisites + errors = self.capabilities.check_for_build() + if errors: + raise RuntimeError("Build prerequisites not met:\n" + "\n".join(errors)) + + binaries_dir = binaries_dir or Path(self.config.binaries_dir) + + # Resolve commits to full hashes + base_hash = git_rev_parse(base_commit, self.repo_path) + head_hash = git_rev_parse(head_commit, self.repo_path) + + logger.info("Building binaries for comparison:") + logger.info(f" Base: {base_hash[:12]} ({base_commit})") + logger.info(f" Head: {head_hash[:12]} ({head_commit})") + + # Setup output directories + base_dir = binaries_dir / "base" + head_dir = binaries_dir / "head" + base_dir.mkdir(parents=True, exist_ok=True) + head_dir.mkdir(parents=True, exist_ok=True) + + base_binary = base_dir / "bitcoind" + head_binary = head_dir / "bitcoind" + + # Check if we can skip existing builds + if self.config.skip_existing: + if base_binary.exists() and head_binary.exists(): + logger.info( + "Both binaries exist and --skip-existing set, skipping build" + ) + return BuildResult( + base_binary=base_binary, + head_binary=head_binary, + base_commit=base_hash, + head_commit=head_hash, + ) + + # Save git state for restoration + git_state = GitState(self.repo_path) + git_state.save() + + try: + # Build both commits + builds = [ + ("base", base_hash, base_binary), + ("head", head_hash, head_binary), + ] + + for name, commit, output_path in builds: + if self.config.skip_existing and output_path.exists(): + logger.info(f"Skipping {name} build - binary exists") + continue + + self._build_commit(name, commit, output_path) + + finally: + # Always restore git state + git_state.restore() + + return BuildResult( + base_binary=base_binary, + head_binary=head_binary, + base_commit=base_hash, + head_commit=head_hash, + ) + + def _build_commit(self, name: str, commit: str, output_path: Path) -> None: + """Build bitcoind for a single commit.""" + logger.info(f"Building {name} ({commit[:12]})") + + if self.config.dry_run: + logger.info(f"[DRY RUN] Would build {commit[:12]} -> {output_path}") + return + + # Checkout the commit + git_checkout(commit, self.repo_path) + + # Build with nix + cmd = [] + + # Add CPU affinity if available + if self.capabilities.can_pin_cpu and not self.config.no_cpu_pinning: + cmd += ["taskset", "-c", "0-15"] + + cmd += ["nix", "build", "-L"] + + logger.debug(f"Running: {' '.join(cmd)}") + result = subprocess.run( + cmd, + cwd=self.repo_path, + ) + + if result.returncode != 0: + raise RuntimeError(f"Build failed for {name} ({commit[:12]})") + + # Copy binary to output location + nix_binary = self.repo_path / "result" / "bin" / "bitcoind" + if not nix_binary.exists(): + raise RuntimeError(f"Built binary not found at {nix_binary}") + + # Remove existing binary if present (may be read-only from nix) + if output_path.exists(): + output_path.chmod(0o755) + output_path.unlink() + + shutil.copy2(nix_binary, output_path) + output_path.chmod(0o755) # Ensure it's executable and writable + logger.info(f"Built {name} binary: {output_path}") + + # Clean up nix result symlink + result_link = self.repo_path / "result" + if result_link.is_symlink(): + result_link.unlink() diff --git a/bench/capabilities.py b/bench/capabilities.py new file mode 100644 index 000000000000..69ef2d1185c3 --- /dev/null +++ b/bench/capabilities.py @@ -0,0 +1,162 @@ +"""System capability detection for graceful degradation. + +Detects available tools and features, allowing the benchmark to run +on systems without all capabilities (with appropriate warnings). +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +from dataclasses import dataclass +from pathlib import Path + + +# Known paths for drop-caches on NixOS +DROP_CACHES_PATHS = [ + "/run/wrappers/bin/drop-caches", + "/usr/local/bin/drop-caches", +] + + +@dataclass +class Capabilities: + """Detected system capabilities.""" + + # Cache management + can_drop_caches: bool + drop_caches_path: str | None + + # CPU affinity and scheduling + can_pin_cpu: bool + can_set_scheduler: bool + + # Required tools + has_hyperfine: bool + has_flamegraph: bool + has_perf: bool + has_nix: bool + + # System info + cpu_count: int + is_nixos: bool + is_ci: bool + + def check_for_run(self, instrumented: bool = False) -> list[str]: + """Check if we have required capabilities for a benchmark run. + + Returns list of errors (empty if all good). + """ + errors = [] + + if not self.has_hyperfine: + errors.append("hyperfine not found in PATH (required for benchmarking)") + + if instrumented: + if not self.has_flamegraph: + errors.append( + "flamegraph not found in PATH (required for --instrumented)" + ) + if not self.has_perf: + errors.append("perf not found in PATH (required for --instrumented)") + + return errors + + def check_for_build(self) -> list[str]: + """Check if we have required capabilities for building. + + Returns list of errors (empty if all good). + """ + errors = [] + + if not self.has_nix: + errors.append("nix not found in PATH (required for building)") + + return errors + + def get_warnings(self) -> list[str]: + """Get warnings about missing optional capabilities.""" + warnings = [] + + if not self.can_drop_caches: + warnings.append( + "drop-caches not available - cache won't be cleared between runs" + ) + + if not self.can_pin_cpu: + warnings.append("taskset not available - CPU affinity won't be set") + + if not self.can_set_scheduler: + warnings.append("chrt not available - scheduler priority won't be set") + + return warnings + + +def _check_executable(name: str) -> bool: + """Check if an executable is available in PATH.""" + return shutil.which(name) is not None + + +def _find_drop_caches() -> str | None: + """Find drop-caches executable.""" + for path in DROP_CACHES_PATHS: + if Path(path).exists() and os.access(path, os.X_OK): + return path + return None + + +def _check_taskset() -> bool: + """Check if taskset is available and works.""" + if not _check_executable("taskset"): + return False + + # Try to run it to verify it works + try: + result = subprocess.run( + ["taskset", "-c", "0", "true"], + timeout=5, + ) + return result.returncode == 0 + except (subprocess.TimeoutExpired, OSError): + return False + + +def _check_chrt() -> bool: + """Check if chrt is available and works.""" + if not _check_executable("chrt"): + return False + + # Try to run it to verify it works + try: + result = subprocess.run( + ["chrt", "-o", "0", "true"], + timeout=5, + ) + return result.returncode == 0 + except (subprocess.TimeoutExpired, OSError): + return False + + +def _is_nixos() -> bool: + """Check if we're running on NixOS.""" + return Path("/etc/NIXOS").exists() + + +def detect_capabilities() -> Capabilities: + """Auto-detect system capabilities.""" + drop_caches_path = _find_drop_caches() + + return Capabilities( + can_drop_caches=drop_caches_path is not None, + drop_caches_path=drop_caches_path, + can_pin_cpu=_check_taskset(), + can_set_scheduler=_check_chrt(), + has_hyperfine=_check_executable("hyperfine"), + has_flamegraph=_check_executable("flamegraph"), + has_perf=_check_executable("perf"), + has_nix=_check_executable("nix"), + cpu_count=os.cpu_count() or 1, + is_nixos=_is_nixos(), + is_ci=os.environ.get("CI", "").lower() in ("true", "1", "yes"), + ) diff --git a/bench/config.py b/bench/config.py new file mode 100644 index 000000000000..2e0ea2c0967d --- /dev/null +++ b/bench/config.py @@ -0,0 +1,208 @@ +"""Configuration management for benchcoin. + +Layered configuration (lowest to highest priority): +1. Built-in defaults +2. bench.toml config file +3. Environment variables (BENCH_*) +4. CLI arguments +""" + +from __future__ import annotations + +import os +import tomllib +from dataclasses import dataclass +from pathlib import Path +from typing import Any + + +# Built-in defaults +DEFAULTS = { + "chain": "main", + "dbcache": 450, + "stop_height": 855000, + "runs": 3, + "connect": "", # Empty = use public P2P network + "binaries_dir": "./binaries", + "output_dir": "./bench-output", +} + +# Profile overrides +PROFILES = { + "quick": { + "stop_height": 1500, + "runs": 1, + }, + "full": { + "stop_height": 855000, + "runs": 3, + }, + "ci": { + "stop_height": 855000, + "runs": 3, + }, +} + +# Environment variable mapping +ENV_MAPPING = { + "BENCH_DATADIR": "datadir", + "BENCH_TMP_DATADIR": "tmp_datadir", + "BENCH_BINARIES_DIR": "binaries_dir", + "BENCH_OUTPUT_DIR": "output_dir", + "BENCH_STOP_HEIGHT": "stop_height", + "BENCH_DBCACHE": "dbcache", + "BENCH_CONNECT": "connect", + "BENCH_RUNS": "runs", + "BENCH_CHAIN": "chain", +} + + +@dataclass +class Config: + """Benchmark configuration.""" + + # Core benchmark settings + chain: str = "main" + dbcache: int = 450 + stop_height: int = 855000 + runs: int = 3 + connect: str = "" # Empty = use public P2P network + + # Paths + datadir: str | None = None + tmp_datadir: str | None = None + binaries_dir: str = "./binaries" + output_dir: str = "./bench-output" + + # Behavior flags + instrumented: bool = False + skip_existing: bool = False + no_cpu_pinning: bool = False + no_cache_drop: bool = False + verbose: bool = False + dry_run: bool = False + + # Profile used (for reference) + profile: str = "full" + + def __post_init__(self) -> None: + # If tmp_datadir not set, derive from output_dir + if self.tmp_datadir is None: + self.tmp_datadir = str(Path(self.output_dir) / "tmp-datadir") + + # Instrumented mode forces runs=1 + if self.instrumented and self.runs != 1: + self.runs = 1 + + def validate(self) -> list[str]: + """Validate configuration, return list of errors.""" + errors = [] + + if self.datadir is None: + errors.append("--datadir is required") + elif not Path(self.datadir).exists(): + errors.append(f"datadir does not exist: {self.datadir}") + + if self.stop_height < 1: + errors.append("stop_height must be positive") + + if self.dbcache < 1: + errors.append("dbcache must be positive") + + if self.runs < 1: + errors.append("runs must be positive") + + if self.chain not in ("main", "testnet", "signet", "regtest"): + errors.append(f"invalid chain: {self.chain}") + + return errors + + +def load_toml(path: Path) -> dict[str, Any]: + """Load configuration from TOML file.""" + if not path.exists(): + return {} + + with open(path, "rb") as f: + data = tomllib.load(f) + + # Flatten structure: merge [defaults] and [paths] into top level + result = {} + if "defaults" in data: + result.update(data["defaults"]) + if "paths" in data: + result.update(data["paths"]) + + return result + + +def load_env() -> dict[str, Any]: + """Load configuration from environment variables.""" + result = {} + + for env_var, config_key in ENV_MAPPING.items(): + value = os.environ.get(env_var) + if value is not None: + # Convert numeric values + if config_key in ("stop_height", "dbcache", "runs"): + try: + value = int(value) + except ValueError: + pass # Keep as string, will fail validation + result[config_key] = value + + return result + + +def apply_profile(config: dict[str, Any], profile_name: str) -> dict[str, Any]: + """Apply a named profile to configuration.""" + if profile_name not in PROFILES: + return config + + result = config.copy() + result.update(PROFILES[profile_name]) + result["profile"] = profile_name + return result + + +def build_config( + cli_args: dict[str, Any] | None = None, + config_file: Path | None = None, + profile: str = "full", +) -> Config: + """Build configuration from all sources. + + Priority (lowest to highest): + 1. Built-in defaults + 2. Config file (bench.toml) + 3. Profile overrides + 4. Environment variables + 5. CLI arguments + """ + # Start with defaults + config = DEFAULTS.copy() + + # Load config file + if config_file is None: + config_file = Path("bench.toml") + file_config = load_toml(config_file) + config.update(file_config) + + # Apply profile + config = apply_profile(config, profile) + + # Load environment variables + env_config = load_env() + config.update(env_config) + + # Apply CLI arguments (filter out None values) + if cli_args: + for key, value in cli_args.items(): + if value is not None: + config[key] = value + + # Build Config object (filter to only valid fields) + valid_fields = {f.name for f in Config.__dataclass_fields__.values()} + filtered = {k: v for k, v in config.items() if k in valid_fields} + + return Config(**filtered) diff --git a/bench/report.py b/bench/report.py new file mode 100644 index 000000000000..a41a035fbe9a --- /dev/null +++ b/bench/report.py @@ -0,0 +1,453 @@ +"""Report phase - generate HTML reports from benchmark results. + +Ported from the JavaScript logic in .github/workflows/publish-results.yml. +""" + +from __future__ import annotations + +import json +import logging +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +logger = logging.getLogger(__name__) + +# HTML template for individual run report +RUN_REPORT_TEMPLATE = """ + + + Benchmark Results + + + +
+

Benchmark Results

+
+

{title}

+ + +

Run Data

+
+ + + + + + + + + + + + + {run_data_rows} + +
NetworkCommandMean (s)Std DevUser (s)System (s)
+
+ + +

Speedup Summary

+
+ + + + + + + + + {speedup_rows} + +
NetworkSpeedup (%)
+
+ + + {graphs_section} +
+
+ +""" + +# HTML template for main index +INDEX_TEMPLATE = """ + + + Bitcoin Benchmark Results + + + +
+

Bitcoin Benchmark Results

+
+

Available Results

+
    + {run_list} +
+
+
+ +""" + + +@dataclass +class BenchmarkRun: + """Parsed benchmark run data.""" + + network: str + command: str + mean: float + stddev: float | None + user: float + system: float + parameters: dict[str, Any] = field(default_factory=dict) + + +@dataclass +class ReportResult: + """Result of report generation.""" + + output_dir: Path + index_file: Path + speedups: dict[str, float] + + +class ReportGenerator: + """Generate HTML reports from benchmark results.""" + + def __init__( + self, repo_url: str = "https://github.com/bitcoin-dev-tools/benchcoin" + ): + self.repo_url = repo_url + + def generate( + self, + input_dir: Path, + output_dir: Path, + title: str = "Benchmark Results", + ) -> ReportResult: + """Generate HTML report from benchmark artifacts. + + Args: + input_dir: Directory containing results.json and artifacts + output_dir: Where to write the HTML report + title: Title for the report + + Returns: + ReportResult with paths and speedup data + """ + output_dir.mkdir(parents=True, exist_ok=True) + + # Load results.json + results_file = input_dir / "results.json" + if not results_file.exists(): + raise FileNotFoundError(f"results.json not found in {input_dir}") + + with open(results_file) as f: + data = json.load(f) + + # Parse results + runs = self._parse_results(data) + + # Calculate speedups + speedups = self._calculate_speedups(runs) + + # Generate HTML + html = self._generate_html(runs, speedups, title, input_dir, output_dir) + + # Write report + index_file = output_dir / "index.html" + index_file.write_text(html) + logger.info(f"Generated report: {index_file}") + + # Copy artifacts (flamegraphs, plots) + self._copy_artifacts(input_dir, output_dir) + + return ReportResult( + output_dir=output_dir, + index_file=index_file, + speedups=speedups, + ) + + def generate_index( + self, + results_dir: Path, + output_file: Path, + ) -> None: + """Generate main index.html listing all available results. + + Args: + results_dir: Directory containing pr-* subdirectories + output_file: Where to write index.html + """ + runs = [] + + if results_dir.exists(): + for pr_dir in sorted(results_dir.iterdir()): + if pr_dir.is_dir() and pr_dir.name.startswith("pr-"): + pr_num = pr_dir.name.replace("pr-", "") + pr_runs = [] + for run_dir in sorted(pr_dir.iterdir()): + if run_dir.is_dir(): + pr_runs.append(run_dir.name) + if pr_runs: + runs.append((pr_num, pr_runs)) + + run_list_html = "" + for pr_num, pr_runs in runs: + run_links = "\n".join( + f'
  • Run {run}
    • ' + for run in pr_runs + ) + run_list_html += f""" +
  • PR #{pr_num} +
      + {run_links} +
    +
    • + """ + + html = INDEX_TEMPLATE.format(run_list=run_list_html) + output_file.write_text(html) + logger.info(f"Generated index: {output_file}") + + def _parse_results(self, data: dict) -> list[BenchmarkRun]: + """Parse results from hyperfine JSON output.""" + runs = [] + + # Handle both direct hyperfine output and combined results format + results = data.get("results", []) + + for result in results: + runs.append( + BenchmarkRun( + network=result.get("network", "default"), + command=result.get("command", ""), + mean=result.get("mean", 0), + stddev=result.get("stddev"), + user=result.get("user", 0), + system=result.get("system", 0), + parameters=result.get("parameters", {}), + ) + ) + + return runs + + def _calculate_speedups(self, runs: list[BenchmarkRun]) -> dict[str, float]: + """Calculate speedup percentages for each network.""" + speedups = {} + + # Group by network + by_network: dict[str, list[BenchmarkRun]] = {} + for run in runs: + if run.network not in by_network: + by_network[run.network] = [] + by_network[run.network].append(run) + + # Calculate speedup for each network + for network, network_runs in by_network.items(): + base_mean = None + head_mean = None + + for run in network_runs: + if "base" in run.command.lower(): + base_mean = run.mean + elif "head" in run.command.lower(): + head_mean = run.mean + + if base_mean and head_mean and base_mean > 0: + speedup = ((base_mean - head_mean) / base_mean) * 100 + speedups[network] = round(speedup, 1) + + return speedups + + def _generate_html( + self, + runs: list[BenchmarkRun], + speedups: dict[str, float], + title: str, + input_dir: Path, + output_dir: Path, + ) -> str: + """Generate the HTML report.""" + # Sort runs by network then by command (base first) + sorted_runs = sorted( + runs, + key=lambda r: (r.network, 0 if "base" in r.command.lower() else 1), + ) + + # Generate run data rows + run_data_rows = "" + for run in sorted_runs: + # Create commit link if there's a commit hash in the command + command_html = self._linkify_commit(run.command) + + stddev_str = f"{run.stddev:.3f}" if run.stddev else "N/A" + + run_data_rows += f""" + + {run.network} + {command_html} + {run.mean:.3f} + {stddev_str} + {run.user:.3f} + {run.system:.3f} + + """ + + # Generate speedup rows + speedup_rows = "" + for network, speedup in speedups.items(): + color_class = "" + if speedup > 0: + color_class = "text-green-600" + elif speedup < 0: + color_class = "text-red-600" + + speedup_rows += f""" + + {network} + {speedup}% + + """ + + # Generate graphs section + graphs_section = self._generate_graphs_section(runs, input_dir, output_dir) + + return RUN_REPORT_TEMPLATE.format( + title=title, + run_data_rows=run_data_rows, + speedup_rows=speedup_rows, + graphs_section=graphs_section, + ) + + def _linkify_commit(self, command: str) -> str: + """Convert commit hashes in command to links.""" + import re + + def replace_commit(match): + commit = match.group(1) + short_commit = commit[:8] if len(commit) > 8 else commit + return f'({short_commit})' + + return re.sub(r"\(([a-f0-9]{7,40})\)", replace_commit, command) + + def _generate_graphs_section( + self, + runs: list[BenchmarkRun], + input_dir: Path, + output_dir: Path, + ) -> str: + """Generate the flamegraphs and plots section.""" + graphs_html = "" + + for run in runs: + commit = run.parameters.get("commit", "") + if not commit: + # Try to extract from command + import re + + match = re.search(r"\(([a-f0-9]+)\)", run.command) + if match: + commit = match.group(1) + + if not commit: + continue + + short_commit = commit[:12] if len(commit) > 12 else commit + + # Check for flamegraph + flamegraph_name = f"{short_commit}-flamegraph.svg" + flamegraph_path = input_dir / flamegraph_name + + # Check for plots + plots_dir = input_dir / "plots" + plot_files = [] + if plots_dir.exists(): + plot_files = [ + p.name + for p in plots_dir.iterdir() + if p.name.startswith(f"{short_commit}-") and p.suffix == ".png" + ] + + if not flamegraph_path.exists() and not plot_files: + continue + + graphs_html += f""" +
    +

    {run.command}

    + """ + + if flamegraph_path.exists(): + graphs_html += f""" + + """ + + for plot in sorted(plot_files): + graphs_html += f""" + + {plot} + + """ + + graphs_html += "
    " + + if graphs_html: + return f""" +

    Flamegraphs and Plots

    + {graphs_html} + """ + + return "" + + def _copy_artifacts(self, input_dir: Path, output_dir: Path) -> None: + """Copy flamegraphs and plots to output directory.""" + import shutil + + # Skip if input and output are the same directory + if input_dir.resolve() == output_dir.resolve(): + logger.debug("Input and output are the same directory, skipping copy") + return + + # Copy flamegraphs + for svg in input_dir.glob("*-flamegraph.svg"): + dest = output_dir / svg.name + shutil.copy2(svg, dest) + logger.debug(f"Copied {svg.name}") + + # Copy plots directory + plots_dir = input_dir / "plots" + if plots_dir.exists(): + dest_plots = output_dir / "plots" + if dest_plots.exists(): + shutil.rmtree(dest_plots) + shutil.copytree(plots_dir, dest_plots) + logger.debug("Copied plots directory") + + +class ReportPhase: + """Generate reports from benchmark results.""" + + def __init__( + self, repo_url: str = "https://github.com/bitcoin-dev-tools/benchcoin" + ): + self.generator = ReportGenerator(repo_url) + + def run( + self, + input_dir: Path, + output_dir: Path, + title: str = "Benchmark Results", + ) -> ReportResult: + """Generate report from benchmark artifacts. + + Args: + input_dir: Directory containing results.json and artifacts + output_dir: Where to write the HTML report + title: Title for the report + + Returns: + ReportResult with paths and speedup data + """ + return self.generator.generate(input_dir, output_dir, title) diff --git a/bench/utils.py b/bench/utils.py new file mode 100644 index 000000000000..eda158833ebc --- /dev/null +++ b/bench/utils.py @@ -0,0 +1,259 @@ +"""Utility functions for git, datadir, and system operations.""" + +from __future__ import annotations + +import logging +import os +import shutil +import subprocess +import tempfile +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from .capabilities import Capabilities + from .config import Config + +logger = logging.getLogger(__name__) + + +class GitError(Exception): + """Git operation failed.""" + + pass + + +class GitState: + """Saved git state for restoration after operations.""" + + def __init__(self, repo_path: Path | None = None): + self.repo_path = repo_path or Path.cwd() + self.original_branch: str | None = None + self.original_commit: str | None = None + self.was_detached: bool = False + + def save(self) -> None: + """Save current git state.""" + # Check if we're on a branch or detached HEAD + result = subprocess.run( + ["git", "symbolic-ref", "--short", "HEAD"], + capture_output=True, + text=True, + cwd=self.repo_path, + ) + + if result.returncode == 0: + self.original_branch = result.stdout.strip() + self.was_detached = False + else: + # Detached HEAD - save commit hash + result = subprocess.run( + ["git", "rev-parse", "HEAD"], + capture_output=True, + text=True, + check=True, + cwd=self.repo_path, + ) + self.original_commit = result.stdout.strip() + self.was_detached = True + + logger.debug( + f"Saved git state: branch={self.original_branch}, " + f"commit={self.original_commit}, detached={self.was_detached}" + ) + + def restore(self) -> None: + """Restore saved git state.""" + if self.original_branch: + logger.debug(f"Restoring branch: {self.original_branch}") + subprocess.run( + ["git", "checkout", self.original_branch], + check=True, + cwd=self.repo_path, + ) + elif self.original_commit: + logger.debug(f"Restoring detached HEAD: {self.original_commit}") + subprocess.run( + ["git", "checkout", self.original_commit], + check=True, + cwd=self.repo_path, + ) + + +def git_checkout(commit: str, repo_path: Path | None = None) -> None: + """Checkout a specific commit.""" + repo_path = repo_path or Path.cwd() + logger.info(f"Checking out {commit[:12]}") + + result = subprocess.run( + ["git", "checkout", commit], + cwd=repo_path, + capture_output=True, + text=True, + ) + + if result.returncode != 0: + raise GitError(f"Failed to checkout {commit}: {result.stderr}") + + +def git_rev_parse(ref: str, repo_path: Path | None = None) -> str: + """Resolve a git reference to a full commit hash.""" + repo_path = repo_path or Path.cwd() + + result = subprocess.run( + ["git", "rev-parse", ref], + cwd=repo_path, + capture_output=True, + text=True, + ) + + if result.returncode != 0: + raise GitError(f"Failed to resolve {ref}: {result.stderr}") + + return result.stdout.strip() + + +def clean_datadir(datadir: Path) -> None: + """Remove all contents from a data directory.""" + if not datadir.exists(): + return + + logger.debug(f"Cleaning datadir: {datadir}") + for item in datadir.iterdir(): + if item.is_dir(): + shutil.rmtree(item) + else: + item.unlink() + + +def copy_datadir(src: Path, dst: Path, capabilities: Capabilities) -> None: + """Copy blockchain data from source to destination. + + Uses taskset for CPU affinity if available. + """ + logger.info(f"Copying datadir: {src} -> {dst}") + + # Ensure destination exists + dst.mkdir(parents=True, exist_ok=True) + + # Build copy command + cmd = [] + if capabilities.can_pin_cpu: + cmd += ["taskset", "-c", "0-15"] + + cmd += ["cp", "-r"] + # Copy contents, not directory itself + cmd += [str(src) + "/.", str(dst)] + + subprocess.run(cmd, check=True) + + +def drop_caches(capabilities: Capabilities) -> bool: + """Drop filesystem caches if available. + + Returns True if caches were dropped, False if not available. + """ + if not capabilities.can_drop_caches or not capabilities.drop_caches_path: + logger.debug("Cache dropping not available, skipping") + return False + + logger.debug("Dropping filesystem caches") + subprocess.run([capabilities.drop_caches_path], check=True) + return True + + +def clean_debug_logs(datadir: Path) -> None: + """Remove debug.log files from datadir and subdirectories.""" + logger.debug(f"Cleaning debug logs in: {datadir}") + + for log_file in datadir.rglob("debug.log"): + log_file.unlink() + + +def find_debug_log(datadir: Path) -> Path | None: + """Find debug.log in datadir or subdirectories.""" + # Check common locations + candidates = [ + datadir / "debug.log", + datadir / "mainnet" / "debug.log", + datadir / "testnet3" / "debug.log", + datadir / "signet" / "debug.log", + datadir / "regtest" / "debug.log", + ] + + for candidate in candidates: + if candidate.exists(): + return candidate + + # Fallback: search recursively + for log_file in datadir.rglob("debug.log"): + return log_file + + return None + + +def create_temp_script(commands: list[str], name: str = "hook") -> Path: + """Create a temporary shell script for hyperfine hooks. + + Returns path to the script. + """ + script_content = "#!/usr/bin/env bash\nset -euxo pipefail\n" + script_content += "\n".join(commands) + "\n" + + # Create temp file that persists (caller is responsible for cleanup) + fd, path = tempfile.mkstemp(suffix=".sh", prefix=f"bench_{name}_") + os.write(fd, script_content.encode()) + os.close(fd) + os.chmod(path, 0o755) + + return Path(path) + + +def build_bitcoind_cmd( + binary: Path, + datadir: Path, + config: Config, + capabilities: Capabilities, + debug_flags: list[str] | None = None, +) -> list[str]: + """Build the bitcoind command with optional wrappers. + + Args: + binary: Path to bitcoind binary + datadir: Data directory + config: Benchmark configuration + capabilities: System capabilities + debug_flags: Optional debug flags for instrumented mode + """ + cmd = [] + + # Add CPU affinity if available and not disabled + if capabilities.can_pin_cpu and not config.no_cpu_pinning: + cmd += ["taskset", "-c", "2-15"] + + # Add scheduler priority if available + if capabilities.can_set_scheduler and not config.no_cpu_pinning: + cmd += ["chrt", "-o", "0"] + + # Add bitcoind with options + cmd += [ + str(binary), + f"-datadir={datadir}", + f"-dbcache={config.dbcache}", + f"-stopatheight={config.stop_height}", + "-prune=10000", + f"-chain={config.chain}", + "-daemon=0", + "-printtoconsole=0", + ] + + # Add connect address if specified + if config.connect: + cmd.append(f"-connect={config.connect}") + + # Add debug flags for instrumented mode + if debug_flags: + for flag in debug_flags: + cmd.append(f"-debug={flag}") + + return cmd diff --git a/flake.nix b/flake.nix index 3d0c2fa657b4..b42180629d1a 100644 --- a/flake.nix +++ b/flake.nix @@ -30,16 +30,16 @@ ]; CXXFlags = "${CFlags} -fno-omit-frame-pointer"; - nativeBuildInputs = with pkgs; [ - cmake - ninja - pkg-config - python3 + nativeBuildInputs = [ + pkgs.cmake + pkgs.ninja + pkgs.pkg-config + pkgs.python3 ]; - buildInputs = with pkgs; [ - boost188.dev - libevent.dev + buildInputs = [ + pkgs.boost188.dev + pkgs.libevent.dev ]; cmakeFlags = [ @@ -156,6 +156,8 @@ pkgs.just pkgs.perf pkgs.perf-tools + pkgs.python312 + pkgs.python312Packages.matplotlib pkgs.util-linux # Binary patching diff --git a/justfile b/justfile index 5b32a5d7bf85..51faef3160da 100644 --- a/justfile +++ b/justfile @@ -1,34 +1,85 @@ set shell := ["bash", "-uc"] -os := os() - default: just --list -# Build base and head binaries for CI +# ============================================================================ +# Local benchmarking commands +# ============================================================================ + +# Test instrumented run using signet (includes report generation) +[group('local')] +test-instrumented base head datadir: + nix develop --command python3 bench.py --profile quick full --chain signet --instrumented --skip-existing --datadir {{ datadir }} {{ base }} {{ head }} + nix develop --command python3 bench.py report bench-output/ bench-output/ + +# Test uninstrumented run using signet +[group('local')] +test-uninstrumented base head datadir: + nix develop --command python3 bench.py --profile quick full --chain signet --skip-existing --datadir {{ datadir }} {{ base }} {{ head }} + +# Full benchmark with instrumentation (flamegraphs + plots) +[group('local')] +instrumented base head datadir: + python3 bench.py --profile quick full --instrumented --datadir {{ datadir }} {{ base }} {{ head }} + +# Just build binaries (useful for incremental testing) +[group('local')] +build base head: + python3 bench.py build {{ base }} {{ head }} + +# Run benchmark with pre-built binaries +[group('local')] +run base head datadir: + python3 bench.py run --datadir {{ datadir }} {{ base }} {{ head }} + +# Generate plots from a debug.log file +[group('local')] +analyze commit logfile output_dir="./plots": + python3 bench.py analyze {{ commit }} {{ logfile }} --output-dir {{ output_dir }} + +# Generate HTML report from benchmark results +[group('local')] +report input_dir output_dir: + python3 bench.py report {{ input_dir }} {{ output_dir }} + +# ============================================================================ +# CI commands (called by GitHub Actions) +# ============================================================================ + +# Build binaries for CI [group('ci')] -build-binaries base_commit head_commit: - #!/usr/bin/env bash - set -euxo pipefail - ./bench-ci/build_binaries.sh {{ base_commit }} {{ head_commit }} +ci-build base_commit head_commit binaries_dir: + python3 bench.py build --binaries-dir {{ binaries_dir }} {{ base_commit }} {{ head_commit }} -# Run uninstrumented benchmarks on mainnet +# Run uninstrumented benchmarks for CI [group('ci')] -run-mainnet-ci base_commit head_commit TMP_DATADIR ORIGINAL_DATADIR results_file dbcache binaries_dir: - #!/usr/bin/env bash - set -euxo pipefail - unset SOURCE_DATE_EPOCH # needed to run on NixOS - ./bench-ci/run-benchmark.sh {{ base_commit }} {{ head_commit }} {{ TMP_DATADIR }} {{ ORIGINAL_DATADIR }} {{ results_file }} main 855000 "148.251.128.115:33333" {{ dbcache }} {{ binaries_dir }} +ci-run base_commit head_commit datadir tmp_datadir output_dir dbcache binaries_dir: + python3 bench.py --profile ci run \ + --binaries-dir {{ binaries_dir }} \ + --datadir {{ datadir }} \ + --tmp-datadir {{ tmp_datadir }} \ + --output-dir {{ output_dir }} \ + --dbcache {{ dbcache }} \ + {{ base_commit }} {{ head_commit }} -# Run instrumented benchmarks on mainnet +# Run instrumented benchmarks for CI [group('ci')] -run-mainnet-ci-instrumented base_commit head_commit TMP_DATADIR ORIGINAL_DATADIR results_file dbcache png_dir binaries_dir: - #!/usr/bin/env bash - set -euxo pipefail - unset SOURCE_DATE_EPOCH # needed to run on NixOS - ./bench-ci/run-benchmark-instrumented.sh {{ base_commit }} {{ head_commit }} {{ TMP_DATADIR }} {{ ORIGINAL_DATADIR }} {{ results_file }} {{ png_dir }} main 855000 "148.251.128.115:33333" {{ dbcache }} {{ binaries_dir }} +ci-run-instrumented base_commit head_commit datadir tmp_datadir output_dir dbcache binaries_dir: + python3 bench.py --profile ci run \ + --instrumented \ + --binaries-dir {{ binaries_dir }} \ + --datadir {{ datadir }} \ + --tmp-datadir {{ tmp_datadir }} \ + --output-dir {{ output_dir }} \ + --dbcache {{ dbcache }} \ + {{ base_commit }} {{ head_commit }} + +# ============================================================================ +# Git helpers +# ============================================================================ -# Cherry-pick commits from a bitcoin core PR onto this branch +# Cherry-pick commits from a Bitcoin Core PR onto this branch [group('git')] pick-pr pr_number: #!/usr/bin/env bash @@ -36,7 +87,7 @@ pick-pr pr_number: if ! git remote get-url upstream 2>/dev/null | grep -q "bitcoin/bitcoin"; then echo "Error: 'upstream' remote not found or doesn't point to bitcoin/bitcoin" - echo "Please add it with: `git remote add upstream https://github.com/bitcoin/bitcoin.git`" + echo "Please add it with: git remote add upstream https://github.com/bitcoin/bitcoin.git" exit 1 fi From 6dc81c1a26c42c5926bc4a0b4f62dc5bf6ffe55c Mon Sep 17 00:00:00 2001 From: will Date: Mon, 8 Dec 2025 14:53:05 +0000 Subject: [PATCH 31/48] fixup! use python runner --- .github/workflows/benchmark.yml | 6 ++---- bench.py | 30 +++++++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 46aa545e8cf5..d9861a0644f7 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -72,8 +72,7 @@ jobs: - name: Run benchmark run: | - nix develop --command python3 bench.py run \ - --profile ci \ + nix develop --command python3 bench.py --profile ci run \ --binaries-dir ${{ runner.temp }}/binaries \ --datadir $ORIGINAL_DATADIR \ --tmp-datadir ${{ runner.temp }}/datadir \ @@ -142,8 +141,7 @@ jobs: - name: Run instrumented benchmark run: | - nix develop --command python3 bench.py run \ - --profile ci \ + nix develop --command python3 bench.py --profile ci run \ --instrumented \ --binaries-dir ${{ runner.temp }}/binaries \ --datadir $ORIGINAL_DATADIR \ diff --git a/bench.py b/bench.py index 072f4ef95fc4..df2dd6108703 100755 --- a/bench.py +++ b/bench.py @@ -120,6 +120,7 @@ def cmd_run(args: argparse.Namespace) -> int: return 1 phase = BenchmarkPhase(config, capabilities) + output_dir = Path(config.output_dir) try: result = phase.run( @@ -128,9 +129,36 @@ def cmd_run(args: argparse.Namespace) -> int: base_binary=base_binary, head_binary=head_binary, datadir=Path(config.datadir), - output_dir=Path(config.output_dir), + output_dir=output_dir, ) logger.info(f"Results saved to: {result.results_file}") + + # For instrumented runs, also generate plots + if config.instrumented: + from bench.analyze import AnalyzePhase + + analyze_phase = AnalyzePhase(config) + + if result.debug_log_base: + try: + analyze_phase.run( + commit=args.base_commit, + log_file=result.debug_log_base, + output_dir=output_dir / "plots", + ) + except Exception as e: + logger.warning(f"Analysis for base failed: {e}") + + if result.debug_log_head: + try: + analyze_phase.run( + commit=args.head_commit, + log_file=result.debug_log_head, + output_dir=output_dir / "plots", + ) + except Exception as e: + logger.warning(f"Analysis for head failed: {e}") + return 0 except Exception as e: logger.error(f"Benchmark failed: {e}") From dd2a4c55274578f6f835552e0f48ebcbabb3dcd1 Mon Sep 17 00:00:00 2001 From: will Date: Mon, 8 Dec 2025 14:53:36 +0000 Subject: [PATCH 32/48] fixup! use python runner --- .github/workflows/publish-results.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/publish-results.yml b/.github/workflows/publish-results.yml index 11d0327192c5..a785374b4308 100644 --- a/.github/workflows/publish-results.yml +++ b/.github/workflows/publish-results.yml @@ -101,6 +101,11 @@ jobs: // Add network name to each result and collect means networkResults.results.forEach(result => { result.network = network; + // Extract commit from command string like "base (364a7bb8701e)" + const commitMatch = result.command.match(/\(([a-f0-9]+)\)/); + if (commitMatch) { + result.parameters = { commit: commitMatch[1] }; + } combinedResults.results.push(result); if (result.command.includes('base')) { baseMean = result.mean; From a096e429a01300589b328e7034e13d201b354122 Mon Sep 17 00:00:00 2001 From: will Date: Mon, 8 Dec 2025 15:20:26 +0000 Subject: [PATCH 33/48] fixup! use python runner --- bench/build.py | 9 +++------ bench/config.py | 1 + 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/bench/build.py b/bench/build.py index feac1b0d5875..b03093f8843a 100644 --- a/bench/build.py +++ b/bench/build.py @@ -134,13 +134,10 @@ def _build_commit(self, name: str, commit: str, output_path: Path) -> None: # Checkout the commit git_checkout(commit, self.repo_path) - # Build with nix + # Build with nix (use all available cores for faster builds) cmd = [] - - # Add CPU affinity if available - if self.capabilities.can_pin_cpu and not self.config.no_cpu_pinning: - cmd += ["taskset", "-c", "0-15"] - + if self.capabilities.can_pin_cpu: + cmd += ["taskset", "-c", f"2-{self.capabilities.cpu_count - 1}"] cmd += ["nix", "build", "-L"] logger.debug(f"Running: {' '.join(cmd)}") diff --git a/bench/config.py b/bench/config.py index 2e0ea2c0967d..a0e2926986fe 100644 --- a/bench/config.py +++ b/bench/config.py @@ -40,6 +40,7 @@ "ci": { "stop_height": 855000, "runs": 3, + "connect": "148.251.128.115:33333", }, } From 845914541a50beba48ee2379712b18737b1de1b0 Mon Sep 17 00:00:00 2001 From: will Date: Mon, 8 Dec 2025 20:12:31 +0000 Subject: [PATCH 34/48] use nix shell for jq --- .github/workflows/benchmark.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index d9861a0644f7..31ed48b1bd10 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -92,7 +92,7 @@ jobs: RUNNER_CONTEXT: ${{ toJSON(runner) }} run: | mkdir -p ${{ runner.temp }}/contexts - echo "$GITHUB_CONTEXT" | jq "del(.token)" > ${{ runner.temp }}/contexts/github.json + echo "$GITHUB_CONTEXT" | nix develop --command jq "del(.token)" > ${{ runner.temp }}/contexts/github.json echo "$RUNNER_CONTEXT" > ${{ runner.temp }}/contexts/runner.json - name: Upload context metadata @@ -176,7 +176,7 @@ jobs: RUNNER_CONTEXT: ${{ toJSON(runner) }} run: | mkdir -p ${{ runner.temp }}/contexts - echo "$GITHUB_CONTEXT" | jq "del(.token)" > ${{ runner.temp }}/contexts/github.json + echo "$GITHUB_CONTEXT" | nix develop --command jq "del(.token)" > ${{ runner.temp }}/contexts/github.json echo "$RUNNER_CONTEXT" > ${{ runner.temp }}/contexts/runner.json - name: Upload context metadata From ba5433d24276a310fe3abd0f865bf31f72b0296b Mon Sep 17 00:00:00 2001 From: will Date: Mon, 8 Dec 2025 20:45:13 +0000 Subject: [PATCH 35/48] remove isolated cpus --- bench.py | 18 ------------------ bench/README.md | 5 +---- bench/benchmark.py | 20 ++------------------ bench/build.py | 7 ++----- bench/capabilities.py | 44 ------------------------------------------- bench/config.py | 1 - bench/utils.py | 24 +++-------------------- 7 files changed, 8 insertions(+), 111 deletions(-) diff --git a/bench.py b/bench.py index df2dd6108703..d327cacd0993 100755 --- a/bench.py +++ b/bench.py @@ -39,7 +39,6 @@ def cmd_build(args: argparse.Namespace) -> int: cli_args={ "binaries_dir": args.binaries_dir, "skip_existing": args.skip_existing, - "no_cpu_pinning": args.no_cpu_pinning, "dry_run": args.dry_run, "verbose": args.verbose, }, @@ -83,7 +82,6 @@ def cmd_run(args: argparse.Namespace) -> int: "connect": args.connect, "chain": args.chain, "instrumented": args.instrumented, - "no_cpu_pinning": args.no_cpu_pinning, "no_cache_drop": args.no_cache_drop, "dry_run": args.dry_run, "verbose": args.verbose, @@ -263,7 +261,6 @@ def cmd_full(args: argparse.Namespace) -> int: "chain": args.chain, "instrumented": args.instrumented, "skip_existing": args.skip_existing, - "no_cpu_pinning": args.no_cpu_pinning, "no_cache_drop": args.no_cache_drop, "dry_run": args.dry_run, "verbose": args.verbose, @@ -396,11 +393,6 @@ def main() -> int: action="store_true", help="Skip build if binary already exists", ) - build_parser.add_argument( - "--no-cpu-pinning", - action="store_true", - help="Disable CPU affinity", - ) build_parser.set_defaults(func=cmd_build) # Run command @@ -461,11 +453,6 @@ def main() -> int: action="store_true", help="Enable profiling (flamegraph + debug logging)", ) - run_parser.add_argument( - "--no-cpu-pinning", - action="store_true", - help="Disable CPU affinity and scheduler priority", - ) run_parser.add_argument( "--no-cache-drop", action="store_true", @@ -562,11 +549,6 @@ def main() -> int: action="store_true", help="Skip build if binary already exists", ) - full_parser.add_argument( - "--no-cpu-pinning", - action="store_true", - help="Disable CPU affinity and scheduler priority", - ) full_parser.add_argument( "--no-cache-drop", action="store_true", diff --git a/bench/README.md b/bench/README.md index 9cf11ac9df4b..ca0d011303de 100644 --- a/bench/README.md +++ b/bench/README.md @@ -21,7 +21,6 @@ just quick HEAD~1 HEAD /path/to/signet/datadir Optional (auto-detected, gracefully degrades without): - `/run/wrappers/bin/drop-caches` (NixOS) - clears page cache between runs -- `taskset` / `chrt` - CPU affinity and scheduler priority ## Commands @@ -70,7 +69,6 @@ Options: - `--instrumented` - Enable flamegraph profiling and debug logging - `--connect ADDR` - P2P node to sync from (empty = public network) - `--chain {main,signet,testnet,regtest}` - Which chain -- `--no-cpu-pinning` - Disable taskset/chrt - `--no-cache-drop` - Don't clear page cache between runs ### analyze @@ -193,14 +191,13 @@ The tool auto-detects system capabilities and gracefully degrades: ```python from bench.capabilities import detect_capabilities caps = detect_capabilities() -# caps.has_hyperfine, caps.can_drop_caches, caps.can_pin_cpu, etc. +# caps.has_hyperfine, caps.can_drop_caches, etc. ``` Missing optional features emit warnings but don't fail: ``` WARNING: drop-caches not available - cache won't be cleared between runs -WARNING: taskset not available - CPU affinity won't be set ``` Missing required features (hyperfine, flamegraph for instrumented) cause errors. diff --git a/bench/benchmark.py b/bench/benchmark.py index 40022eeaee24..fae2cd7fee15 100644 --- a/bench/benchmark.py +++ b/bench/benchmark.py @@ -202,13 +202,8 @@ def _create_prepare_script(self, tmp_datadir: Path, original_datadir: Path) -> P f'rm -rf "{tmp_datadir}"/*', ] - # Copy datadir with optional CPU affinity - if self.capabilities.can_pin_cpu and not self.config.no_cpu_pinning: - commands.append( - f'taskset -c 0-15 cp -r "{original_datadir}"/* "{tmp_datadir}"' - ) - else: - commands.append(f'cp -r "{original_datadir}"/* "{tmp_datadir}"') + # Copy datadir + commands.append(f'cp -r "{original_datadir}"/* "{tmp_datadir}"') # Drop caches if available if self.capabilities.can_drop_caches and not self.config.no_cache_drop: @@ -238,23 +233,12 @@ def _build_bitcoind_cmd( # Add flamegraph wrapper for instrumented mode if self.config.instrumented: - # Flamegraph runs on core 1 - if self.capabilities.can_pin_cpu and not self.config.no_cpu_pinning: - parts.append("taskset -c 1") parts.append("flamegraph") parts.append("--palette bitcoin") parts.append("--title 'bitcoind IBD'") parts.append("-c 'record -F 101 --call-graph fp'") parts.append("--") - # Add CPU affinity for bitcoind (cores 2-15) - if self.capabilities.can_pin_cpu and not self.config.no_cpu_pinning: - parts.append("taskset -c 2-15") - - # Add scheduler priority - if self.capabilities.can_set_scheduler and not self.config.no_cpu_pinning: - parts.append("chrt -o 0") - # Bitcoind command parts.append(str(binary)) parts.append(f"-datadir={tmp_datadir}") diff --git a/bench/build.py b/bench/build.py index b03093f8843a..7b7c749b433f 100644 --- a/bench/build.py +++ b/bench/build.py @@ -134,11 +134,8 @@ def _build_commit(self, name: str, commit: str, output_path: Path) -> None: # Checkout the commit git_checkout(commit, self.repo_path) - # Build with nix (use all available cores for faster builds) - cmd = [] - if self.capabilities.can_pin_cpu: - cmd += ["taskset", "-c", f"2-{self.capabilities.cpu_count - 1}"] - cmd += ["nix", "build", "-L"] + # Build with nix + cmd = ["nix", "build", "-L"] logger.debug(f"Running: {' '.join(cmd)}") result = subprocess.run( diff --git a/bench/capabilities.py b/bench/capabilities.py index 69ef2d1185c3..b01ce2f3a711 100644 --- a/bench/capabilities.py +++ b/bench/capabilities.py @@ -28,10 +28,6 @@ class Capabilities: can_drop_caches: bool drop_caches_path: str | None - # CPU affinity and scheduling - can_pin_cpu: bool - can_set_scheduler: bool - # Required tools has_hyperfine: bool has_flamegraph: bool @@ -84,12 +80,6 @@ def get_warnings(self) -> list[str]: "drop-caches not available - cache won't be cleared between runs" ) - if not self.can_pin_cpu: - warnings.append("taskset not available - CPU affinity won't be set") - - if not self.can_set_scheduler: - warnings.append("chrt not available - scheduler priority won't be set") - return warnings @@ -106,38 +96,6 @@ def _find_drop_caches() -> str | None: return None -def _check_taskset() -> bool: - """Check if taskset is available and works.""" - if not _check_executable("taskset"): - return False - - # Try to run it to verify it works - try: - result = subprocess.run( - ["taskset", "-c", "0", "true"], - timeout=5, - ) - return result.returncode == 0 - except (subprocess.TimeoutExpired, OSError): - return False - - -def _check_chrt() -> bool: - """Check if chrt is available and works.""" - if not _check_executable("chrt"): - return False - - # Try to run it to verify it works - try: - result = subprocess.run( - ["chrt", "-o", "0", "true"], - timeout=5, - ) - return result.returncode == 0 - except (subprocess.TimeoutExpired, OSError): - return False - - def _is_nixos() -> bool: """Check if we're running on NixOS.""" return Path("/etc/NIXOS").exists() @@ -150,8 +108,6 @@ def detect_capabilities() -> Capabilities: return Capabilities( can_drop_caches=drop_caches_path is not None, drop_caches_path=drop_caches_path, - can_pin_cpu=_check_taskset(), - can_set_scheduler=_check_chrt(), has_hyperfine=_check_executable("hyperfine"), has_flamegraph=_check_executable("flamegraph"), has_perf=_check_executable("perf"), diff --git a/bench/config.py b/bench/config.py index a0e2926986fe..e17c0e31a921 100644 --- a/bench/config.py +++ b/bench/config.py @@ -78,7 +78,6 @@ class Config: # Behavior flags instrumented: bool = False skip_existing: bool = False - no_cpu_pinning: bool = False no_cache_drop: bool = False verbose: bool = False dry_run: bool = False diff --git a/bench/utils.py b/bench/utils.py index eda158833ebc..9cd28373d19d 100644 --- a/bench/utils.py +++ b/bench/utils.py @@ -127,21 +127,14 @@ def clean_datadir(datadir: Path) -> None: def copy_datadir(src: Path, dst: Path, capabilities: Capabilities) -> None: - """Copy blockchain data from source to destination. - - Uses taskset for CPU affinity if available. - """ + """Copy blockchain data from source to destination.""" logger.info(f"Copying datadir: {src} -> {dst}") # Ensure destination exists dst.mkdir(parents=True, exist_ok=True) # Build copy command - cmd = [] - if capabilities.can_pin_cpu: - cmd += ["taskset", "-c", "0-15"] - - cmd += ["cp", "-r"] + cmd = ["cp", "-r"] # Copy contents, not directory itself cmd += [str(src) + "/.", str(dst)] @@ -225,18 +218,7 @@ def build_bitcoind_cmd( capabilities: System capabilities debug_flags: Optional debug flags for instrumented mode """ - cmd = [] - - # Add CPU affinity if available and not disabled - if capabilities.can_pin_cpu and not config.no_cpu_pinning: - cmd += ["taskset", "-c", "2-15"] - - # Add scheduler priority if available - if capabilities.can_set_scheduler and not config.no_cpu_pinning: - cmd += ["chrt", "-o", "0"] - - # Add bitcoind with options - cmd += [ + cmd = [ str(binary), f"-datadir={datadir}", f"-dbcache={config.dbcache}", From 760a2229518f4dbd025a10961f3c601ad647c03c Mon Sep 17 00:00:00 2001 From: will Date: Tue, 9 Dec 2025 09:43:28 +0000 Subject: [PATCH 36/48] fixup! use python runner --- bench.py | 13 ++-- bench/analyze.py | 7 --- bench/report.py | 8 +-- bench/utils.py | 150 +++-------------------------------------------- 4 files changed, 13 insertions(+), 165 deletions(-) diff --git a/bench.py b/bench.py index d327cacd0993..157072f95c28 100755 --- a/bench.py +++ b/bench.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 """Benchcoin - Bitcoin Core benchmarking toolkit. -A unified CLI for building, benchmarking, analyzing, and reporting -on Bitcoin Core performance. +A CLI for building, benchmarking, analyzing, and reporting on Bitcoin Core +performance. Usage: bench.py build BASE HEAD Build bitcoind at two commits @@ -22,7 +22,6 @@ from bench.capabilities import detect_capabilities from bench.config import build_config -# Setup logging logging.basicConfig( level=logging.INFO, format="%(levelname)s: %(message)s", @@ -93,14 +92,12 @@ def cmd_run(args: argparse.Namespace) -> int: if args.verbose: logging.getLogger().setLevel(logging.DEBUG) - # Validate config errors = config.validate() if errors: for error in errors: logger.error(error) return 1 - # Check binaries exist binaries_dir = ( Path(args.binaries_dir) if args.binaries_dir else Path(config.binaries_dir) ) @@ -135,7 +132,7 @@ def cmd_run(args: argparse.Namespace) -> int: if config.instrumented: from bench.analyze import AnalyzePhase - analyze_phase = AnalyzePhase(config) + analyze_phase = AnalyzePhase() if result.debug_log_base: try: @@ -245,7 +242,6 @@ def cmd_full(args: argparse.Namespace) -> int: from bench.analyze import AnalyzePhase from bench.benchmark import BenchmarkPhase from bench.build import BuildPhase - from bench.utils import find_debug_log capabilities = detect_capabilities() config = build_config( @@ -316,7 +312,7 @@ def cmd_full(args: argparse.Namespace) -> int: # Phase 3: Analyze (for instrumented runs) if config.instrumented: logger.info("=== Phase 3: Analyze ===") - analyze_phase = AnalyzePhase(config) + analyze_phase = AnalyzePhase() # Analyze base debug log if benchmark_result.debug_log_base: @@ -353,7 +349,6 @@ def main() -> int: epilog=__doc__, ) - # Global options parser.add_argument( "--config", metavar="PATH", diff --git a/bench/analyze.py b/bench/analyze.py index a31b807c422b..423b67465a83 100644 --- a/bench/analyze.py +++ b/bench/analyze.py @@ -11,7 +11,6 @@ from collections import OrderedDict from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING # matplotlib is optional - gracefully handle if not installed try: @@ -21,9 +20,6 @@ except ImportError: HAS_MATPLOTLIB = False -if TYPE_CHECKING: - from .config import Config - logger = logging.getLogger(__name__) # Bitcoin fork heights for plot annotations @@ -488,9 +484,6 @@ def _add_fork_markers(self, min_x: float, max_x: float, max_y: float) -> None: class AnalyzePhase: """Analyze benchmark results and generate plots.""" - def __init__(self, config: Config | None = None): - self.config = config - def run( self, commit: str, diff --git a/bench/report.py b/bench/report.py index a41a035fbe9a..16445bde5246 100644 --- a/bench/report.py +++ b/bench/report.py @@ -7,6 +7,8 @@ import json import logging +import re +import shutil from dataclasses import dataclass, field from pathlib import Path from typing import Any @@ -324,8 +326,6 @@ def _generate_html( def _linkify_commit(self, command: str) -> str: """Convert commit hashes in command to links.""" - import re - def replace_commit(match): commit = match.group(1) short_commit = commit[:8] if len(commit) > 8 else commit @@ -346,8 +346,6 @@ def _generate_graphs_section( commit = run.parameters.get("commit", "") if not commit: # Try to extract from command - import re - match = re.search(r"\(([a-f0-9]+)\)", run.command) if match: commit = match.group(1) @@ -403,8 +401,6 @@ def _generate_graphs_section( def _copy_artifacts(self, input_dir: Path, output_dir: Path) -> None: """Copy flamegraphs and plots to output directory.""" - import shutil - # Skip if input and output are the same directory if input_dir.resolve() == output_dir.resolve(): logger.debug("Input and output are the same directory, skipping copy") diff --git a/bench/utils.py b/bench/utils.py index 9cd28373d19d..df454cf0644e 100644 --- a/bench/utils.py +++ b/bench/utils.py @@ -1,28 +1,14 @@ -"""Utility functions for git, datadir, and system operations.""" +"""Utility functions for git operations.""" from __future__ import annotations import logging -import os -import shutil import subprocess -import tempfile from pathlib import Path -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from .capabilities import Capabilities - from .config import Config logger = logging.getLogger(__name__) -class GitError(Exception): - """Git operation failed.""" - - pass - - class GitState: """Saved git state for restoration after operations.""" @@ -80,6 +66,12 @@ def restore(self) -> None: ) +class GitError(Exception): + """Git operation failed.""" + + pass + + def git_checkout(commit: str, repo_path: Path | None = None) -> None: """Checkout a specific commit.""" repo_path = repo_path or Path.cwd() @@ -111,131 +103,3 @@ def git_rev_parse(ref: str, repo_path: Path | None = None) -> str: raise GitError(f"Failed to resolve {ref}: {result.stderr}") return result.stdout.strip() - - -def clean_datadir(datadir: Path) -> None: - """Remove all contents from a data directory.""" - if not datadir.exists(): - return - - logger.debug(f"Cleaning datadir: {datadir}") - for item in datadir.iterdir(): - if item.is_dir(): - shutil.rmtree(item) - else: - item.unlink() - - -def copy_datadir(src: Path, dst: Path, capabilities: Capabilities) -> None: - """Copy blockchain data from source to destination.""" - logger.info(f"Copying datadir: {src} -> {dst}") - - # Ensure destination exists - dst.mkdir(parents=True, exist_ok=True) - - # Build copy command - cmd = ["cp", "-r"] - # Copy contents, not directory itself - cmd += [str(src) + "/.", str(dst)] - - subprocess.run(cmd, check=True) - - -def drop_caches(capabilities: Capabilities) -> bool: - """Drop filesystem caches if available. - - Returns True if caches were dropped, False if not available. - """ - if not capabilities.can_drop_caches or not capabilities.drop_caches_path: - logger.debug("Cache dropping not available, skipping") - return False - - logger.debug("Dropping filesystem caches") - subprocess.run([capabilities.drop_caches_path], check=True) - return True - - -def clean_debug_logs(datadir: Path) -> None: - """Remove debug.log files from datadir and subdirectories.""" - logger.debug(f"Cleaning debug logs in: {datadir}") - - for log_file in datadir.rglob("debug.log"): - log_file.unlink() - - -def find_debug_log(datadir: Path) -> Path | None: - """Find debug.log in datadir or subdirectories.""" - # Check common locations - candidates = [ - datadir / "debug.log", - datadir / "mainnet" / "debug.log", - datadir / "testnet3" / "debug.log", - datadir / "signet" / "debug.log", - datadir / "regtest" / "debug.log", - ] - - for candidate in candidates: - if candidate.exists(): - return candidate - - # Fallback: search recursively - for log_file in datadir.rglob("debug.log"): - return log_file - - return None - - -def create_temp_script(commands: list[str], name: str = "hook") -> Path: - """Create a temporary shell script for hyperfine hooks. - - Returns path to the script. - """ - script_content = "#!/usr/bin/env bash\nset -euxo pipefail\n" - script_content += "\n".join(commands) + "\n" - - # Create temp file that persists (caller is responsible for cleanup) - fd, path = tempfile.mkstemp(suffix=".sh", prefix=f"bench_{name}_") - os.write(fd, script_content.encode()) - os.close(fd) - os.chmod(path, 0o755) - - return Path(path) - - -def build_bitcoind_cmd( - binary: Path, - datadir: Path, - config: Config, - capabilities: Capabilities, - debug_flags: list[str] | None = None, -) -> list[str]: - """Build the bitcoind command with optional wrappers. - - Args: - binary: Path to bitcoind binary - datadir: Data directory - config: Benchmark configuration - capabilities: System capabilities - debug_flags: Optional debug flags for instrumented mode - """ - cmd = [ - str(binary), - f"-datadir={datadir}", - f"-dbcache={config.dbcache}", - f"-stopatheight={config.stop_height}", - "-prune=10000", - f"-chain={config.chain}", - "-daemon=0", - "-printtoconsole=0", - ] - - # Add connect address if specified - if config.connect: - cmd.append(f"-connect={config.connect}") - - # Add debug flags for instrumented mode - if debug_flags: - for flag in debug_flags: - cmd.append(f"-debug={flag}") - - return cmd From 031e3677ebfc3bf0979f9c4cf20364864f8e2085 Mon Sep 17 00:00:00 2001 From: will Date: Tue, 9 Dec 2025 09:48:37 +0000 Subject: [PATCH 37/48] fixup! use python runner --- bench/analyze.py | 9 +++++++++ bench/benchmark.py | 24 +++++++++++++++++++++++- bench/build.py | 11 +++++++++-- 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/bench/analyze.py b/bench/analyze.py index 423b67465a83..a5f52b4ce035 100644 --- a/bench/analyze.py +++ b/bench/analyze.py @@ -515,7 +515,16 @@ def run( parser = LogParser() data = parser.parse_file(log_file) + # Log parsed data summary + logger.info(f" UpdateTip entries: {len(data.update_tip)}") + logger.info(f" LevelDB compact entries: {len(data.leveldb_compact)}") + logger.info(f" LevelDB gen table entries: {len(data.leveldb_gen_table)}") + logger.info(f" Validation txadd entries: {len(data.validation_txadd)}") + logger.info(f" CoinDB write batch entries: {len(data.coindb_write_batch)}") + logger.info(f" CoinDB commit entries: {len(data.coindb_commit)}") + logger.info(f"Generating plots for {commit[:12]}") + logger.info(f" Output directory: {output_dir}") generator = PlotGenerator(commit[:12], output_dir) plots = generator.generate_all(data) diff --git a/bench/benchmark.py b/bench/benchmark.py index fae2cd7fee15..886245d11546 100644 --- a/bench/benchmark.py +++ b/bench/benchmark.py @@ -87,6 +87,9 @@ def run( results_file = output_dir / "results.json" logger.info("Starting benchmark") + logger.info(f" Output dir: {output_dir}") + logger.info(f" Temp datadir: {tmp_datadir}") + logger.info(f" Source datadir: {datadir}") logger.info(f" Base: {base_commit[:12]}") logger.info(f" Head: {head_commit[:12]}") logger.info(f" Instrumented: {self.config.instrumented}") @@ -114,6 +117,14 @@ def run( output_dir=output_dir, ) + # Log the commands being benchmarked + base_cmd = self._build_bitcoind_cmd(base_binary, tmp_datadir) + head_cmd = self._build_bitcoind_cmd(head_binary, tmp_datadir) + logger.info("Base command:") + logger.info(f" {base_cmd}") + logger.info("Head command:") + logger.info(f" {head_cmd}") + if self.config.dry_run: logger.info(f"[DRY RUN] Would run: {' '.join(cmd)}") return BenchmarkResult( @@ -123,8 +134,10 @@ def run( instrumented=self.config.instrumented, ) - # Run hyperfine + # Log the full hyperfine command logger.info("Running hyperfine...") + logger.info(f" Command: {' '.join(cmd[:7])} ...") # First few args + logger.debug(f" Full command: {' '.join(cmd)}") _result = subprocess.run(cmd, check=True) # Collect results @@ -137,6 +150,7 @@ def run( # For instrumented runs, collect flamegraphs and debug logs if self.config.instrumented: + logger.info("Collecting instrumented artifacts...") base_fg = output_dir / f"{base_commit[:12]}-flamegraph.svg" head_fg = output_dir / f"{head_commit[:12]}-flamegraph.svg" base_log = output_dir / f"{base_commit[:12]}-debug.log" @@ -149,16 +163,21 @@ def run( ]: src = Path(src_name) if src.exists(): + logger.info(f" Moving {src_name} -> {dest}") shutil.move(str(src), str(dest)) if base_fg.exists(): benchmark_result.flamegraph_base = base_fg + logger.info(f" Flamegraph (base): {base_fg}") if head_fg.exists(): benchmark_result.flamegraph_head = head_fg + logger.info(f" Flamegraph (head): {head_fg}") if base_log.exists(): benchmark_result.debug_log_base = base_log + logger.info(f" Debug log (base): {base_log}") if head_log.exists(): benchmark_result.debug_log_head = head_log + logger.info(f" Debug log (head): {head_log}") # Clean up tmp_datadir if tmp_datadir.exists(): @@ -186,6 +205,9 @@ def _create_temp_script(self, commands: list[str], name: str) -> Path: script_path = Path(path) self._temp_scripts.append(script_path) + logger.debug(f"Created {name} script: {script_path}") + for cmd in commands: + logger.debug(f" {cmd}") return script_path def _create_setup_script(self, tmp_datadir: Path) -> Path: diff --git a/bench/build.py b/bench/build.py index 7b7c749b433f..71e7e30b3288 100644 --- a/bench/build.py +++ b/bench/build.py @@ -71,6 +71,8 @@ def run( logger.info("Building binaries for comparison:") logger.info(f" Base: {base_hash[:12]} ({base_commit})") logger.info(f" Head: {head_hash[:12]} ({head_commit})") + logger.info(f" Repo: {self.repo_path}") + logger.info(f" Output: {binaries_dir}") # Setup output directories base_dir = binaries_dir / "base" @@ -132,12 +134,14 @@ def _build_commit(self, name: str, commit: str, output_path: Path) -> None: return # Checkout the commit + logger.info(f" Checking out {commit[:12]}...") git_checkout(commit, self.repo_path) # Build with nix cmd = ["nix", "build", "-L"] - logger.debug(f"Running: {' '.join(cmd)}") + logger.info(f" Running: {' '.join(cmd)}") + logger.info(f" Working directory: {self.repo_path}") result = subprocess.run( cmd, cwd=self.repo_path, @@ -151,6 +155,8 @@ def _build_commit(self, name: str, commit: str, output_path: Path) -> None: if not nix_binary.exists(): raise RuntimeError(f"Built binary not found at {nix_binary}") + logger.info(f" Copying {nix_binary} -> {output_path}") + # Remove existing binary if present (may be read-only from nix) if output_path.exists(): output_path.chmod(0o755) @@ -158,9 +164,10 @@ def _build_commit(self, name: str, commit: str, output_path: Path) -> None: shutil.copy2(nix_binary, output_path) output_path.chmod(0o755) # Ensure it's executable and writable - logger.info(f"Built {name} binary: {output_path}") + logger.info(f" Built {name} binary: {output_path}") # Clean up nix result symlink result_link = self.repo_path / "result" if result_link.is_symlink(): + logger.debug(f" Removing nix result symlink: {result_link}") result_link.unlink() From c8cc28f8df16a67391b5b94d8422af8172db85f4 Mon Sep 17 00:00:00 2001 From: will Date: Tue, 9 Dec 2025 10:10:35 +0000 Subject: [PATCH 38/48] arbitrary bins --- .github/workflows/benchmark.yml | 12 +- bench.py | 380 ++++++++++++-------------------- bench/benchmark.py | 171 +++++++------- bench/build.py | 148 +++++++------ bench/capabilities.py | 1 - bench/compare.py | 180 +++++++++++++++ bench/report.py | 54 +++-- justfile | 49 ++-- 8 files changed, 558 insertions(+), 437 deletions(-) create mode 100644 bench/compare.py diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 31ed48b1bd10..f58da7c5bce5 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -23,8 +23,8 @@ jobs: - name: Build both binaries run: | nix develop --command python3 bench.py build \ - --binaries-dir ${{ runner.temp }}/binaries \ - $BASE_SHA $HEAD_SHA + -o ${{ runner.temp }}/binaries \ + $BASE_SHA:base $HEAD_SHA:head - name: Upload binaries uses: actions/upload-artifact@v4 @@ -73,12 +73,12 @@ jobs: - name: Run benchmark run: | nix develop --command python3 bench.py --profile ci run \ - --binaries-dir ${{ runner.temp }}/binaries \ --datadir $ORIGINAL_DATADIR \ --tmp-datadir ${{ runner.temp }}/datadir \ --output-dir ${{ runner.temp }}/output \ --dbcache ${{ matrix.dbcache }} \ - $BASE_SHA $HEAD_SHA + base:${{ runner.temp }}/binaries/base/bitcoind \ + head:${{ runner.temp }}/binaries/head/bitcoind - name: Upload results uses: actions/upload-artifact@v4 @@ -143,12 +143,12 @@ jobs: run: | nix develop --command python3 bench.py --profile ci run \ --instrumented \ - --binaries-dir ${{ runner.temp }}/binaries \ --datadir $ORIGINAL_DATADIR \ --tmp-datadir ${{ runner.temp }}/datadir \ --output-dir ${{ runner.temp }}/output \ --dbcache ${{ matrix.dbcache }} \ - $BASE_SHA $HEAD_SHA + base:${{ runner.temp }}/binaries/base/bitcoind \ + head:${{ runner.temp }}/binaries/head/bitcoind - name: Upload results uses: actions/upload-artifact@v4 diff --git a/bench.py b/bench.py index 157072f95c28..04acef1a0d51 100755 --- a/bench.py +++ b/bench.py @@ -5,11 +5,24 @@ performance. Usage: - bench.py build BASE HEAD Build bitcoind at two commits - bench.py run BASE HEAD Run benchmark - bench.py analyze LOGFILE Generate plots from debug.log + bench.py build COMMIT[:NAME]... Build bitcoind at one or more commits + bench.py run NAME:BINARY... Benchmark one or more binaries + bench.py analyze COMMIT LOGFILE Generate plots from debug.log + bench.py compare RESULTS... Compare benchmark results bench.py report INPUT OUTPUT Generate HTML report - bench.py full BASE HEAD Complete pipeline: build → run → analyze + +Examples: + # Build two commits + bench.py build HEAD~1:before HEAD:after + + # Benchmark built binaries + bench.py run before:./binaries/before/bitcoind after:./binaries/after/bitcoind --datadir /data + + # Compare results + bench.py compare ./bench-output/results.json + + # Generate HTML report + bench.py report ./bench-output ./report """ from __future__ import annotations @@ -30,13 +43,13 @@ def cmd_build(args: argparse.Namespace) -> int: - """Build bitcoind at two commits.""" + """Build bitcoind at one or more commits.""" from bench.build import BuildPhase capabilities = detect_capabilities() config = build_config( cli_args={ - "binaries_dir": args.binaries_dir, + "binaries_dir": args.output_dir, "skip_existing": args.skip_existing, "dry_run": args.dry_run, "verbose": args.verbose, @@ -52,12 +65,12 @@ def cmd_build(args: argparse.Namespace) -> int: try: result = phase.run( - args.base_commit, - args.head_commit, - binaries_dir=Path(args.binaries_dir) if args.binaries_dir else None, + args.commits, + output_dir=Path(args.output_dir) if args.output_dir else None, ) - logger.info(f"Built base binary: {result.base_binary}") - logger.info(f"Built head binary: {result.head_binary}") + logger.info(f"Built {len(result.binaries)} binary(ies):") + for binary in result.binaries: + logger.info(f" {binary.name}: {binary.path}") return 0 except Exception as e: logger.error(f"Build failed: {e}") @@ -65,15 +78,14 @@ def cmd_build(args: argparse.Namespace) -> int: def cmd_run(args: argparse.Namespace) -> int: - """Run benchmark comparing two commits.""" - from bench.benchmark import BenchmarkPhase + """Run benchmark on one or more binaries.""" + from bench.benchmark import BenchmarkPhase, parse_binary_spec capabilities = detect_capabilities() config = build_config( cli_args={ "datadir": args.datadir, "tmp_datadir": args.tmp_datadir, - "binaries_dir": args.binaries_dir, "output_dir": args.output_dir, "stop_height": args.stop_height, "dbcache": args.dbcache, @@ -98,31 +110,25 @@ def cmd_run(args: argparse.Namespace) -> int: logger.error(error) return 1 - binaries_dir = ( - Path(args.binaries_dir) if args.binaries_dir else Path(config.binaries_dir) - ) - base_binary = binaries_dir / "base" / "bitcoind" - head_binary = binaries_dir / "head" / "bitcoind" - - if not base_binary.exists(): - logger.error(f"Base binary not found: {base_binary}") - logger.error("Run 'bench.py build' first") + # Parse binary specs + try: + binaries = [parse_binary_spec(spec) for spec in args.binaries] + except ValueError as e: + logger.error(str(e)) return 1 - if not head_binary.exists(): - logger.error(f"Head binary not found: {head_binary}") - logger.error("Run 'bench.py build' first") - return 1 + # Validate binaries exist + for name, path in binaries: + if not path.exists(): + logger.error(f"Binary not found: {path} ({name})") + return 1 phase = BenchmarkPhase(config, capabilities) output_dir = Path(config.output_dir) try: result = phase.run( - base_commit=args.base_commit, - head_commit=args.head_commit, - base_binary=base_binary, - head_binary=head_binary, + binaries=binaries, datadir=Path(config.datadir), output_dir=output_dir, ) @@ -134,25 +140,16 @@ def cmd_run(args: argparse.Namespace) -> int: analyze_phase = AnalyzePhase() - if result.debug_log_base: - try: - analyze_phase.run( - commit=args.base_commit, - log_file=result.debug_log_base, - output_dir=output_dir / "plots", - ) - except Exception as e: - logger.warning(f"Analysis for base failed: {e}") - - if result.debug_log_head: - try: - analyze_phase.run( - commit=args.head_commit, - log_file=result.debug_log_head, - output_dir=output_dir / "plots", - ) - except Exception as e: - logger.warning(f"Analysis for head failed: {e}") + for binary_result in result.binaries: + if binary_result.debug_log: + try: + analyze_phase.run( + commit=binary_result.name, + log_file=binary_result.debug_log, + output_dir=output_dir / "plots", + ) + except Exception as e: + logger.warning(f"Analysis for {binary_result.name} failed: {e}") return 0 except Exception as e: @@ -164,6 +161,46 @@ def cmd_run(args: argparse.Namespace) -> int: return 1 +def cmd_compare(args: argparse.Namespace) -> int: + """Compare benchmark results from multiple files.""" + from bench.compare import ComparePhase + + if args.verbose: + logging.getLogger().setLevel(logging.DEBUG) + + results_files = [Path(f) for f in args.results_files] + + # Validate files exist + for f in results_files: + if not f.exists(): + logger.error(f"Results file not found: {f}") + return 1 + + phase = ComparePhase() + + try: + result = phase.run(results_files, baseline=args.baseline) + + # Output results + output_json = phase.to_json(result) + + if args.output: + output_path = Path(args.output) + output_path.write_text(output_json) + logger.info(f"Comparison saved to: {output_path}") + else: + print(output_json) + + return 0 + except Exception as e: + logger.error(f"Comparison failed: {e}") + if args.verbose: + import traceback + + traceback.print_exc() + return 1 + + def cmd_analyze(args: argparse.Namespace) -> int: """Generate plots from debug.log.""" from bench.analyze import AnalyzePhase @@ -237,110 +274,6 @@ def cmd_report(args: argparse.Namespace) -> int: return 1 -def cmd_full(args: argparse.Namespace) -> int: - """Run full pipeline: build → run → analyze.""" - from bench.analyze import AnalyzePhase - from bench.benchmark import BenchmarkPhase - from bench.build import BuildPhase - - capabilities = detect_capabilities() - config = build_config( - cli_args={ - "datadir": args.datadir, - "tmp_datadir": args.tmp_datadir, - "binaries_dir": args.binaries_dir, - "output_dir": args.output_dir, - "stop_height": args.stop_height, - "dbcache": args.dbcache, - "runs": args.runs, - "connect": args.connect, - "chain": args.chain, - "instrumented": args.instrumented, - "skip_existing": args.skip_existing, - "no_cache_drop": args.no_cache_drop, - "dry_run": args.dry_run, - "verbose": args.verbose, - }, - config_file=Path(args.config) if args.config else None, - profile=args.profile, - ) - - if args.verbose: - logging.getLogger().setLevel(logging.DEBUG) - - # Validate config - errors = config.validate() - if errors: - for error in errors: - logger.error(error) - return 1 - - output_dir = Path(config.output_dir) - binaries_dir = Path(config.binaries_dir) - - # Phase 1: Build - logger.info("=== Phase 1: Build ===") - build_phase = BuildPhase(config, capabilities) - - try: - build_result = build_phase.run( - args.base_commit, - args.head_commit, - binaries_dir=binaries_dir, - ) - except Exception as e: - logger.error(f"Build failed: {e}") - return 1 - - # Phase 2: Benchmark - logger.info("=== Phase 2: Benchmark ===") - benchmark_phase = BenchmarkPhase(config, capabilities) - - try: - benchmark_result = benchmark_phase.run( - base_commit=build_result.base_commit, - head_commit=build_result.head_commit, - base_binary=build_result.base_binary, - head_binary=build_result.head_binary, - datadir=Path(config.datadir), - output_dir=output_dir, - ) - except Exception as e: - logger.error(f"Benchmark failed: {e}") - return 1 - - # Phase 3: Analyze (for instrumented runs) - if config.instrumented: - logger.info("=== Phase 3: Analyze ===") - analyze_phase = AnalyzePhase() - - # Analyze base debug log - if benchmark_result.debug_log_base: - try: - analyze_phase.run( - commit=build_result.base_commit, - log_file=benchmark_result.debug_log_base, - output_dir=output_dir / "plots", - ) - except Exception as e: - logger.warning(f"Analysis for base failed: {e}") - - # Analyze head debug log - if benchmark_result.debug_log_head: - try: - analyze_phase.run( - commit=build_result.head_commit, - log_file=benchmark_result.debug_log_head, - output_dir=output_dir / "plots", - ) - except Exception as e: - logger.warning(f"Analysis for head failed: {e}") - - logger.info("=== Complete ===") - logger.info(f"Results: {benchmark_result.results_file}") - return 0 - - def main() -> int: """Main entry point.""" parser = argparse.ArgumentParser( @@ -375,11 +308,21 @@ def main() -> int: subparsers = parser.add_subparsers(dest="command", help="Commands") # Build command - build_parser = subparsers.add_parser("build", help="Build bitcoind at two commits") - build_parser.add_argument("base_commit", help="Base commit (for comparison)") - build_parser.add_argument("head_commit", help="Head commit (new code)") + build_parser = subparsers.add_parser( + "build", + help="Build bitcoind at one or more commits", + description="Build bitcoind binaries from git commits. " + "Each commit can optionally have a name suffix: COMMIT:NAME", + ) + build_parser.add_argument( + "commits", + nargs="+", + metavar="COMMIT[:NAME]", + help="Commit(s) to build. Format: COMMIT or COMMIT:NAME (e.g., HEAD:latest, abc123:v27)", + ) build_parser.add_argument( - "--binaries-dir", + "-o", + "--output-dir", metavar="PATH", help="Where to store binaries (default: ./binaries)", ) @@ -391,9 +334,18 @@ def main() -> int: build_parser.set_defaults(func=cmd_build) # Run command - run_parser = subparsers.add_parser("run", help="Run benchmark") - run_parser.add_argument("base_commit", help="Base commit hash") - run_parser.add_argument("head_commit", help="Head commit hash") + run_parser = subparsers.add_parser( + "run", + help="Run benchmark on one or more binaries", + description="Benchmark bitcoind binaries using hyperfine. " + "Each binary must have a name and path: NAME:PATH", + ) + run_parser.add_argument( + "binaries", + nargs="+", + metavar="NAME:PATH", + help="Binary(ies) to benchmark. Format: NAME:PATH (e.g., v27:./binaries/v27/bitcoind)", + ) run_parser.add_argument( "--datadir", required=True, @@ -406,14 +358,10 @@ def main() -> int: help="Temp datadir for benchmark runs", ) run_parser.add_argument( - "--binaries-dir", - metavar="PATH", - help="Location of pre-built binaries", - ) - run_parser.add_argument( + "-o", "--output-dir", metavar="PATH", - help="Output directory for results", + help="Output directory for results (default: ./bench-output)", ) run_parser.add_argument( "--stop-height", @@ -469,6 +417,32 @@ def main() -> int: ) analyze_parser.set_defaults(func=cmd_analyze) + # Compare command + compare_parser = subparsers.add_parser( + "compare", + help="Compare benchmark results from multiple files", + description="Load and compare results from one or more results.json files. " + "Calculates speedup percentages relative to a baseline.", + ) + compare_parser.add_argument( + "results_files", + nargs="+", + metavar="RESULTS_FILE", + help="results.json file(s) to compare", + ) + compare_parser.add_argument( + "--baseline", + metavar="NAME", + help="Name of the baseline entry (default: first entry)", + ) + compare_parser.add_argument( + "-o", + "--output", + metavar="FILE", + help="Output file for comparison JSON (default: stdout)", + ) + compare_parser.set_defaults(func=cmd_compare) + # Report command report_parser = subparsers.add_parser("report", help="Generate HTML report") report_parser.add_argument("input_dir", help="Directory with results.json") @@ -479,78 +453,6 @@ def main() -> int: ) report_parser.set_defaults(func=cmd_report) - # Full command - full_parser = subparsers.add_parser( - "full", help="Full pipeline: build → run → analyze" - ) - full_parser.add_argument("base_commit", help="Base commit (for comparison)") - full_parser.add_argument("head_commit", help="Head commit (new code)") - full_parser.add_argument( - "--datadir", - required=True, - metavar="PATH", - help="Source datadir with blockchain snapshot", - ) - full_parser.add_argument( - "--tmp-datadir", - metavar="PATH", - help="Temp datadir for benchmark runs", - ) - full_parser.add_argument( - "--binaries-dir", - metavar="PATH", - help="Where to store binaries", - ) - full_parser.add_argument( - "--output-dir", - metavar="PATH", - help="Output directory for results", - ) - full_parser.add_argument( - "--stop-height", - type=int, - metavar="N", - help="Block height to stop at", - ) - full_parser.add_argument( - "--dbcache", - type=int, - metavar="N", - help="Database cache size in MB", - ) - full_parser.add_argument( - "--runs", - type=int, - metavar="N", - help="Number of benchmark iterations", - ) - full_parser.add_argument( - "--connect", - metavar="ADDR", - help="Connect address for sync", - ) - full_parser.add_argument( - "--chain", - choices=["main", "testnet", "signet", "regtest"], - help="Chain to use", - ) - full_parser.add_argument( - "--instrumented", - action="store_true", - help="Enable profiling (flamegraph + debug logging)", - ) - full_parser.add_argument( - "--skip-existing", - action="store_true", - help="Skip build if binary already exists", - ) - full_parser.add_argument( - "--no-cache-drop", - action="store_true", - help="Skip cache dropping between runs", - ) - full_parser.set_defaults(func=cmd_full) - args = parser.parse_args() if not args.command: diff --git a/bench/benchmark.py b/bench/benchmark.py index 886245d11546..5dc25a0430e6 100644 --- a/bench/benchmark.py +++ b/bench/benchmark.py @@ -1,4 +1,4 @@ -"""Benchmark phase - run hyperfine benchmarks comparing two bitcoind binaries.""" +"""Benchmark phase - run hyperfine benchmarks on bitcoind binaries.""" from __future__ import annotations @@ -7,7 +7,7 @@ import shutil import subprocess import tempfile -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path from typing import TYPE_CHECKING @@ -22,22 +22,39 @@ INSTRUMENTED_DEBUG_FLAGS = ["coindb", "leveldb", "bench", "validation"] +@dataclass +class BinaryResult: + """Result for a single binary.""" + + name: str + flamegraph: Path | None = None + debug_log: Path | None = None + + @dataclass class BenchmarkResult: """Result of the benchmark phase.""" results_file: Path - base_commit: str - head_commit: str instrumented: bool - flamegraph_base: Path | None = None - flamegraph_head: Path | None = None - debug_log_base: Path | None = None - debug_log_head: Path | None = None + binaries: list[BinaryResult] = field(default_factory=list) + + +def parse_binary_spec(spec: str) -> tuple[str, Path]: + """Parse a binary spec like 'name:/path/to/binary'. + + Returns (name, path). + """ + if ":" not in spec: + raise ValueError(f"Invalid binary spec '{spec}': must be NAME:PATH") + name, path_str = spec.split(":", 1) + if not name: + raise ValueError(f"Invalid binary spec '{spec}': name cannot be empty") + return name, Path(path_str) class BenchmarkPhase: - """Run hyperfine benchmarks comparing two bitcoind binaries.""" + """Run hyperfine benchmarks on bitcoind binaries.""" def __init__( self, @@ -50,26 +67,28 @@ def __init__( def run( self, - base_commit: str, - head_commit: str, - base_binary: Path, - head_binary: Path, + binaries: list[tuple[str, Path]], datadir: Path, output_dir: Path, ) -> BenchmarkResult: - """Run benchmarks comparing base and head binaries. + """Run benchmarks on given binaries. Args: - base_commit: Git hash of base commit - head_commit: Git hash of head commit - base_binary: Path to base bitcoind binary - head_binary: Path to head bitcoind binary + binaries: List of (name, binary_path) tuples datadir: Source datadir with blockchain snapshot output_dir: Where to store results Returns: BenchmarkResult with paths to outputs """ + if not binaries: + raise ValueError("At least one binary is required") + + # Validate all binaries exist + for name, path in binaries: + if not path.exists(): + raise FileNotFoundError(f"Binary not found: {path} ({name})") + # Check prerequisites errors = self.capabilities.check_for_run(self.config.instrumented) if errors: @@ -90,8 +109,9 @@ def run( logger.info(f" Output dir: {output_dir}") logger.info(f" Temp datadir: {tmp_datadir}") logger.info(f" Source datadir: {datadir}") - logger.info(f" Base: {base_commit[:12]}") - logger.info(f" Head: {head_commit[:12]}") + logger.info(f" Binaries: {len(binaries)}") + for name, path in binaries: + logger.info(f" {name}: {path}") logger.info(f" Instrumented: {self.config.instrumented}") logger.info(f" Runs: {self.config.runs}") logger.info(f" Stop height: {self.config.stop_height}") @@ -105,10 +125,7 @@ def run( # Build hyperfine command cmd = self._build_hyperfine_cmd( - base_commit=base_commit, - head_commit=head_commit, - base_binary=base_binary, - head_binary=head_binary, + binaries=binaries, tmp_datadir=tmp_datadir, results_file=results_file, setup_script=setup_script, @@ -118,19 +135,15 @@ def run( ) # Log the commands being benchmarked - base_cmd = self._build_bitcoind_cmd(base_binary, tmp_datadir) - head_cmd = self._build_bitcoind_cmd(head_binary, tmp_datadir) - logger.info("Base command:") - logger.info(f" {base_cmd}") - logger.info("Head command:") - logger.info(f" {head_cmd}") + logger.info("Commands to benchmark:") + for name, path in binaries: + bitcoind_cmd = self._build_bitcoind_cmd(path, tmp_datadir) + logger.info(f" {name}: {bitcoind_cmd}") if self.config.dry_run: logger.info(f"[DRY RUN] Would run: {' '.join(cmd)}") return BenchmarkResult( results_file=results_file, - base_commit=base_commit, - head_commit=head_commit, instrumented=self.config.instrumented, ) @@ -138,46 +151,31 @@ def run( logger.info("Running hyperfine...") logger.info(f" Command: {' '.join(cmd[:7])} ...") # First few args logger.debug(f" Full command: {' '.join(cmd)}") - _result = subprocess.run(cmd, check=True) + subprocess.run(cmd, check=True) # Collect results benchmark_result = BenchmarkResult( results_file=results_file, - base_commit=base_commit, - head_commit=head_commit, instrumented=self.config.instrumented, ) # For instrumented runs, collect flamegraphs and debug logs if self.config.instrumented: logger.info("Collecting instrumented artifacts...") - base_fg = output_dir / f"{base_commit[:12]}-flamegraph.svg" - head_fg = output_dir / f"{head_commit[:12]}-flamegraph.svg" - base_log = output_dir / f"{base_commit[:12]}-debug.log" - head_log = output_dir / f"{head_commit[:12]}-debug.log" - - # Move flamegraphs from current directory if they exist - for src_name, dest in [ - ("base-flamegraph.svg", base_fg), - ("head-flamegraph.svg", head_fg), - ]: - src = Path(src_name) - if src.exists(): - logger.info(f" Moving {src_name} -> {dest}") - shutil.move(str(src), str(dest)) - - if base_fg.exists(): - benchmark_result.flamegraph_base = base_fg - logger.info(f" Flamegraph (base): {base_fg}") - if head_fg.exists(): - benchmark_result.flamegraph_head = head_fg - logger.info(f" Flamegraph (head): {head_fg}") - if base_log.exists(): - benchmark_result.debug_log_base = base_log - logger.info(f" Debug log (base): {base_log}") - if head_log.exists(): - benchmark_result.debug_log_head = head_log - logger.info(f" Debug log (head): {head_log}") + for name, _path in binaries: + binary_result = BinaryResult(name=name) + + flamegraph_file = output_dir / f"{name}-flamegraph.svg" + debug_log_file = output_dir / f"{name}-debug.log" + + if flamegraph_file.exists(): + binary_result.flamegraph = flamegraph_file + logger.info(f" Flamegraph ({name}): {flamegraph_file}") + if debug_log_file.exists(): + binary_result.debug_log = debug_log_file + logger.info(f" Debug log ({name}): {debug_log_file}") + + benchmark_result.binaries.append(binary_result) # Clean up tmp_datadir if tmp_datadir.exists(): @@ -283,10 +281,7 @@ def _build_bitcoind_cmd( def _build_hyperfine_cmd( self, - base_commit: str, - head_commit: str, - base_binary: Path, - head_binary: Path, + binaries: list[tuple[str, Path]], tmp_datadir: Path, results_file: Path, setup_script: Path, @@ -306,52 +301,42 @@ def _build_hyperfine_cmd( "--show-output", ] - # For instrumented runs, we need separate conclude scripts per commit - # since hyperfine's parameter substitution doesn't work with --conclude - if self.config.instrumented: - base_conclude = self._create_conclude_script_for_commit( - base_commit[:12], tmp_datadir, output_dir - ) - head_conclude = self._create_conclude_script_for_commit( - head_commit[:12], tmp_datadir, output_dir - ) - # We'll handle conclude differently - see below - - # Command names - cmd.append(f"--command-name=base ({base_commit[:12]})") - cmd.append(f"--command-name=head ({head_commit[:12]})") + # Add command names and build commands + for name, binary_path in binaries: + cmd.append(f"--command-name={name}") # Build the actual commands to benchmark - base_cmd = self._build_bitcoind_cmd(base_binary, tmp_datadir) - head_cmd = self._build_bitcoind_cmd(head_binary, tmp_datadir) + for name, binary_path in binaries: + bitcoind_cmd = self._build_bitcoind_cmd(binary_path, tmp_datadir) - # For instrumented runs, append the conclude logic to each command - if self.config.instrumented: - base_cmd += f" && {base_conclude}" - head_cmd += f" && {head_conclude}" + # For instrumented runs, append the conclude logic to each command + if self.config.instrumented: + conclude = self._create_conclude_commands(name, tmp_datadir, output_dir) + bitcoind_cmd += f" && {conclude}" - cmd.append(base_cmd) - cmd.append(head_cmd) + cmd.append(bitcoind_cmd) return cmd - def _create_conclude_script_for_commit( + def _create_conclude_commands( self, - commit: str, + name: str, tmp_datadir: Path, output_dir: Path, ) -> str: - """Create inline conclude commands for a specific commit.""" + """Create inline conclude commands for a specific binary.""" # Return shell commands to run after each benchmark commands = [] # Move flamegraph if exists - commands.append(f'if [ -e flamegraph.svg ]; then mv flamegraph.svg "{output_dir}/{commit}-flamegraph.svg"; fi') + commands.append( + f'if [ -e flamegraph.svg ]; then mv flamegraph.svg "{output_dir}/{name}-flamegraph.svg"; fi' + ) # Copy debug log if exists commands.append( f'debug_log=$(find "{tmp_datadir}" -name debug.log -print -quit); ' - f'if [ -n "$debug_log" ]; then cp "$debug_log" "{output_dir}/{commit}-debug.log"; fi' + f'if [ -n "$debug_log" ]; then cp "$debug_log" "{output_dir}/{name}-debug.log"; fi' ) return " && ".join(commands) diff --git a/bench/build.py b/bench/build.py index 71e7e30b3288..6187263a73de 100644 --- a/bench/build.py +++ b/bench/build.py @@ -18,18 +18,35 @@ logger = logging.getLogger(__name__) +@dataclass +class BuiltBinary: + """A single built binary.""" + + name: str + path: Path + commit: str + + @dataclass class BuildResult: """Result of the build phase.""" - base_binary: Path - head_binary: Path - base_commit: str - head_commit: str + binaries: list[BuiltBinary] + + +def parse_commit_spec(spec: str) -> tuple[str, str | None]: + """Parse a commit spec like 'abc123:name' or 'abc123'. + + Returns (commit, name) where name may be None. + """ + if ":" in spec: + commit, name = spec.split(":", 1) + return commit, name + return spec, None class BuildPhase: - """Build bitcoind binaries at two commits for comparison.""" + """Build bitcoind binaries at specified commits.""" def __init__( self, @@ -43,94 +60,101 @@ def __init__( def run( self, - base_commit: str, - head_commit: str, - binaries_dir: Path | None = None, + commit_specs: list[str], + output_dir: Path | None = None, ) -> BuildResult: - """Build bitcoind at both commits. + """Build bitcoind at given commits. Args: - base_commit: Git ref for base (comparison) commit - head_commit: Git ref for head (new) commit - binaries_dir: Where to store binaries (default: ./binaries) + commit_specs: List of commit specs like 'abc123:name' or 'abc123' + output_dir: Where to store binaries (default: ./binaries) Returns: - BuildResult with paths to built binaries + BuildResult with list of built binaries """ # Check prerequisites errors = self.capabilities.check_for_build() if errors: raise RuntimeError("Build prerequisites not met:\n" + "\n".join(errors)) - binaries_dir = binaries_dir or Path(self.config.binaries_dir) - - # Resolve commits to full hashes - base_hash = git_rev_parse(base_commit, self.repo_path) - head_hash = git_rev_parse(head_commit, self.repo_path) - - logger.info("Building binaries for comparison:") - logger.info(f" Base: {base_hash[:12]} ({base_commit})") - logger.info(f" Head: {head_hash[:12]} ({head_commit})") + output_dir = output_dir or Path(self.config.binaries_dir) + + # Parse commit specs and resolve to full hashes + commits: list[tuple[str, str, str]] = [] # (commit_hash, name, original_spec) + for spec in commit_specs: + commit, name = parse_commit_spec(spec) + commit_hash = git_rev_parse(commit, self.repo_path) + # Default name to short hash if not provided + if name is None: + name = commit_hash[:12] + commits.append((commit_hash, name, spec)) + + logger.info(f"Building {len(commits)} binary(ies):") + for commit_hash, name, spec in commits: + logger.info(f" {name}: {commit_hash[:12]} ({spec})") logger.info(f" Repo: {self.repo_path}") - logger.info(f" Output: {binaries_dir}") - - # Setup output directories - base_dir = binaries_dir / "base" - head_dir = binaries_dir / "head" - base_dir.mkdir(parents=True, exist_ok=True) - head_dir.mkdir(parents=True, exist_ok=True) - - base_binary = base_dir / "bitcoind" - head_binary = head_dir / "bitcoind" + logger.info(f" Output: {output_dir}") # Check if we can skip existing builds - if self.config.skip_existing: - if base_binary.exists() and head_binary.exists(): - logger.info( - "Both binaries exist and --skip-existing set, skipping build" - ) - return BuildResult( - base_binary=base_binary, - head_binary=head_binary, - base_commit=base_hash, - head_commit=head_hash, - ) + binaries_to_build: list[ + tuple[str, str, Path] + ] = [] # (commit_hash, name, output_path) + for commit_hash, name, _spec in commits: + binary_dir = output_dir / name + binary_dir.mkdir(parents=True, exist_ok=True) + binary_path = binary_dir / "bitcoind" + + if self.config.skip_existing and binary_path.exists(): + logger.info(f" Skipping {name} - binary exists") + else: + binaries_to_build.append((commit_hash, name, binary_path)) + + if not binaries_to_build: + logger.info("All binaries exist and --skip-existing set, skipping build") + return BuildResult( + binaries=[ + BuiltBinary( + name=name, + path=output_dir / name / "bitcoind", + commit=commit_hash, + ) + for commit_hash, name, _spec in commits + ] + ) # Save git state for restoration git_state = GitState(self.repo_path) git_state.save() - try: - # Build both commits - builds = [ - ("base", base_hash, base_binary), - ("head", head_hash, head_binary), - ] - - for name, commit, output_path in builds: - if self.config.skip_existing and output_path.exists(): - logger.info(f"Skipping {name} build - binary exists") - continue + built_binaries: list[BuiltBinary] = [] - self._build_commit(name, commit, output_path) + try: + for commit_hash, name, output_path in binaries_to_build: + self._build_commit(name, commit_hash, output_path) + built_binaries.append( + BuiltBinary(name=name, path=output_path, commit=commit_hash) + ) finally: # Always restore git state git_state.restore() - return BuildResult( - base_binary=base_binary, - head_binary=head_binary, - base_commit=base_hash, - head_commit=head_hash, - ) + # Include skipped binaries in result + all_binaries = [] + for commit_hash, name, _spec in commits: + binary_path = output_dir / name / "bitcoind" + all_binaries.append( + BuiltBinary(name=name, path=binary_path, commit=commit_hash) + ) + + return BuildResult(binaries=all_binaries) def _build_commit(self, name: str, commit: str, output_path: Path) -> None: """Build bitcoind for a single commit.""" logger.info(f"Building {name} ({commit[:12]})") if self.config.dry_run: - logger.info(f"[DRY RUN] Would build {commit[:12]} -> {output_path}") + logger.info(f" [DRY RUN] Would build {commit[:12]} -> {output_path}") return # Checkout the commit diff --git a/bench/capabilities.py b/bench/capabilities.py index b01ce2f3a711..31b6bd59f05f 100644 --- a/bench/capabilities.py +++ b/bench/capabilities.py @@ -8,7 +8,6 @@ import os import shutil -import subprocess from dataclasses import dataclass from pathlib import Path diff --git a/bench/compare.py b/bench/compare.py new file mode 100644 index 000000000000..fac328841634 --- /dev/null +++ b/bench/compare.py @@ -0,0 +1,180 @@ +"""Compare phase - compare benchmark results from multiple runs.""" + +from __future__ import annotations + +import json +import logging +from dataclasses import dataclass +from pathlib import Path + +logger = logging.getLogger(__name__) + + +@dataclass +class BenchmarkEntry: + """A single benchmark entry from results.json.""" + + command: str + mean: float + stddev: float | None + user: float + system: float + min: float + max: float + times: list[float] + + +@dataclass +class Comparison: + """Comparison of one entry against the baseline.""" + + name: str + mean: float + baseline_mean: float + speedup_percent: float + stddev: float | None + + +@dataclass +class CompareResult: + """Result of comparison.""" + + baseline: str + comparisons: list[Comparison] + + +class ComparePhase: + """Compare benchmark results from multiple results.json files.""" + + def run( + self, + results_files: list[Path], + baseline: str | None = None, + ) -> CompareResult: + """Compare benchmark results. + + Args: + results_files: List of results.json files to compare + baseline: Name of the baseline entry (default: first entry) + + Returns: + CompareResult with comparison data + """ + if not results_files: + raise ValueError("At least one results file is required") + + # Load all entries from all files + all_entries: list[BenchmarkEntry] = [] + for results_file in results_files: + if not results_file.exists(): + raise FileNotFoundError(f"Results file not found: {results_file}") + + logger.info(f"Loading results from: {results_file}") + with open(results_file) as f: + data = json.load(f) + + entries = self._parse_results(data) + logger.info(f" Found {len(entries)} entries") + all_entries.extend(entries) + + if not all_entries: + raise ValueError("No benchmark entries found in results files") + + # Determine baseline + if baseline is None: + baseline = all_entries[0].command + logger.info(f"Using baseline: {baseline}") + + # Find baseline entry + baseline_entry = None + for entry in all_entries: + if entry.command == baseline: + baseline_entry = entry + break + + if baseline_entry is None: + available = [e.command for e in all_entries] + raise ValueError( + f"Baseline '{baseline}' not found. Available: {', '.join(available)}" + ) + + # Calculate comparisons + comparisons: list[Comparison] = [] + for entry in all_entries: + if entry.command == baseline: + continue + + speedup = self._calculate_speedup(baseline_entry.mean, entry.mean) + comparisons.append( + Comparison( + name=entry.command, + mean=entry.mean, + baseline_mean=baseline_entry.mean, + speedup_percent=speedup, + stddev=entry.stddev, + ) + ) + + # Log results + logger.info("Comparison results:") + logger.info(f" Baseline ({baseline}): {baseline_entry.mean:.3f}s") + for comp in comparisons: + sign = "+" if comp.speedup_percent > 0 else "" + logger.info( + f" {comp.name}: {comp.mean:.3f}s ({sign}{comp.speedup_percent:.1f}%)" + ) + + return CompareResult( + baseline=baseline, + comparisons=comparisons, + ) + + def _parse_results(self, data: dict) -> list[BenchmarkEntry]: + """Parse results from hyperfine JSON output.""" + entries = [] + + results = data.get("results", []) + for result in results: + entries.append( + BenchmarkEntry( + command=result.get("command", "unknown"), + mean=result.get("mean", 0), + stddev=result.get("stddev"), + user=result.get("user", 0), + system=result.get("system", 0), + min=result.get("min", 0), + max=result.get("max", 0), + times=result.get("times", []), + ) + ) + + return entries + + def _calculate_speedup(self, baseline_mean: float, other_mean: float) -> float: + """Calculate speedup percentage. + + Positive = faster than baseline + Negative = slower than baseline + """ + if baseline_mean == 0: + return 0.0 + return round(((baseline_mean - other_mean) / baseline_mean) * 100, 1) + + def to_json(self, result: CompareResult) -> str: + """Convert comparison result to JSON.""" + return json.dumps( + { + "baseline": result.baseline, + "comparisons": [ + { + "name": c.name, + "mean": c.mean, + "baseline_mean": c.baseline_mean, + "speedup_percent": c.speedup_percent, + "stddev": c.stddev, + } + for c in result.comparisons + ], + }, + indent=2, + ) diff --git a/bench/report.py b/bench/report.py index 16445bde5246..25e3f9530be0 100644 --- a/bench/report.py +++ b/bench/report.py @@ -237,30 +237,29 @@ def _parse_results(self, data: dict) -> list[BenchmarkRun]: return runs def _calculate_speedups(self, runs: list[BenchmarkRun]) -> dict[str, float]: - """Calculate speedup percentages for each network.""" + """Calculate speedup percentages. + + Uses the first entry as baseline and compares all others against it. + Returns a dict mapping command name to speedup percentage. + """ speedups = {} - # Group by network - by_network: dict[str, list[BenchmarkRun]] = {} - for run in runs: - if run.network not in by_network: - by_network[run.network] = [] - by_network[run.network].append(run) + if len(runs) < 2: + return speedups - # Calculate speedup for each network - for network, network_runs in by_network.items(): - base_mean = None - head_mean = None + # Use first run as baseline + baseline = runs[0] + baseline_mean = baseline.mean - for run in network_runs: - if "base" in run.command.lower(): - base_mean = run.mean - elif "head" in run.command.lower(): - head_mean = run.mean + if baseline_mean <= 0: + return speedups - if base_mean and head_mean and base_mean > 0: - speedup = ((base_mean - head_mean) / base_mean) * 100 - speedups[network] = round(speedup, 1) + # Calculate speedup for each other run + for run in runs[1:]: + speedup = ((baseline_mean - run.mean) / baseline_mean) * 100 + # Use command name as key, extracting just the name part + name = run.command + speedups[name] = round(speedup, 1) return speedups @@ -300,17 +299,27 @@ def _generate_html( # Generate speedup rows speedup_rows = "" - for network, speedup in speedups.items(): + if sorted_runs: + # Add baseline row + baseline = sorted_runs[0] + speedup_rows += f""" + + {baseline.command} (baseline) + - + + """ + for name, speedup in speedups.items(): color_class = "" if speedup > 0: color_class = "text-green-600" elif speedup < 0: color_class = "text-red-600" + sign = "+" if speedup > 0 else "" speedup_rows += f""" - {network} - {speedup}% + {name} + {sign}{speedup}% """ @@ -326,6 +335,7 @@ def _generate_html( def _linkify_commit(self, command: str) -> str: """Convert commit hashes in command to links.""" + def replace_commit(match): commit = match.group(1) short_commit = commit[:8] if len(commit) > 8 else commit diff --git a/justfile b/justfile index 51faef3160da..d128c7e8b195 100644 --- a/justfile +++ b/justfile @@ -10,34 +10,55 @@ default: # Test instrumented run using signet (includes report generation) [group('local')] test-instrumented base head datadir: - nix develop --command python3 bench.py --profile quick full --chain signet --instrumented --skip-existing --datadir {{ datadir }} {{ base }} {{ head }} + nix develop --command python3 bench.py build --skip-existing {{ base }}:base {{ head }}:head + nix develop --command python3 bench.py --profile quick run \ + --chain signet \ + --instrumented \ + --datadir {{ datadir }} \ + base:./binaries/base/bitcoind \ + head:./binaries/head/bitcoind nix develop --command python3 bench.py report bench-output/ bench-output/ # Test uninstrumented run using signet [group('local')] test-uninstrumented base head datadir: - nix develop --command python3 bench.py --profile quick full --chain signet --skip-existing --datadir {{ datadir }} {{ base }} {{ head }} + nix develop --command python3 bench.py build --skip-existing {{ base }}:base {{ head }}:head + nix develop --command python3 bench.py --profile quick run \ + --chain signet \ + --datadir {{ datadir }} \ + base:./binaries/base/bitcoind \ + head:./binaries/head/bitcoind # Full benchmark with instrumentation (flamegraphs + plots) [group('local')] instrumented base head datadir: - python3 bench.py --profile quick full --instrumented --datadir {{ datadir }} {{ base }} {{ head }} + python3 bench.py build {{ base }}:base {{ head }}:head + python3 bench.py --profile quick run \ + --instrumented \ + --datadir {{ datadir }} \ + base:./binaries/base/bitcoind \ + head:./binaries/head/bitcoind # Just build binaries (useful for incremental testing) [group('local')] -build base head: - python3 bench.py build {{ base }} {{ head }} +build *commits: + python3 bench.py build {{ commits }} # Run benchmark with pre-built binaries [group('local')] -run base head datadir: - python3 bench.py run --datadir {{ datadir }} {{ base }} {{ head }} +run datadir *binaries: + python3 bench.py run --datadir {{ datadir }} {{ binaries }} # Generate plots from a debug.log file [group('local')] analyze commit logfile output_dir="./plots": python3 bench.py analyze {{ commit }} {{ logfile }} --output-dir {{ output_dir }} +# Compare benchmark results +[group('local')] +compare *results_files: + python3 bench.py compare {{ results_files }} + # Generate HTML report from benchmark results [group('local')] report input_dir output_dir: @@ -50,30 +71,30 @@ report input_dir output_dir: # Build binaries for CI [group('ci')] ci-build base_commit head_commit binaries_dir: - python3 bench.py build --binaries-dir {{ binaries_dir }} {{ base_commit }} {{ head_commit }} + python3 bench.py build -o {{ binaries_dir }} {{ base_commit }}:base {{ head_commit }}:head # Run uninstrumented benchmarks for CI [group('ci')] -ci-run base_commit head_commit datadir tmp_datadir output_dir dbcache binaries_dir: +ci-run datadir tmp_datadir output_dir dbcache binaries_dir: python3 bench.py --profile ci run \ - --binaries-dir {{ binaries_dir }} \ --datadir {{ datadir }} \ --tmp-datadir {{ tmp_datadir }} \ --output-dir {{ output_dir }} \ --dbcache {{ dbcache }} \ - {{ base_commit }} {{ head_commit }} + base:{{ binaries_dir }}/base/bitcoind \ + head:{{ binaries_dir }}/head/bitcoind # Run instrumented benchmarks for CI [group('ci')] -ci-run-instrumented base_commit head_commit datadir tmp_datadir output_dir dbcache binaries_dir: +ci-run-instrumented datadir tmp_datadir output_dir dbcache binaries_dir: python3 bench.py --profile ci run \ --instrumented \ - --binaries-dir {{ binaries_dir }} \ --datadir {{ datadir }} \ --tmp-datadir {{ tmp_datadir }} \ --output-dir {{ output_dir }} \ --dbcache {{ dbcache }} \ - {{ base_commit }} {{ head_commit }} + base:{{ binaries_dir }}/base/bitcoind \ + head:{{ binaries_dir }}/head/bitcoind # ============================================================================ # Git helpers From 448f60c6ad684c794f4438a3d0e98aa3765bc8e9 Mon Sep 17 00:00:00 2001 From: will Date: Tue, 9 Dec 2025 11:33:29 +0000 Subject: [PATCH 39/48] patch dynamic interpreter of binary if needed --- bench.toml | 2 +- bench/benchmark.py | 7 +++ bench/patchelf.py | 135 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 143 insertions(+), 1 deletion(-) create mode 100644 bench/patchelf.py diff --git a/bench.toml b/bench.toml index 0513f4a0abf5..9a61bd8048f0 100644 --- a/bench.toml +++ b/bench.toml @@ -17,7 +17,7 @@ output_dir = "./bench-output" # Usage: bench.py --profile quick full HEAD~1 HEAD [profiles.quick] -stop_height = 1100 +stop_height = 2000 runs = 3 [profiles.full] diff --git a/bench/benchmark.py b/bench/benchmark.py index 5dc25a0430e6..788e4e53e94d 100644 --- a/bench/benchmark.py +++ b/bench/benchmark.py @@ -11,6 +11,8 @@ from pathlib import Path from typing import TYPE_CHECKING +from .patchelf import ensure_binary_runnable + if TYPE_CHECKING: from .capabilities import Capabilities from .config import Config @@ -89,6 +91,11 @@ def run( if not path.exists(): raise FileNotFoundError(f"Binary not found: {path} ({name})") + # Ensure binaries can run on this system (patches guix binaries on NixOS) + for name, path in binaries: + if not ensure_binary_runnable(path): + raise RuntimeError(f"Binary {name} at {path} cannot be made runnable") + # Check prerequisites errors = self.capabilities.check_for_run(self.config.instrumented) if errors: diff --git a/bench/patchelf.py b/bench/patchelf.py new file mode 100644 index 000000000000..6da1e00867cf --- /dev/null +++ b/bench/patchelf.py @@ -0,0 +1,135 @@ +"""Patchelf utilities for fixing guix-built binaries on NixOS.""" + +from __future__ import annotations + +import logging +import os +import subprocess +from pathlib import Path + +logger = logging.getLogger(__name__) + + +def get_nix_interpreter() -> str | None: + """Get the path to the nix store's dynamic linker. + + Returns None if not on NixOS or can't find it. + """ + # Check if we're on NixOS + if not Path("/etc/NIXOS").exists(): + return None + + # Find the interpreter from the current glibc + # We can get this by checking what the current shell uses + try: + result = subprocess.run( + ["patchelf", "--print-interpreter", "/bin/sh"], + capture_output=True, + text=True, + ) + if result.returncode == 0: + interp = result.stdout.strip() + if interp and Path(interp).exists(): + return interp + except FileNotFoundError: + pass + + return None + + +def get_binary_interpreter(binary: Path) -> str | None: + """Get the interpreter (dynamic linker) of a binary.""" + try: + result = subprocess.run( + ["patchelf", "--print-interpreter", str(binary)], + capture_output=True, + text=True, + ) + if result.returncode == 0: + return result.stdout.strip() + except FileNotFoundError: + logger.debug("patchelf not found") + return None + + +def needs_patching(binary: Path) -> bool: + """Check if a binary needs to be patched for NixOS. + + Returns True if: + - We're on NixOS + - The binary has a non-nix interpreter (e.g., /lib64/ld-linux-x86-64.so.2) + """ + nix_interp = get_nix_interpreter() + if not nix_interp: + # Not on NixOS, no patching needed + return False + + binary_interp = get_binary_interpreter(binary) + if not binary_interp: + # Can't determine interpreter, assume no patching needed + return False + + # Check if the binary's interpreter is already in the nix store + if binary_interp.startswith("/nix/store/"): + return False + + # Binary uses a non-nix interpreter (e.g., /lib64/...) + return True + + +def patch_binary(binary: Path) -> bool: + """Patch a binary to use the nix store's dynamic linker. + + Returns True if patching was successful or not needed. + """ + if not needs_patching(binary): + logger.debug(f"Binary {binary} does not need patching") + return True + + nix_interp = get_nix_interpreter() + if not nix_interp: + logger.warning("Cannot patch binary: unable to find nix interpreter") + return False + + original_interp = get_binary_interpreter(binary) + logger.info(f"Patching binary: {binary}") + logger.info(f" Original interpreter: {original_interp}") + logger.info(f" New interpreter: {nix_interp}") + + # Make sure binary is writable + try: + os.chmod(binary, 0o755) + except OSError as e: + logger.warning(f"Could not make binary writable: {e}") + + try: + result = subprocess.run( + ["patchelf", "--set-interpreter", nix_interp, str(binary)], + capture_output=True, + text=True, + ) + if result.returncode != 0: + logger.error(f"patchelf failed: {result.stderr}") + return False + logger.info(" Patching successful") + return True + except FileNotFoundError: + logger.error("patchelf not found - install it or use nix develop") + return False + + +def ensure_binary_runnable(binary: Path) -> bool: + """Ensure a binary can run on this system. + + Patches the binary if necessary (on NixOS with non-nix binaries). + Returns True if the binary should be runnable. + """ + if not binary.exists(): + logger.error(f"Binary not found: {binary}") + return False + + # Check if patching is needed and do it + if needs_patching(binary): + return patch_binary(binary) + + return True From febe56d7039a53f2da2b31c3bd8f409622678a92 Mon Sep 17 00:00:00 2001 From: will Date: Tue, 9 Dec 2025 12:19:09 +0000 Subject: [PATCH 40/48] increase toml config prio --- bench/analyze.py | 3 ++- bench/config.py | 55 ++++++++++++++++++++++++++++++++++-------------- bench/report.py | 20 ++++++------------ 3 files changed, 47 insertions(+), 31 deletions(-) diff --git a/bench/analyze.py b/bench/analyze.py index a5f52b4ce035..baedd97d745c 100644 --- a/bench/analyze.py +++ b/bench/analyze.py @@ -432,7 +432,8 @@ def _plot( plt.grid(True) min_x, max_x = min(x), max(x) - plt.xlim(min_x, max_x) + if min_x < max_x: + plt.xlim(min_x, max_x) # Add fork markers for height-based plots if is_height_based: diff --git a/bench/config.py b/bench/config.py index e17c0e31a921..7991fee31bff 100644 --- a/bench/config.py +++ b/bench/config.py @@ -118,10 +118,14 @@ def validate(self) -> list[str]: return errors -def load_toml(path: Path) -> dict[str, Any]: - """Load configuration from TOML file.""" +def load_toml(path: Path) -> tuple[dict[str, Any], dict[str, dict[str, Any]]]: + """Load configuration from TOML file. + + Returns: + Tuple of (base_config, profiles_dict) + """ if not path.exists(): - return {} + return {}, {} with open(path, "rb") as f: data = tomllib.load(f) @@ -133,7 +137,10 @@ def load_toml(path: Path) -> dict[str, Any]: if "paths" in data: result.update(data["paths"]) - return result + # Extract profiles + profiles = data.get("profiles", {}) + + return result, profiles def load_env() -> dict[str, Any]: @@ -154,14 +161,29 @@ def load_env() -> dict[str, Any]: return result -def apply_profile(config: dict[str, Any], profile_name: str) -> dict[str, Any]: - """Apply a named profile to configuration.""" - if profile_name not in PROFILES: - return config +def apply_profile( + config: dict[str, Any], + profile_name: str, + toml_profiles: dict[str, dict[str, Any]] | None = None, +) -> dict[str, Any]: + """Apply a named profile to configuration. + Args: + config: Base configuration dict + profile_name: Name of profile to apply + toml_profiles: Profiles loaded from TOML file (override built-in) + """ result = config.copy() - result.update(PROFILES[profile_name]) result["profile"] = profile_name + + # Apply built-in profile first + if profile_name in PROFILES: + result.update(PROFILES[profile_name]) + + # Then apply TOML profile (overrides built-in) + if toml_profiles and profile_name in toml_profiles: + result.update(toml_profiles[profile_name]) + return result @@ -174,10 +196,11 @@ def build_config( Priority (lowest to highest): 1. Built-in defaults - 2. Config file (bench.toml) - 3. Profile overrides - 4. Environment variables - 5. CLI arguments + 2. Config file (bench.toml) base settings + 3. Built-in profile overrides + 4. Config file profile overrides + 5. Environment variables + 6. CLI arguments """ # Start with defaults config = DEFAULTS.copy() @@ -185,11 +208,11 @@ def build_config( # Load config file if config_file is None: config_file = Path("bench.toml") - file_config = load_toml(config_file) + file_config, toml_profiles = load_toml(config_file) config.update(file_config) - # Apply profile - config = apply_profile(config, profile) + # Apply profile (built-in first, then TOML overrides) + config = apply_profile(config, profile, toml_profiles) # Load environment variables env_config = load_env() diff --git a/bench/report.py b/bench/report.py index 25e3f9530be0..8574f75c4074 100644 --- a/bench/report.py +++ b/bench/report.py @@ -353,20 +353,12 @@ def _generate_graphs_section( graphs_html = "" for run in runs: - commit = run.parameters.get("commit", "") - if not commit: - # Try to extract from command - match = re.search(r"\(([a-f0-9]+)\)", run.command) - if match: - commit = match.group(1) - - if not commit: - continue - - short_commit = commit[:12] if len(commit) > 12 else commit + # Use the command/name directly (e.g., "base", "head") + # This is the name given to the binary in the benchmark + name = run.command # Check for flamegraph - flamegraph_name = f"{short_commit}-flamegraph.svg" + flamegraph_name = f"{name}-flamegraph.svg" flamegraph_path = input_dir / flamegraph_name # Check for plots @@ -376,7 +368,7 @@ def _generate_graphs_section( plot_files = [ p.name for p in plots_dir.iterdir() - if p.name.startswith(f"{short_commit}-") and p.suffix == ".png" + if p.name.startswith(f"{name}-") and p.suffix == ".png" ] if not flamegraph_path.exists() and not plot_files: @@ -384,7 +376,7 @@ def _generate_graphs_section( graphs_html += f"""
    -

    {run.command}

    +

    {name}

    """ if flamegraph_path.exists(): From dfe7c43af27edb3d26e760a28657c1b478c0619c Mon Sep 17 00:00:00 2001 From: will Date: Tue, 9 Dec 2025 14:10:47 +0000 Subject: [PATCH 41/48] use report in publish job --- .github/workflows/publish-results.yml | 335 +++++--------------------- bench.py | 89 ++++++- bench/report.py | 241 ++++++++++++++++-- 3 files changed, 369 insertions(+), 296 deletions(-) diff --git a/.github/workflows/publish-results.yml b/.github/workflows/publish-results.yml index a785374b4308..72b1bdd39527 100644 --- a/.github/workflows/publish-results.yml +++ b/.github/workflows/publish-results.yml @@ -14,12 +14,20 @@ jobs: env: NETWORKS: "mainnet-default-instrumented,mainnet-large-instrumented,mainnet-default-uninstrumented,mainnet-large-uninstrumented" outputs: - speedups: ${{ steps.organize.outputs.speedups }} - pr-number: ${{ steps.organize.outputs.pr-number }} + speedups: ${{ steps.generate.outputs.speedups }} + pr-number: ${{ steps.metadata.outputs.pr-number }} + result-url: ${{ steps.generate.outputs.result-url }} steps: - uses: actions/checkout@v4 with: ref: gh-pages + + - name: Checkout benchcoin tools + uses: actions/checkout@v4 + with: + ref: master + path: benchcoin-tools + - name: Download artifacts env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -29,298 +37,86 @@ jobs: - name: Extract artifacts run: | for network in ${NETWORKS//,/ }; do + # Create network-specific directories with results if [ -d "result-${network}" ]; then mkdir -p "${network}-results" mv "result-${network}/results.json" "${network}-results/" fi + # Copy flamegraphs into network results directory if [ -d "flamegraph-${network}" ]; then - mkdir -p "${network}-flamegraph" - mv "flamegraph-${network}"/* "${network}-flamegraph/" + cp -r "flamegraph-${network}"/* "${network}-results/" 2>/dev/null || true + fi + + # Copy plots into network results directory + if [ -d "pngs-${network}" ]; then + mkdir -p "${network}-results/plots" + cp -r "pngs-${network}"/* "${network}-results/plots/" 2>/dev/null || true fi + # Keep metadata separate for extraction if [ -d "run-metadata-${network}" ]; then mkdir -p "${network}-metadata" mv "run-metadata-${network}"/* "${network}-metadata/" fi + done - if [ -d "pngs-${network}" ]; then - mkdir -p "${network}-plots" - mv "pngs-${network}"/* "${network}-plots/" + - name: Extract metadata + id: metadata + run: | + # Find PR number and run ID from any available metadata + for network in ${NETWORKS//,/ }; do + if [ -f "${network}-metadata/github.json" ]; then + PR_NUMBER=$(jq -r '.event.pull_request.number // "main"' "${network}-metadata/github.json") + RUN_ID=$(jq -r '.run_id' "${network}-metadata/github.json") + echo "pr-number=${PR_NUMBER}" >> $GITHUB_OUTPUT + echo "run-id=${RUN_ID}" >> $GITHUB_OUTPUT + echo "Found metadata: PR=${PR_NUMBER}, Run=${RUN_ID}" + break fi done - - name: Organize results - id: organize - uses: actions/github-script@v7 - with: - script: | - const fs = require('fs'); - const path = require('path'); - const networks = process.env.NETWORKS.split(','); - let prNumber = 'main'; - let runId; - - // First, extract metadata and get PR number - for (const network of networks) { - if (fs.existsSync(`${network}-metadata/github.json`)) { - const metadata = JSON.parse(fs.readFileSync(`${network}-metadata/github.json`, 'utf8')); - prNumber = metadata.event.pull_request?.number || prNumber; - runId = metadata.run_id; - } - } - - if (!runId) { - console.error('No valid metadata found for any network'); - process.exit(1); - } - - // Create directory structure - const resultDir = `results/pr-${prNumber}/${runId}`; - fs.mkdirSync(resultDir, { recursive: true }); - - // Now copy metadata files - for (const network of networks) { - if (fs.existsSync(`${network}-metadata/github.json`)) { - const metadataDir = `${resultDir}/${network}-metadata`; - fs.mkdirSync(metadataDir, { recursive: true }); - fs.copyFileSync(`${network}-metadata/github.json`, `${metadataDir}/github.json`); - } - } - - // Process each network's results - const combinedResults = { - results: [], - speedups: {} - }; - - for (const network of networks) { - if (fs.existsSync(`${network}-results`)) { - const networkResults = JSON.parse(fs.readFileSync(`${network}-results/results.json`, 'utf8')); - let baseMean, headMean; - - // Add network name to each result and collect means - networkResults.results.forEach(result => { - result.network = network; - // Extract commit from command string like "base (364a7bb8701e)" - const commitMatch = result.command.match(/\(([a-f0-9]+)\)/); - if (commitMatch) { - result.parameters = { commit: commitMatch[1] }; - } - combinedResults.results.push(result); - if (result.command.includes('base')) { - baseMean = result.mean; - } else if (result.command.includes('head')) { - headMean = result.mean; - } - }); - - // Calculate speedup if we have both measurements - if (baseMean && headMean) { - const speedup = baseMean > 0 ? ((baseMean - headMean) / baseMean * 100).toFixed(1) : 'N/A'; - combinedResults.speedups[network] = speedup; - } - - // Move flamegraphs - if (fs.existsSync(`${network}-flamegraph`)) { - fs.readdirSync(`${network}-flamegraph`).forEach(file => { - const sourceFile = `${network}-flamegraph/${file}`; - const targetFile = `${resultDir}/${network}-${file}`; - fs.copyFileSync(sourceFile, targetFile); - }); - } - // Move plots - if (fs.existsSync(`${network}-plots`)) { - const targetPlotsDir = `${resultDir}/${network}-plots`; - fs.mkdirSync(targetPlotsDir, { recursive: true }); - fs.readdirSync(`${network}-plots`).forEach(plot => { - const sourcePlot = `${network}-plots/${plot}`; - const targetPlot = `${targetPlotsDir}/${plot}`; - fs.copyFileSync(sourcePlot, targetPlot); - }); - } - } - } - - // Write combined results - fs.writeFileSync(`${resultDir}/results.json`, JSON.stringify(combinedResults, null, 2)); - - // Create index.html for this run - // Sort results by network then by command type (base first) - const sortedResults = combinedResults.results.sort((a, b) => { - if (a.network !== b.network) return a.network.localeCompare(b.network); - const aIsBase = a.command.includes('base'); - const bIsBase = b.command.includes('base'); - return bIsBase - aIsBase; // base first - }); - - const indexHtml = ` - - - Benchmark Results - - - -
    -

    Benchmark Results

    -
    -

    PR #${prNumber} - Run ${runId}

    - - -

    Run Data

    -
    - - - - - - - - - - - - - ${sortedResults.map(result => ` - - - - - - - - - `).join('')} - -
    NetworkCommandMean (s)Std DevUser (s)System (s)
    ${result.network} - ${result.command.replace( - /\((\w+)\)/, - (_, commit) => `(${commit.slice(0, 8)})` - )} - ${result.mean.toFixed(3)}${result.stddev?.toFixed(3) || 'N/A'}${result.user.toFixed(3)}${result.system.toFixed(3)}
    -
    - - -

    Speedup Summary

    -
    - - - - - - - - - ${Object.entries(combinedResults.speedups).map(([network, speedup]) => ` - - - - - `).join('')} - -
    NetworkSpeedup (%)
    ${network}${speedup}%
    -
    - - - ${networks.filter(network => network.includes('instrumented')).map(network => { - const networkResults = combinedResults.results.filter(r => r.network === network); - const graphsHtml = networkResults.map(result => { - const flameGraphFile = `${network}-${result.parameters.commit}-flamegraph.svg`; - const flameGraphPath = `${resultDir}/${network}-${result.parameters.commit}-flamegraph.svg`; - - const plotDir = `${resultDir}/${network}-plots`; - const plots = fs.existsSync(plotDir) - ? fs.readdirSync(plotDir) - .filter(plot => plot.startsWith(`${result.parameters.commit}-`)) - .map(plot => ` - - ${plot} - - `) - .join('') - : ''; - - if (!fs.existsSync(flameGraphPath) && !plots) return ''; - - return ` -
    -

    ${result.command.replace(/\((\w+)\)/, (_, commit) => `(${commit.slice(0, 8)})`)}

    - ${fs.existsSync(flameGraphPath) ? ` - - ` : ''} - ${plots} -
    - `; - }).join(''); - - if (!graphsHtml.trim()) return ''; - - return ` -
    -

    ${network} Graphs

    - ${graphsHtml} -
    - `; - }).join('')} -
    -
    - - `; + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' - fs.writeFileSync(`${resultDir}/index.html`, indexHtml); + - name: Generate report + id: generate + env: + PR_NUMBER: ${{ steps.metadata.outputs.pr-number }} + RUN_ID: ${{ steps.metadata.outputs.run-id }} + run: | + cd benchcoin-tools - // Update main index.html - const prs = fs.readdirSync('results') - .filter(dir => dir.startsWith('pr-')) - .map(dir => ({ - pr: dir.replace('pr-', ''), - runs: fs.readdirSync(`results/${dir}`) - })); + # Build network arguments + NETWORK_ARGS="" + for network in ${NETWORKS//,/ }; do + if [ -d "../${network}-results" ]; then + NETWORK_ARGS="${NETWORK_ARGS} --network ${network}:../${network}-results" + fi + done - const mainIndexHtml = ` - - - Bitcoin Benchmark Results - - - -
    -

    Bitcoin Benchmark Results

    -
    -

    Available Results

    -
      - ${prs.map(({pr, runs}) => ` -
    • PR #${pr} -
        - ${runs.map(run => ` -
      • Run ${run}
        • - `).join('')} -
      -
      • - `).join('')} -
    -
    -
    - - `; + # Generate report + python3 bench.py report \ + ${NETWORK_ARGS} \ + --pr-number "${PR_NUMBER}" \ + --run-id "${RUN_ID}" \ + --update-index \ + "../results/pr-${PR_NUMBER}/${RUN_ID}" - fs.writeFileSync('index.html', mainIndexHtml); + # Read speedups from generated results.json + SPEEDUPS=$(jq -r '.speedups | to_entries | map(select(.key | contains("uninstrumented"))) | map("\(.key): \(.value)%") | join(", ")' "../results/pr-${PR_NUMBER}/${RUN_ID}/results.json") + echo "speedups=${SPEEDUPS}" >> $GITHUB_OUTPUT - // Set outputs for use in PR comment - const resultUrl = `https://${context.repo.owner}.github.io/${context.repo.name}/results/pr-${prNumber}/${runId}/index.html`; - const speedupString = Object.entries(combinedResults.speedups) - .filter(([network]) => network.includes('uninstrumented')) - .map(([network, speedup]) => `${network}: ${speedup}%`) - .join(', '); + RESULT_URL="https://${{ github.repository_owner }}.github.io/${{ github.event.repository.name }}/results/pr-${PR_NUMBER}/${RUN_ID}/index.html" + echo "result-url=${RESULT_URL}" >> $GITHUB_OUTPUT - core.setOutput('result-url', resultUrl); - core.setOutput('speedups', speedupString); - core.setOutput('pr-number', prNumber); - return { url: resultUrl, speedups: speedupString }; - name: Upload Pages artifact uses: actions/upload-pages-artifact@v3 with: path: results + - name: Commit and push to gh-pages run: | git config --global user.name "github-actions[bot]" @@ -328,6 +124,7 @@ jobs: git add results/ index.html git commit -m "Update benchmark results from run ${{ github.event.workflow_run.id }}" git push origin gh-pages + comment-pr: needs: build runs-on: ubuntu-latest @@ -342,5 +139,5 @@ jobs: run: | gh pr comment ${{ needs.build.outputs.pr-number }} \ --repo ${{ github.repository }} \ - --body "📊 Benchmark results for this run (${{ github.event.workflow_run.id }}) will be available at: https://${{ github.repository_owner }}.github.io/${{ github.event.repository.name }}/results/pr-${{ needs.build.outputs.pr-number }}/${{ github.event.workflow_run.id }}/index.html after the github pages \"build and deployment\" action has completed. + --body "📊 Benchmark results for this run (${{ github.event.workflow_run.id }}) will be available at: ${{ needs.build.outputs.result-url }} after the github pages \"build and deployment\" action has completed. 🚀 Speedups: ${{ needs.build.outputs.speedups }}" diff --git a/bench.py b/bench.py index 04acef1a0d51..aba690a25ca2 100755 --- a/bench.py +++ b/bench.py @@ -241,21 +241,52 @@ def cmd_report(args: argparse.Namespace) -> int: if args.verbose: logging.getLogger().setLevel(logging.DEBUG) - input_dir = Path(args.input_dir) output_dir = Path(args.output_dir) - - if not input_dir.exists(): - logger.error(f"Input directory not found: {input_dir}") - return 1 - phase = ReportPhase() try: - result = phase.run( - input_dir=input_dir, - output_dir=output_dir, - title=args.title or "Benchmark Results", - ) + # CI multi-network mode + if args.networks: + network_dirs = {} + for spec in args.networks: + if ":" not in spec: + logger.error(f"Invalid network spec '{spec}': must be NETWORK:PATH") + return 1 + network, path = spec.split(":", 1) + network_dirs[network] = Path(path) + + # Validate directories exist + for network, path in network_dirs.items(): + if not path.exists(): + logger.error(f"Network directory not found: {path} ({network})") + return 1 + + result = phase.run_multi_network( + network_dirs=network_dirs, + output_dir=output_dir, + title=args.title or "Benchmark Results", + pr_number=args.pr_number, + run_id=args.run_id, + ) + + # Update main index if we have a results directory + if args.update_index: + results_base = output_dir.parent.parent # Go up from pr-N/run-id + if results_base.exists(): + phase.update_index(results_base, results_base.parent / "index.html") + else: + # Standard single-directory mode + input_dir = Path(args.input_dir) + + if not input_dir.exists(): + logger.error(f"Input directory not found: {input_dir}") + return 1 + + result = phase.run( + input_dir=input_dir, + output_dir=output_dir, + title=args.title or "Benchmark Results", + ) # Print speedups if result.speedups: @@ -444,13 +475,45 @@ def main() -> int: compare_parser.set_defaults(func=cmd_compare) # Report command - report_parser = subparsers.add_parser("report", help="Generate HTML report") - report_parser.add_argument("input_dir", help="Directory with results.json") + report_parser = subparsers.add_parser( + "report", + help="Generate HTML report", + description="Generate HTML report from benchmark results. " + "Use --network for multi-network CI reports.", + ) + report_parser.add_argument( + "input_dir", + nargs="?", + help="Directory with results.json (for single-network mode)", + ) report_parser.add_argument("output_dir", help="Output directory for report") report_parser.add_argument( "--title", help="Report title", ) + # CI multi-network options + report_parser.add_argument( + "--network", + dest="networks", + action="append", + metavar="NAME:PATH", + help="Network results directory (repeatable, e.g., --network mainnet:./mainnet-results)", + ) + report_parser.add_argument( + "--pr-number", + metavar="N", + help="PR number (for CI reports)", + ) + report_parser.add_argument( + "--run-id", + metavar="ID", + help="Run ID (for CI reports)", + ) + report_parser.add_argument( + "--update-index", + action="store_true", + help="Update main index.html (for CI reports)", + ) report_parser.set_defaults(func=cmd_report) args = parser.parse_args() diff --git a/bench/report.py b/bench/report.py index 8574f75c4074..7f95d0c1ea47 100644 --- a/bench/report.py +++ b/bench/report.py @@ -122,6 +122,103 @@ def __init__( ): self.repo_url = repo_url + def generate_multi_network( + self, + network_dirs: dict[str, Path], + output_dir: Path, + title: str = "Benchmark Results", + pr_number: str | None = None, + run_id: str | None = None, + ) -> ReportResult: + """Generate HTML report from multiple network benchmark results. + + Args: + network_dirs: Dict mapping network name to directory containing results.json + output_dir: Where to write the HTML report + title: Title for the report + pr_number: PR number (for CI reports) + run_id: Run ID (for CI reports) + + Returns: + ReportResult with paths and speedup data + """ + output_dir.mkdir(parents=True, exist_ok=True) + + # Combine results from all networks + all_runs: list[BenchmarkRun] = [] + for network, input_dir in network_dirs.items(): + results_file = input_dir / "results.json" + if not results_file.exists(): + logger.warning( + f"results.json not found in {input_dir} for network {network}" + ) + continue + + with open(results_file) as f: + data = json.load(f) + + # Parse and add network to each run + for result in data.get("results", []): + all_runs.append( + BenchmarkRun( + network=network, + command=result.get("command", ""), + mean=result.get("mean", 0), + stddev=result.get("stddev"), + user=result.get("user", 0), + system=result.get("system", 0), + parameters=result.get("parameters", {}), + ) + ) + + # Copy artifacts from this network + self._copy_network_artifacts(network, input_dir, output_dir) + + if not all_runs: + raise ValueError("No benchmark results found in any network directory") + + # Calculate speedups per network + speedups = self._calculate_speedups_per_network(all_runs) + + # Build title with PR/run info if provided + full_title = title + if pr_number and run_id: + full_title = f"PR #{pr_number} - Run {run_id}" + + # Generate HTML + html = self._generate_html( + all_runs, speedups, full_title, output_dir, output_dir + ) + + # Write report + index_file = output_dir / "index.html" + index_file.write_text(html) + logger.info(f"Generated report: {index_file}") + + # Write combined results.json + combined_results = { + "results": [ + { + "network": run.network, + "command": run.command, + "mean": run.mean, + "stddev": run.stddev, + "user": run.user, + "system": run.system, + } + for run in all_runs + ], + "speedups": speedups, + } + results_file = output_dir / "results.json" + results_file.write_text(json.dumps(combined_results, indent=2)) + + return ReportResult( + output_dir=output_dir, + index_file=index_file, + speedups=speedups, + ) + def generate( self, input_dir: Path, @@ -263,6 +360,59 @@ def _calculate_speedups(self, runs: list[BenchmarkRun]) -> dict[str, float]: return speedups + def _calculate_speedups_per_network( + self, runs: list[BenchmarkRun] + ) -> dict[str, float]: + """Calculate speedup percentages per network. + + For each network, uses 'base' as baseline and calculates speedup for 'head'. + Returns a dict mapping network name to speedup percentage. + """ + speedups = {} + + # Group runs by network + networks: dict[str, list[BenchmarkRun]] = {} + for run in runs: + if run.network not in networks: + networks[run.network] = [] + networks[run.network].append(run) + + # Calculate speedup for each network + for network, network_runs in networks.items(): + base_mean = None + head_mean = None + + for run in network_runs: + if run.command == "base": + base_mean = run.mean + elif run.command == "head": + head_mean = run.mean + + if base_mean and head_mean and base_mean > 0: + speedup = ((base_mean - head_mean) / base_mean) * 100 + speedups[network] = round(speedup, 1) + + return speedups + + def _copy_network_artifacts( + self, network: str, input_dir: Path, output_dir: Path + ) -> None: + """Copy artifacts from a network directory with network prefix.""" + # Copy flamegraphs with network prefix + for svg in input_dir.glob("*-flamegraph.svg"): + dest = output_dir / f"{network}-{svg.name}" + shutil.copy2(svg, dest) + logger.debug(f"Copied {svg.name} as {dest.name}") + + # Copy plots directory with network prefix + plots_dir = input_dir / "plots" + if plots_dir.exists(): + dest_plots = output_dir / f"{network}-plots" + if dest_plots.exists(): + shutil.rmtree(dest_plots) + shutil.copytree(plots_dir, dest_plots) + logger.debug(f"Copied plots to {dest_plots}") + def _generate_html( self, runs: list[BenchmarkRun], @@ -354,40 +504,70 @@ def _generate_graphs_section( for run in runs: # Use the command/name directly (e.g., "base", "head") - # This is the name given to the binary in the benchmark name = run.command + network = run.network + + # Check for flamegraph - try both with and without network prefix + # Network-prefixed: {network}-{name}-flamegraph.svg (for multi-network reports) + # Non-prefixed: {name}-flamegraph.svg (for single-network reports) + flamegraph_name = None + flamegraph_path = None - # Check for flamegraph - flamegraph_name = f"{name}-flamegraph.svg" - flamegraph_path = input_dir / flamegraph_name + network_prefixed = f"{network}-{name}-flamegraph.svg" + non_prefixed = f"{name}-flamegraph.svg" - # Check for plots - plots_dir = input_dir / "plots" + if (output_dir / network_prefixed).exists(): + flamegraph_name = network_prefixed + flamegraph_path = output_dir / network_prefixed + elif (input_dir / non_prefixed).exists(): + flamegraph_name = non_prefixed + flamegraph_path = input_dir / non_prefixed + + # Check for plots - try both network-prefixed and non-prefixed directories plot_files = [] - if plots_dir.exists(): + plots_dir = None + + network_plots_dir = output_dir / f"{network}-plots" + regular_plots_dir = input_dir / "plots" + + if network_plots_dir.exists(): + plots_dir = network_plots_dir + plot_files = [ + p.name + for p in plots_dir.iterdir() + if p.name.startswith(f"{name}-") and p.suffix == ".png" + ] + elif regular_plots_dir.exists(): + plots_dir = regular_plots_dir plot_files = [ p.name for p in plots_dir.iterdir() if p.name.startswith(f"{name}-") and p.suffix == ".png" ] - if not flamegraph_path.exists() and not plot_files: + if not flamegraph_path and not plot_files: continue + # Build display label + display_label = f"{network} - {name}" if network != "default" else name + graphs_html += f"""
    -

    {name}

    +

    {display_label}

    """ - if flamegraph_path.exists(): + if flamegraph_path: graphs_html += f""" """ - for plot in sorted(plot_files): - graphs_html += f""" - - {plot} + if plot_files and plots_dir: + # Determine the relative path for plots + plots_rel_path = plots_dir.name + for plot in sorted(plot_files): + graphs_html += f""" + + {plot} """ @@ -449,3 +629,36 @@ def run( ReportResult with paths and speedup data """ return self.generator.generate(input_dir, output_dir, title) + + def run_multi_network( + self, + network_dirs: dict[str, Path], + output_dir: Path, + title: str = "Benchmark Results", + pr_number: str | None = None, + run_id: str | None = None, + ) -> ReportResult: + """Generate report from multiple network benchmark results. + + Args: + network_dirs: Dict mapping network name to directory containing results.json + output_dir: Where to write the HTML report + title: Title for the report + pr_number: PR number (for CI reports) + run_id: Run ID (for CI reports) + + Returns: + ReportResult with paths and speedup data + """ + return self.generator.generate_multi_network( + network_dirs, output_dir, title, pr_number, run_id + ) + + def update_index(self, results_dir: Path, output_file: Path) -> None: + """Update the main index.html listing all results. + + Args: + results_dir: Directory containing pr-* subdirectories + output_file: Where to write index.html + """ + self.generator.generate_index(results_dir, output_file) From d887ff9955fa74a10e084411a3b0b4e9564bb46a Mon Sep 17 00:00:00 2001 From: will Date: Tue, 9 Dec 2025 16:25:04 +0000 Subject: [PATCH 42/48] remove instrumented from speedups --- bench/report.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bench/report.py b/bench/report.py index 7f95d0c1ea47..41562a9a6158 100644 --- a/bench/report.py +++ b/bench/report.py @@ -459,6 +459,10 @@ def _generate_html( """ for name, speedup in speedups.items(): + # Skip instrumented runs in speedup summary + if name.lower().endswith("-instrumented"): + continue + color_class = "" if speedup > 0: color_class = "text-green-600" From d9d7b462b314709c2331c5755ab36a07162252cf Mon Sep 17 00:00:00 2001 From: will Date: Wed, 10 Dec 2025 09:16:52 +0000 Subject: [PATCH 43/48] remove unneeded python project files --- pyproject.toml | 15 --- requirements.txt | 28 ------ uv.lock | 251 ----------------------------------------------- 3 files changed, 294 deletions(-) delete mode 100644 pyproject.toml delete mode 100644 requirements.txt delete mode 100644 uv.lock diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index 26605fc84930..000000000000 --- a/pyproject.toml +++ /dev/null @@ -1,15 +0,0 @@ -[project] -name = "bitcoin-core-deps" -version = "0.1.0" -dependencies = [ - "codespell==2.2.6", - "lief==0.13.2", - "mypy==1.4.1", - "pyzmq==25.1.0", - # Removing in favour of packaged nixpkgs bin which is not dynamically linked - # "ruff==0.5.5", - "vulture==2.6", - "pyperf==2.8.0", - "matplotlib==3.8.0", - "numpy==1.26.0" -] diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index c9b220b6fe46..000000000000 --- a/requirements.txt +++ /dev/null @@ -1,28 +0,0 @@ -# This file was autogenerated by uv via the following command: -# uv pip compile pyproject.toml -o requirements.txt -codespell==2.2.6 - # via bitcoin-core-deps (pyproject.toml) -lief==0.13.2 - # via bitcoin-core-deps (pyproject.toml) -matplotlib==3.8.0 - # via bitcoin-core-deps (pyproject.toml) -mypy==1.4.1 - # via bitcoin-core-deps (pyproject.toml) -mypy-extensions==1.0.0 - # via mypy -numpy==1.26.0 - # via bitcoin-core-deps (pyproject.toml) -psutil==6.1.0 - # via pyperf -pyperf==2.8.0 - # via bitcoin-core-deps (pyproject.toml) -pyzmq==25.1.0 - # via bitcoin-core-deps (pyproject.toml) -toml==0.10.2 - # via vulture -tomli==2.0.2 - # via mypy -typing-extensions==4.12.2 - # via mypy -vulture==2.6 - # via bitcoin-core-deps (pyproject.toml) diff --git a/uv.lock b/uv.lock deleted file mode 100644 index 090e5f1cb4f1..000000000000 --- a/uv.lock +++ /dev/null @@ -1,251 +0,0 @@ -version = 1 -requires-python = ">=3.10" - -[[package]] -name = "bitcoin-core-deps" -version = "0.1.0" -source = { virtual = "." } -dependencies = [ - { name = "codespell" }, - { name = "lief" }, - { name = "mypy" }, - { name = "pyperf" }, - { name = "pyzmq" }, - { name = "vulture" }, -] - -[package.metadata] -requires-dist = [ - { name = "codespell", specifier = "==2.2.6" }, - { name = "lief", specifier = "==0.13.2" }, - { name = "mypy", specifier = "==1.4.1" }, - { name = "pyperf" }, - { name = "pyzmq", specifier = "==25.1.0" }, - { name = "vulture", specifier = "==2.6" }, -] - -[[package]] -name = "cffi" -version = "1.17.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pycparser" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/fc/97/c783634659c2920c3fc70419e3af40972dbaf758daa229a7d6ea6135c90d/cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824", size = 516621 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/90/07/f44ca684db4e4f08a3fdc6eeb9a0d15dc6883efc7b8c90357fdbf74e186c/cffi-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14", size = 182191 }, - { url = "https://files.pythonhosted.org/packages/08/fd/cc2fedbd887223f9f5d170c96e57cbf655df9831a6546c1727ae13fa977a/cffi-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67", size = 178592 }, - { url = "https://files.pythonhosted.org/packages/de/cc/4635c320081c78d6ffc2cab0a76025b691a91204f4aa317d568ff9280a2d/cffi-1.17.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:edae79245293e15384b51f88b00613ba9f7198016a5948b5dddf4917d4d26382", size = 426024 }, - { url = "https://files.pythonhosted.org/packages/b6/7b/3b2b250f3aab91abe5f8a51ada1b717935fdaec53f790ad4100fe2ec64d1/cffi-1.17.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45398b671ac6d70e67da8e4224a065cec6a93541bb7aebe1b198a61b58c7b702", size = 448188 }, - { url = "https://files.pythonhosted.org/packages/d3/48/1b9283ebbf0ec065148d8de05d647a986c5f22586b18120020452fff8f5d/cffi-1.17.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ad9413ccdeda48c5afdae7e4fa2192157e991ff761e7ab8fdd8926f40b160cc3", size = 455571 }, - { url = "https://files.pythonhosted.org/packages/40/87/3b8452525437b40f39ca7ff70276679772ee7e8b394934ff60e63b7b090c/cffi-1.17.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5da5719280082ac6bd9aa7becb3938dc9f9cbd57fac7d2871717b1feb0902ab6", size = 436687 }, - { url = "https://files.pythonhosted.org/packages/8d/fb/4da72871d177d63649ac449aec2e8a29efe0274035880c7af59101ca2232/cffi-1.17.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bb1a08b8008b281856e5971307cc386a8e9c5b625ac297e853d36da6efe9c17", size = 446211 }, - { url = "https://files.pythonhosted.org/packages/ab/a0/62f00bcb411332106c02b663b26f3545a9ef136f80d5df746c05878f8c4b/cffi-1.17.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:045d61c734659cc045141be4bae381a41d89b741f795af1dd018bfb532fd0df8", size = 461325 }, - { url = "https://files.pythonhosted.org/packages/36/83/76127035ed2e7e27b0787604d99da630ac3123bfb02d8e80c633f218a11d/cffi-1.17.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6883e737d7d9e4899a8a695e00ec36bd4e5e4f18fabe0aca0efe0a4b44cdb13e", size = 438784 }, - { url = "https://files.pythonhosted.org/packages/21/81/a6cd025db2f08ac88b901b745c163d884641909641f9b826e8cb87645942/cffi-1.17.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6b8b4a92e1c65048ff98cfe1f735ef8f1ceb72e3d5f0c25fdb12087a23da22be", size = 461564 }, - { url = "https://files.pythonhosted.org/packages/f8/fe/4d41c2f200c4a457933dbd98d3cf4e911870877bd94d9656cc0fcb390681/cffi-1.17.1-cp310-cp310-win32.whl", hash = "sha256:c9c3d058ebabb74db66e431095118094d06abf53284d9c81f27300d0e0d8bc7c", size = 171804 }, - { url = "https://files.pythonhosted.org/packages/d1/b6/0b0f5ab93b0df4acc49cae758c81fe4e5ef26c3ae2e10cc69249dfd8b3ab/cffi-1.17.1-cp310-cp310-win_amd64.whl", hash = "sha256:0f048dcf80db46f0098ccac01132761580d28e28bc0f78ae0d58048063317e15", size = 181299 }, - { url = "https://files.pythonhosted.org/packages/6b/f4/927e3a8899e52a27fa57a48607ff7dc91a9ebe97399b357b85a0c7892e00/cffi-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a45e3c6913c5b87b3ff120dcdc03f6131fa0065027d0ed7ee6190736a74cd401", size = 182264 }, - { url = "https://files.pythonhosted.org/packages/6c/f5/6c3a8efe5f503175aaddcbea6ad0d2c96dad6f5abb205750d1b3df44ef29/cffi-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:30c5e0cb5ae493c04c8b42916e52ca38079f1b235c2f8ae5f4527b963c401caf", size = 178651 }, - { url = "https://files.pythonhosted.org/packages/94/dd/a3f0118e688d1b1a57553da23b16bdade96d2f9bcda4d32e7d2838047ff7/cffi-1.17.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f75c7ab1f9e4aca5414ed4d8e5c0e303a34f4421f8a0d47a4d019ceff0ab6af4", size = 445259 }, - { url = "https://files.pythonhosted.org/packages/2e/ea/70ce63780f096e16ce8588efe039d3c4f91deb1dc01e9c73a287939c79a6/cffi-1.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1ed2dd2972641495a3ec98445e09766f077aee98a1c896dcb4ad0d303628e41", size = 469200 }, - { url = "https://files.pythonhosted.org/packages/1c/a0/a4fa9f4f781bda074c3ddd57a572b060fa0df7655d2a4247bbe277200146/cffi-1.17.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:46bf43160c1a35f7ec506d254e5c890f3c03648a4dbac12d624e4490a7046cd1", size = 477235 }, - { url = "https://files.pythonhosted.org/packages/62/12/ce8710b5b8affbcdd5c6e367217c242524ad17a02fe5beec3ee339f69f85/cffi-1.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a24ed04c8ffd54b0729c07cee15a81d964e6fee0e3d4d342a27b020d22959dc6", size = 459721 }, - { url = "https://files.pythonhosted.org/packages/ff/6b/d45873c5e0242196f042d555526f92aa9e0c32355a1be1ff8c27f077fd37/cffi-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:610faea79c43e44c71e1ec53a554553fa22321b65fae24889706c0a84d4ad86d", size = 467242 }, - { url = "https://files.pythonhosted.org/packages/1a/52/d9a0e523a572fbccf2955f5abe883cfa8bcc570d7faeee06336fbd50c9fc/cffi-1.17.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a9b15d491f3ad5d692e11f6b71f7857e7835eb677955c00cc0aefcd0669adaf6", size = 477999 }, - { url = "https://files.pythonhosted.org/packages/44/74/f2a2460684a1a2d00ca799ad880d54652841a780c4c97b87754f660c7603/cffi-1.17.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:de2ea4b5833625383e464549fec1bc395c1bdeeb5f25c4a3a82b5a8c756ec22f", size = 454242 }, - { url = "https://files.pythonhosted.org/packages/f8/4a/34599cac7dfcd888ff54e801afe06a19c17787dfd94495ab0c8d35fe99fb/cffi-1.17.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b", size = 478604 }, - { url = "https://files.pythonhosted.org/packages/34/33/e1b8a1ba29025adbdcda5fb3a36f94c03d771c1b7b12f726ff7fef2ebe36/cffi-1.17.1-cp311-cp311-win32.whl", hash = "sha256:85a950a4ac9c359340d5963966e3e0a94a676bd6245a4b55bc43949eee26a655", size = 171727 }, - { url = "https://files.pythonhosted.org/packages/3d/97/50228be003bb2802627d28ec0627837ac0bf35c90cf769812056f235b2d1/cffi-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:caaf0640ef5f5517f49bc275eca1406b0ffa6aa184892812030f04c2abf589a0", size = 181400 }, - { url = "https://files.pythonhosted.org/packages/5a/84/e94227139ee5fb4d600a7a4927f322e1d4aea6fdc50bd3fca8493caba23f/cffi-1.17.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:805b4371bf7197c329fcb3ead37e710d1bca9da5d583f5073b799d5c5bd1eee4", size = 183178 }, - { url = "https://files.pythonhosted.org/packages/da/ee/fb72c2b48656111c4ef27f0f91da355e130a923473bf5ee75c5643d00cca/cffi-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:733e99bc2df47476e3848417c5a4540522f234dfd4ef3ab7fafdf555b082ec0c", size = 178840 }, - { url = "https://files.pythonhosted.org/packages/cc/b6/db007700f67d151abadf508cbfd6a1884f57eab90b1bb985c4c8c02b0f28/cffi-1.17.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1257bdabf294dceb59f5e70c64a3e2f462c30c7ad68092d01bbbfb1c16b1ba36", size = 454803 }, - { url = "https://files.pythonhosted.org/packages/1a/df/f8d151540d8c200eb1c6fba8cd0dfd40904f1b0682ea705c36e6c2e97ab3/cffi-1.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da95af8214998d77a98cc14e3a3bd00aa191526343078b530ceb0bd710fb48a5", size = 478850 }, - { url = "https://files.pythonhosted.org/packages/28/c0/b31116332a547fd2677ae5b78a2ef662dfc8023d67f41b2a83f7c2aa78b1/cffi-1.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d63afe322132c194cf832bfec0dc69a99fb9bb6bbd550f161a49e9e855cc78ff", size = 485729 }, - { url = "https://files.pythonhosted.org/packages/91/2b/9a1ddfa5c7f13cab007a2c9cc295b70fbbda7cb10a286aa6810338e60ea1/cffi-1.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f79fc4fc25f1c8698ff97788206bb3c2598949bfe0fef03d299eb1b5356ada99", size = 471256 }, - { url = "https://files.pythonhosted.org/packages/b2/d5/da47df7004cb17e4955df6a43d14b3b4ae77737dff8bf7f8f333196717bf/cffi-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b62ce867176a75d03a665bad002af8e6d54644fad99a3c70905c543130e39d93", size = 479424 }, - { url = "https://files.pythonhosted.org/packages/0b/ac/2a28bcf513e93a219c8a4e8e125534f4f6db03e3179ba1c45e949b76212c/cffi-1.17.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:386c8bf53c502fff58903061338ce4f4950cbdcb23e2902d86c0f722b786bbe3", size = 484568 }, - { url = "https://files.pythonhosted.org/packages/d4/38/ca8a4f639065f14ae0f1d9751e70447a261f1a30fa7547a828ae08142465/cffi-1.17.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4ceb10419a9adf4460ea14cfd6bc43d08701f0835e979bf821052f1805850fe8", size = 488736 }, - { url = "https://files.pythonhosted.org/packages/86/c5/28b2d6f799ec0bdecf44dced2ec5ed43e0eb63097b0f58c293583b406582/cffi-1.17.1-cp312-cp312-win32.whl", hash = "sha256:a08d7e755f8ed21095a310a693525137cfe756ce62d066e53f502a83dc550f65", size = 172448 }, - { url = "https://files.pythonhosted.org/packages/50/b9/db34c4755a7bd1cb2d1603ac3863f22bcecbd1ba29e5ee841a4bc510b294/cffi-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:51392eae71afec0d0c8fb1a53b204dbb3bcabcb3c9b807eedf3e1e6ccf2de903", size = 181976 }, - { url = "https://files.pythonhosted.org/packages/8d/f8/dd6c246b148639254dad4d6803eb6a54e8c85c6e11ec9df2cffa87571dbe/cffi-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f3a2b4222ce6b60e2e8b337bb9596923045681d71e5a082783484d845390938e", size = 182989 }, - { url = "https://files.pythonhosted.org/packages/8b/f1/672d303ddf17c24fc83afd712316fda78dc6fce1cd53011b839483e1ecc8/cffi-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0984a4925a435b1da406122d4d7968dd861c1385afe3b45ba82b750f229811e2", size = 178802 }, - { url = "https://files.pythonhosted.org/packages/0e/2d/eab2e858a91fdff70533cab61dcff4a1f55ec60425832ddfdc9cd36bc8af/cffi-1.17.1-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d01b12eeeb4427d3110de311e1774046ad344f5b1a7403101878976ecd7a10f3", size = 454792 }, - { url = "https://files.pythonhosted.org/packages/75/b2/fbaec7c4455c604e29388d55599b99ebcc250a60050610fadde58932b7ee/cffi-1.17.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:706510fe141c86a69c8ddc029c7910003a17353970cff3b904ff0686a5927683", size = 478893 }, - { url = "https://files.pythonhosted.org/packages/4f/b7/6e4a2162178bf1935c336d4da8a9352cccab4d3a5d7914065490f08c0690/cffi-1.17.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de55b766c7aa2e2a3092c51e0483d700341182f08e67c63630d5b6f200bb28e5", size = 485810 }, - { url = "https://files.pythonhosted.org/packages/c7/8a/1d0e4a9c26e54746dc08c2c6c037889124d4f59dffd853a659fa545f1b40/cffi-1.17.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c59d6e989d07460165cc5ad3c61f9fd8f1b4796eacbd81cee78957842b834af4", size = 471200 }, - { url = "https://files.pythonhosted.org/packages/26/9f/1aab65a6c0db35f43c4d1b4f580e8df53914310afc10ae0397d29d697af4/cffi-1.17.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd398dbc6773384a17fe0d3e7eeb8d1a21c2200473ee6806bb5e6a8e62bb73dd", size = 479447 }, - { url = "https://files.pythonhosted.org/packages/5f/e4/fb8b3dd8dc0e98edf1135ff067ae070bb32ef9d509d6cb0f538cd6f7483f/cffi-1.17.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3edc8d958eb099c634dace3c7e16560ae474aa3803a5df240542b305d14e14ed", size = 484358 }, - { url = "https://files.pythonhosted.org/packages/f1/47/d7145bf2dc04684935d57d67dff9d6d795b2ba2796806bb109864be3a151/cffi-1.17.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:72e72408cad3d5419375fc87d289076ee319835bdfa2caad331e377589aebba9", size = 488469 }, - { url = "https://files.pythonhosted.org/packages/bf/ee/f94057fa6426481d663b88637a9a10e859e492c73d0384514a17d78ee205/cffi-1.17.1-cp313-cp313-win32.whl", hash = "sha256:e03eab0a8677fa80d646b5ddece1cbeaf556c313dcfac435ba11f107ba117b5d", size = 172475 }, - { url = "https://files.pythonhosted.org/packages/7c/fc/6a8cb64e5f0324877d503c854da15d76c1e50eb722e320b15345c4d0c6de/cffi-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a", size = 182009 }, -] - -[[package]] -name = "codespell" -version = "2.2.6" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e1/97/df3e00b4d795c96233e35d269c211131c5572503d2270afb6fed7d859cc2/codespell-2.2.6.tar.gz", hash = "sha256:a8c65d8eb3faa03deabab6b3bbe798bea72e1799c7e9e955d57eca4096abcff9", size = 300968 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/46/e0/5437cc96b74467c4df6e13b7128cc482c48bb43146fb4c11cf2bcd604e1f/codespell-2.2.6-py3-none-any.whl", hash = "sha256:9ee9a3e5df0990604013ac2a9f22fa8e57669c827124a2e961fe8a1da4cacc07", size = 301382 }, -] - -[[package]] -name = "lief" -version = "0.13.2" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d7/e2/c4125c279eb2a23ecc86cdb188ed06e9d81a9c700e9412f9be866afc2c7d/lief-0.13.2-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:0390cfaaf0e9aed46bebf26f00f34852768f76bc7f90abf7ceb384566200e5f5", size = 3424746 }, - { url = "https://files.pythonhosted.org/packages/5f/d6/72235d648c6630c37ef52b9f6f4e2f3337842bc4b08c75abcae3052b2c17/lief-0.13.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5581bf0072c1e7a9ea2fb2e2252b8582016e8b298804b5461e552b402c9cd4e9", size = 3249141 }, - { url = "https://files.pythonhosted.org/packages/d7/cc/9895dff094cad3e88636195640b4b47caefe3d300d3f37b653bd109348df/lief-0.13.2-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:dbbf2fb3d7807e815f345c77e287da162e081100f059ec03005995befc295d7f", size = 3793938 }, - { url = "https://files.pythonhosted.org/packages/0d/1b/f4bf63bfce187ae210980bdd1a20ea7d8e080381eef09e7d26c585eaa614/lief-0.13.2-cp310-cp310-manylinux_2_24_x86_64.whl", hash = "sha256:d344d37334c2b488dc02f04cb13c22cd61aa065eeb9bca7424588e0c8c23bdfb", size = 4045328 }, - { url = "https://files.pythonhosted.org/packages/2c/2a/abac2e42c3cc56f2b5020e58b99f700c4d3236d49451607add0f628d737b/lief-0.13.2-cp310-cp310-win32.whl", hash = "sha256:bc041b28b94139843a33c014e355822a9276b35f3c5ae10d82da56bf572f8222", size = 2493454 }, - { url = "https://files.pythonhosted.org/packages/ed/14/34a12787dc4328227e0e84a97db8142aa1e2b33e0aabc538e93abf7d6e5a/lief-0.13.2-cp310-cp310-win_amd64.whl", hash = "sha256:01d4075bbc3541e9dd3ef008045fa1eb128294a0c5b0c1f69ce60d8948d248c7", size = 3089949 }, - { url = "https://files.pythonhosted.org/packages/2e/95/9d7377095fb7cf195aca8f64d9696705c71884dcba16663472ce17139b9c/lief-0.13.2-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:6570dacebe107ad60c2ba0968d1a865d316009d43cc85af3719d3eeb0911abf3", size = 3424752 }, - { url = "https://files.pythonhosted.org/packages/00/2b/7ac8e15ca198a5c50397aec32102e81ef97fd573a4285ee889ec9084d110/lief-0.13.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7ce2e3f7c791efba327c2bb3499dbef81e682027109045a9bae696c62e2aeeb0", size = 3249263 }, - { url = "https://files.pythonhosted.org/packages/d6/8d/b50cc4ad91278015e5ac18fc76f32098ed6887c371bef6f4997af4cb97c9/lief-0.13.2-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:11ab900e0644b6735ecdef2bbd04439b4866a527650fc054470c195d6cfe2917", size = 3792343 }, - { url = "https://files.pythonhosted.org/packages/6b/bd/ea25e9c8ff0a55b5534e5881fa6e5eeca0ed3eeb7c772a276984b8c182d9/lief-0.13.2-cp311-cp311-manylinux_2_24_x86_64.whl", hash = "sha256:042ad2105a136b11a7494b9af8178468e8cb32b8fa2a0a55cb659a5605aeb069", size = 4045112 }, - { url = "https://files.pythonhosted.org/packages/d9/06/ddacd724f65fa8e7eca438c335aa77878a260fbc714cdba252387c33a4cc/lief-0.13.2-cp311-cp311-win32.whl", hash = "sha256:1ce289b6ab3cf4be654270007e8a2c0d2e42116180418c29d3ce83762955de63", size = 2493336 }, - { url = "https://files.pythonhosted.org/packages/82/95/1de9a497946fed9d15f847d8a4a0630dfda6d186c044f8731f53d0d3d758/lief-0.13.2-cp311-cp311-win_amd64.whl", hash = "sha256:eccb248ffb598e410fd2ef7c1f171a3cde57a40c9bb8c4fa15d8e7b90eb4eb2d", size = 3090328 }, -] - -[[package]] -name = "mypy" -version = "1.4.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "mypy-extensions" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b3/28/d8a8233ff167d06108e53b7aefb4a8d7350adbbf9d7abd980f17fdb7a3a6/mypy-1.4.1.tar.gz", hash = "sha256:9bbcd9ab8ea1f2e1c8031c21445b511442cc45c89951e49bbf852cbb70755b1b", size = 2855162 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fb/3b/1c7363863b56c059f60a1dfdca9ac774a22ba64b7a4da0ee58ee53e5243f/mypy-1.4.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:566e72b0cd6598503e48ea610e0052d1b8168e60a46e0bfd34b3acf2d57f96a8", size = 10451043 }, - { url = "https://files.pythonhosted.org/packages/a7/24/6f0df1874118839db1155fed62a4bd7e80c181367ff8ea07d40fbaffcfb4/mypy-1.4.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ca637024ca67ab24a7fd6f65d280572c3794665eaf5edcc7e90a866544076878", size = 9542079 }, - { url = "https://files.pythonhosted.org/packages/04/5c/deeac94fcccd11aa621e6b350df333e1b809b11443774ea67582cc0205da/mypy-1.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0dde1d180cd84f0624c5dcaaa89c89775550a675aff96b5848de78fb11adabcd", size = 11974913 }, - { url = "https://files.pythonhosted.org/packages/e5/2f/de3c455c54e8cf5e37ea38705c1920f2df470389f8fc051084d2dd8c9c59/mypy-1.4.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8c4d8e89aa7de683e2056a581ce63c46a0c41e31bd2b6d34144e2c80f5ea53dc", size = 12044492 }, - { url = "https://files.pythonhosted.org/packages/e7/d3/6f65357dcb68109946de70cd55bd2e60f10114f387471302f48d54ff5dae/mypy-1.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:bfdca17c36ae01a21274a3c387a63aa1aafe72bff976522886869ef131b937f1", size = 8831655 }, - { url = "https://files.pythonhosted.org/packages/94/01/e34e37a044325af4d4af9825c15e8a0d26d89b5a9624b4d0908449d3411b/mypy-1.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7549fbf655e5825d787bbc9ecf6028731973f78088fbca3a1f4145c39ef09462", size = 10338636 }, - { url = "https://files.pythonhosted.org/packages/92/58/ccc0b714ecbd1a64b34d8ce1c38763ff6431de1d82551904ecc3711fbe05/mypy-1.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:98324ec3ecf12296e6422939e54763faedbfcc502ea4a4c38502082711867258", size = 9444172 }, - { url = "https://files.pythonhosted.org/packages/73/72/dfc0b46e6905eafd598e7c48c0c4f2e232647e4e36547425c64e6c850495/mypy-1.4.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:141dedfdbfe8a04142881ff30ce6e6653c9685b354876b12e4fe6c78598b45e2", size = 11855450 }, - { url = "https://files.pythonhosted.org/packages/66/f4/60739a2d336f3adf5628e7c9b920d16e8af6dc078550d615e4ba2a1d7759/mypy-1.4.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8207b7105829eca6f3d774f64a904190bb2231de91b8b186d21ffd98005f14a7", size = 11928679 }, - { url = "https://files.pythonhosted.org/packages/8c/26/6ff2b55bf8b605a4cc898883654c2ca4dd4feedf0bb04ecaacf60d165cde/mypy-1.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:16f0db5b641ba159eff72cff08edc3875f2b62b2fa2bc24f68c1e7a4e8232d01", size = 8831134 }, - { url = "https://files.pythonhosted.org/packages/3d/9a/e13addb8d652cb068f835ac2746d9d42f85b730092f581bb17e2059c28f1/mypy-1.4.1-py3-none-any.whl", hash = "sha256:45d32cec14e7b97af848bddd97d85ea4f0db4d5a149ed9676caa4eb2f7402bb4", size = 2451741 }, -] - -[[package]] -name = "mypy-extensions" -version = "1.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/98/a4/1ab47638b92648243faf97a5aeb6ea83059cc3624972ab6b8d2316078d3f/mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782", size = 4433 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/e2/5d3f6ada4297caebe1a2add3b126fe800c96f56dbe5d1988a2cbe0b267aa/mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d", size = 4695 }, -] - -[[package]] -name = "psutil" -version = "6.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/26/10/2a30b13c61e7cf937f4adf90710776b7918ed0a9c434e2c38224732af310/psutil-6.1.0.tar.gz", hash = "sha256:353815f59a7f64cdaca1c0307ee13558a0512f6db064e92fe833784f08539c7a", size = 508565 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/01/9e/8be43078a171381953cfee33c07c0d628594b5dbfc5157847b85022c2c1b/psutil-6.1.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:6e2dcd475ce8b80522e51d923d10c7871e45f20918e027ab682f94f1c6351688", size = 247762 }, - { url = "https://files.pythonhosted.org/packages/1d/cb/313e80644ea407f04f6602a9e23096540d9dc1878755f3952ea8d3d104be/psutil-6.1.0-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:0895b8414afafc526712c498bd9de2b063deaac4021a3b3c34566283464aff8e", size = 248777 }, - { url = "https://files.pythonhosted.org/packages/65/8e/bcbe2025c587b5d703369b6a75b65d41d1367553da6e3f788aff91eaf5bd/psutil-6.1.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9dcbfce5d89f1d1f2546a2090f4fcf87c7f669d1d90aacb7d7582addece9fb38", size = 284259 }, - { url = "https://files.pythonhosted.org/packages/58/4d/8245e6f76a93c98aab285a43ea71ff1b171bcd90c9d238bf81f7021fb233/psutil-6.1.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:498c6979f9c6637ebc3a73b3f87f9eb1ec24e1ce53a7c5173b8508981614a90b", size = 287255 }, - { url = "https://files.pythonhosted.org/packages/27/c2/d034856ac47e3b3cdfa9720d0e113902e615f4190d5d1bdb8df4b2015fb2/psutil-6.1.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d905186d647b16755a800e7263d43df08b790d709d575105d419f8b6ef65423a", size = 288804 }, - { url = "https://files.pythonhosted.org/packages/ea/55/5389ed243c878725feffc0d6a3bc5ef6764312b6fc7c081faaa2cfa7ef37/psutil-6.1.0-cp37-abi3-win32.whl", hash = "sha256:1ad45a1f5d0b608253b11508f80940985d1d0c8f6111b5cb637533a0e6ddc13e", size = 250386 }, - { url = "https://files.pythonhosted.org/packages/11/91/87fa6f060e649b1e1a7b19a4f5869709fbf750b7c8c262ee776ec32f3028/psutil-6.1.0-cp37-abi3-win_amd64.whl", hash = "sha256:a8fb3752b491d246034fa4d279ff076501588ce8cbcdbb62c32fd7a377d996be", size = 254228 }, -] - -[[package]] -name = "pycparser" -version = "2.22" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1d/b2/31537cf4b1ca988837256c910a668b553fceb8f069bedc4b1c826024b52c/pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6", size = 172736 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/13/a3/a812df4e2dd5696d1f351d58b8fe16a405b234ad2886a0dab9183fb78109/pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc", size = 117552 }, -] - -[[package]] -name = "pyperf" -version = "2.8.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "psutil" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/02/2a/758b3c4cc9843bd385bc595b777345fbf4cd00733b7830cdff43e30002c0/pyperf-2.8.0.tar.gz", hash = "sha256:b30a20465819daf102b6543b512f6799a5a879ff2a123981e6cd732d0e6a7a79", size = 225186 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7f/f7/bb8965520a9b0a3d720b282e67b5cb7f3305b96e4bacaee2794550e67e94/pyperf-2.8.0-py3-none-any.whl", hash = "sha256:1a775b5a09882f18bf876430ef78e07646f773f50774546f5f6a8b34d60e3968", size = 142508 }, -] - -[[package]] -name = "pyzmq" -version = "25.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cffi", marker = "implementation_name == 'pypy'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/64/9c/2b2614b0b86ff703b3a33ea5e044923bd7d100adc8c829d579a9b71ea9e7/pyzmq-25.1.0.tar.gz", hash = "sha256:80c41023465d36280e801564a69cbfce8ae85ff79b080e1913f6e90481fb8957", size = 1224640 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/53/17/6a70f84b79e361af34f6c99064ecf9e87112c4c48b9c7ea78f8e680b57d8/pyzmq-25.1.0-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:1a6169e69034eaa06823da6a93a7739ff38716142b3596c180363dee729d713d", size = 1826810 }, - { url = "https://files.pythonhosted.org/packages/2f/53/fc7dbdd32e275aee0961e2a5bed1bb64223846f959fd6e0c9a39aab08eed/pyzmq-25.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:19d0383b1f18411d137d891cab567de9afa609b214de68b86e20173dc624c101", size = 1236489 }, - { url = "https://files.pythonhosted.org/packages/04/0b/bff5b6c1680e248bad2df8248a060645709fe2aef9689e9f7c81c587bad4/pyzmq-25.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1e931d9a92f628858a50f5bdffdfcf839aebe388b82f9d2ccd5d22a38a789dc", size = 864304 }, - { url = "https://files.pythonhosted.org/packages/5e/9e/32074bd8bcf2a5cf282d8817458fd5479c68b487b6c3a5d4627711ad38f5/pyzmq-25.1.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:97d984b1b2f574bc1bb58296d3c0b64b10e95e7026f8716ed6c0b86d4679843f", size = 1116061 }, - { url = "https://files.pythonhosted.org/packages/fa/fb/a114ba641eb873c165106d3c8ee75eb49d6ea3204168808708d866de360d/pyzmq-25.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:154bddda2a351161474b36dba03bf1463377ec226a13458725183e508840df89", size = 1065090 }, - { url = "https://files.pythonhosted.org/packages/ca/db/f9976803f1a660e753d0f2426065975bad5db8272fd5284efaf488dc0ce1/pyzmq-25.1.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:cb6d161ae94fb35bb518b74bb06b7293299c15ba3bc099dccd6a5b7ae589aee3", size = 1062464 }, - { url = "https://files.pythonhosted.org/packages/94/3a/c3964c0a86c3535ae240799d3b7c8e13527e7a092080dda9012b1401fa86/pyzmq-25.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:90146ab578931e0e2826ee39d0c948d0ea72734378f1898939d18bc9c823fcf9", size = 1391159 }, - { url = "https://files.pythonhosted.org/packages/a1/87/92556ffa8fbe7dc497d847e39d5c46134f9ad047b23f5bcefc8fbd0c2c9c/pyzmq-25.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:831ba20b660b39e39e5ac8603e8193f8fce1ee03a42c84ade89c36a251449d80", size = 1721009 }, - { url = "https://files.pythonhosted.org/packages/66/96/129706be681649f43bde93811416f566acfefcd3fb18156d5df349c360ab/pyzmq-25.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:3a522510e3434e12aff80187144c6df556bb06fe6b9d01b2ecfbd2b5bfa5c60c", size = 1611290 }, - { url = "https://files.pythonhosted.org/packages/64/db/e19f69fe9b1a4e53f6382274f553358e2e7305d2a2b9d9db36087bf52d5e/pyzmq-25.1.0-cp310-cp310-win32.whl", hash = "sha256:be24a5867b8e3b9dd5c241de359a9a5217698ff616ac2daa47713ba2ebe30ad1", size = 880070 }, - { url = "https://files.pythonhosted.org/packages/32/e4/ce4f94009f84c2a688082c2674d490d2e20e0c9058087f5358a2bf29ddf1/pyzmq-25.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:5693dcc4f163481cf79e98cf2d7995c60e43809e325b77a7748d8024b1b7bcba", size = 1137827 }, - { url = "https://files.pythonhosted.org/packages/bb/80/ae792378f98d6d0e39c975c334603d3d2535f7897707fe91f31d37f94fdb/pyzmq-25.1.0-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:13bbe36da3f8aaf2b7ec12696253c0bf6ffe05f4507985a8844a1081db6ec22d", size = 1816147 }, - { url = "https://files.pythonhosted.org/packages/5a/b6/3c2ddd09aa24352e4f6aade53e9b9a1816c0774c844f11b1a2f508ddc0be/pyzmq-25.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:69511d604368f3dc58d4be1b0bad99b61ee92b44afe1cd9b7bd8c5e34ea8248a", size = 1230845 }, - { url = "https://files.pythonhosted.org/packages/26/bb/80535157e8811095901f98688839092afb6dcaf2ff154aa8fa2e575f540d/pyzmq-25.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a983c8694667fd76d793ada77fd36c8317e76aa66eec75be2653cef2ea72883", size = 866042 }, - { url = "https://files.pythonhosted.org/packages/7c/65/bccec1eae7c0e089d90648f350e6c2ff40ccb8c6d1b929548f4cd304b1f7/pyzmq-25.1.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:332616f95eb400492103ab9d542b69d5f0ff628b23129a4bc0a2fd48da6e4e0b", size = 1116285 }, - { url = "https://files.pythonhosted.org/packages/b7/cb/2a36d3eed310efb342fbb7b4adf6b05f46401c4b937154bd1c9b703314e0/pyzmq-25.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58416db767787aedbfd57116714aad6c9ce57215ffa1c3758a52403f7c68cff5", size = 1066280 }, - { url = "https://files.pythonhosted.org/packages/66/f5/15db4c297957f049cd4dcd35eb7fbe9098a72489e0abdb289c529d7327cc/pyzmq-25.1.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:cad9545f5801a125f162d09ec9b724b7ad9b6440151b89645241d0120e119dcc", size = 1061673 }, - { url = "https://files.pythonhosted.org/packages/fa/40/7729719e38324e5e9f2e77f6131fc253f063a3741eab170ef610196098e8/pyzmq-25.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d6128d431b8dfa888bf51c22a04d48bcb3d64431caf02b3cb943269f17fd2994", size = 1393337 }, - { url = "https://files.pythonhosted.org/packages/fd/12/0324dcb2554cd3f2ebb851ddbfbac27c4bb384394ba4a8978dec093fe71d/pyzmq-25.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:2b15247c49d8cbea695b321ae5478d47cffd496a2ec5ef47131a9e79ddd7e46c", size = 1723679 }, - { url = "https://files.pythonhosted.org/packages/04/15/b8ab292f0b74e0440547185fb67167c87454a2b3be429d64de569f7142a2/pyzmq-25.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:442d3efc77ca4d35bee3547a8e08e8d4bb88dadb54a8377014938ba98d2e074a", size = 1612761 }, - { url = "https://files.pythonhosted.org/packages/22/3e/3670e36c6f42e124492ddd2af550ca13bd4a9f1edd562e1ae7c35a1f230b/pyzmq-25.1.0-cp311-cp311-win32.whl", hash = "sha256:65346f507a815a731092421d0d7d60ed551a80d9b75e8b684307d435a5597425", size = 878704 }, - { url = "https://files.pythonhosted.org/packages/a0/db/4e586c563b48dec09b8f7c2728b905e29db61af89b5c58e4eba9ad36fdec/pyzmq-25.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:8b45d722046fea5a5694cba5d86f21f78f0052b40a4bbbbf60128ac55bfcc7b6", size = 1135692 }, -] - -[[package]] -name = "toml" -version = "0.10.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/be/ba/1f744cdc819428fc6b5084ec34d9b30660f6f9daaf70eead706e3203ec3c/toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f", size = 22253 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", size = 16588 }, -] - -[[package]] -name = "tomli" -version = "2.0.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/35/b9/de2a5c0144d7d75a57ff355c0c24054f965b2dc3036456ae03a51ea6264b/tomli-2.0.2.tar.gz", hash = "sha256:d46d457a85337051c36524bc5349dd91b1877838e2979ac5ced3e710ed8a60ed", size = 16096 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cf/db/ce8eda256fa131af12e0a76d481711abe4681b6923c27efb9a255c9e4594/tomli-2.0.2-py3-none-any.whl", hash = "sha256:2ebe24485c53d303f690b0ec092806a085f07af5a5aa1464f3931eec36caaa38", size = 13237 }, -] - -[[package]] -name = "typing-extensions" -version = "4.12.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/df/db/f35a00659bc03fec321ba8bce9420de607a1d37f8342eee1863174c69557/typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8", size = 85321 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/26/9f/ad63fc0248c5379346306f8668cda6e2e2e9c95e01216d2b8ffd9ff037d0/typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d", size = 37438 }, -] - -[[package]] -name = "vulture" -version = "2.6" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "toml" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b9/18/e51a6e575047d19dbcd7394f05b2afa6191fe9ce30bd5bcfb3f850501e0c/vulture-2.6.tar.gz", hash = "sha256:2515fa848181001dc8a73aba6a01a1a17406f5d372f24ec7f7191866f9f4997e", size = 53777 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5e/9d/3c4df0c704ddb5ecf07fcd92cfe6d4a5dc000b7f5459afcb7e98a2ffea1e/vulture-2.6-py2.py3-none-any.whl", hash = "sha256:e792e903ccc063ec4873a8979dcf11b51ea3d65a2d3b31c113d47be48f0cdcae", size = 26494 }, -] From 70bd087c4329d125f964c8a7357124f2714bb298 Mon Sep 17 00:00:00 2001 From: Andrew Toth Date: Sat, 22 Nov 2025 20:15:04 -0500 Subject: [PATCH 44/48] coins: add CoinsViewCacheAsync for parallel input fetching Introduce CoinsViewCacheAsync utility class that manages worker threads to fetch block inputs in parallel. Uses a lock-free MPSC queue design with an atomic counter for work distribution and per-input atomic flags for completion signaling. Provides the same API as CCoinsViewCache, and adds StartFetching and Reset methods to control background fetching. The main thread processes inputs sequentially as they become available, returning fetched coins from the FetchCoin method. If the main thread catches up to the workers, it assists with fetching to maximize parallelism. Co-authored-by: l0rinc Co-authored-by: sedited --- src/coins.cpp | 6 + src/coins.h | 12 +- src/coinsviewcacheasync.h | 247 +++++++++++++++++++++++++ src/test/CMakeLists.txt | 1 + src/test/coinsviewcacheasync_tests.cpp | 203 ++++++++++++++++++++ src/test/fuzz/CMakeLists.txt | 1 + src/test/fuzz/coinsviewcacheasync.cpp | 176 ++++++++++++++++++ 7 files changed, 644 insertions(+), 2 deletions(-) create mode 100644 src/coinsviewcacheasync.h create mode 100644 src/test/coinsviewcacheasync_tests.cpp create mode 100644 src/test/fuzz/coinsviewcacheasync.cpp diff --git a/src/coins.cpp b/src/coins.cpp index 554a3ebe962b..f9ac6dc59862 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -173,6 +173,12 @@ bool CCoinsViewCache::HaveCoinInCache(const COutPoint &outpoint) const { return (it != cacheCoins.end() && !it->second.coin.IsSpent()); } +std::optional CCoinsViewCache::GetPossiblySpentCoinFromCache(const COutPoint& outpoint) const noexcept +{ + if (auto it{cacheCoins.find(outpoint)}; it != cacheCoins.end()) return it->second.coin; + return std::nullopt; +} + uint256 CCoinsViewCache::GetBestBlock() const { if (hashBlock.IsNull()) hashBlock = base->GetBestBlock(); diff --git a/src/coins.h b/src/coins.h index 2fcc764a3fdf..9ae602165dcc 100644 --- a/src/coins.h +++ b/src/coins.h @@ -401,6 +401,14 @@ class CCoinsViewCache : public CCoinsViewBacked */ bool HaveCoinInCache(const COutPoint &outpoint) const; + /** + * Retrieve the coin from the cache even if it is spent, without calling + * the backing CCoinsView if no coin exists. + * Used in CoinsViewCacheAsync to make sure we do not add a coin from the backing + * view when it is spent in the cache but not yet flushed to the parent. + */ + std::optional GetPossiblySpentCoinFromCache(const COutPoint& outpoint) const noexcept; + /** * Return a reference to Coin in the cache, or coinEmpty if not found. This is * more efficient than GetCoin. @@ -441,7 +449,7 @@ class CCoinsViewCache : public CCoinsViewBacked * to be forgotten. * If false is returned, the state of this cache (and its backing view) will be undefined. */ - bool Flush(); + virtual bool Flush(); /** * Push the modifications applied to this cache to its base while retaining @@ -482,7 +490,7 @@ class CCoinsViewCache : public CCoinsViewBacked * @note this is marked const, but may actually append to `cacheCoins`, increasing * memory usage. */ - CCoinsMap::iterator FetchCoin(const COutPoint &outpoint) const; + virtual CCoinsMap::iterator FetchCoin(const COutPoint &outpoint) const; }; //! Utility function to add all of a transaction's outputs to a cache. diff --git a/src/coinsviewcacheasync.h b/src/coinsviewcacheasync.h new file mode 100644 index 000000000000..7114258060e2 --- /dev/null +++ b/src/coinsviewcacheasync.h @@ -0,0 +1,247 @@ +// Copyright (c) The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_COINSVIEWCACHEASYNC_H +#define BITCOIN_COINSVIEWCACHEASYNC_H + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static constexpr int32_t WORKER_THREADS{4}; + +/** + * CCoinsViewCache subclass that asynchronously fetches block inputs in parallel. + * Only used in ConnectBlock to pass as an ephemeral view that can be reset if the block is invalid. + * It provides the same interface as CCoinsViewCache, overriding the FetchCoin private method and Flush. + * It adds an additional StartFetching method to provide the block, and Reset to reset the state if Flush is not called. + * + * The cache spawns a fixed set of worker threads that fetch Coins for each input in a block. + * When FetchCoin() is called, the main thread waits for the corresponding coin to be fetched and returns it. + * While waiting, the main thread will also fetch coins to maximize parallelism. + * + * Worker threads are synchronized with the main thread using a barrier, which is used at the beginning of fetching to + * start the workers and at the end to ensure all workers have finished before the next block is started. + */ +class CoinsViewCacheAsync : public CCoinsViewCache +{ +private: + //! The latest input not yet being fetched. Workers atomically increment this when fetching. + mutable std::atomic_uint32_t m_input_head{0}; + //! The latest input not yet accessed by a consumer. Only the main thread increments this. + mutable uint32_t m_input_tail{0}; + + //! The inputs of the block which is being fetched. + struct InputToFetch { + //! Workers set this after setting the coin. The main thread tests this before reading the coin. + std::atomic_flag ready{}; + //! The outpoint of the input to fetch; + const COutPoint& outpoint; + //! The coin that workers will fetch and main thread will insert into cache. + std::optional coin{std::nullopt}; + + /** + * We only move when m_inputs reallocates during setup. + * We never move after work begins, so we don't have to copy other members. + */ + InputToFetch(InputToFetch&& other) noexcept : outpoint{other.outpoint} {} + explicit InputToFetch(const COutPoint& o LIFETIMEBOUND) noexcept : outpoint{o} {} + }; + mutable std::vector m_inputs{}; + + /** + * The first 8 bytes of txids of all txs in the block being fetched. This is used to filter out inputs that + * are created earlier in the same block, since they will not be in the db or the cache. + * Using only the first 8 bytes is a performance improvement, versus storing the entire 32 bytes. In case of a + * collision of an input being spent having the same first 8 bytes as a txid of a tx elsewhere in the block, + * the input will not be fetched in the background. The input will still be fetched later on the main thread. + * Using a sorted vector and binary search lookups is a performance improvement. It is faster than + * using std::unordered_set with salted hash or std::set. + */ + std::vector m_txids{}; + + //! DB coins view to fetch from. + const CCoinsView& m_db; + + /** + * Similar to CCoinsViewCache::GetCoin, but it does not mutate internally. + * Therefore safe to call from any thread once inside the barrier. + */ + std::optional GetCoinWithoutMutating(const COutPoint& outpoint) const + { + if (auto coin{static_cast(base)->GetPossiblySpentCoinFromCache(outpoint)}) { + if (!coin->IsSpent()) [[likely]] return coin; + return std::nullopt; + } + return m_db.GetCoin(outpoint); + } + + /** + * Claim and fetch the next input in the queue. Safe to call from any thread once inside the barrier. + * + * @return true if there are more inputs in the queue to fetch + * @return false if there are no more inputs in the queue to fetch + */ + bool ProcessInputInBackground() const noexcept + { + const auto i{m_input_head.fetch_add(1, std::memory_order_relaxed)}; + if (i >= m_inputs.size()) [[unlikely]] return false; + + auto& input{m_inputs[i]}; + // Inputs spending a coin from a tx earlier in the block won't be in the cache or db + if (std::ranges::binary_search(m_txids, input.outpoint.hash.ToUint256().GetUint64(0))) { + // We can use relaxed ordering here since we don't write the coin. + input.ready.test_and_set(std::memory_order_relaxed); + input.ready.notify_one(); + return true; + } + + if (auto coin{GetCoinWithoutMutating(input.outpoint)}) [[likely]] input.coin.emplace(std::move(*coin)); + // We need release here, so writing coin in the line above happens before the main thread acquires. + input.ready.test_and_set(std::memory_order_release); + input.ready.notify_one(); + return true; + } + + //! Get the index in m_inputs for the given outpoint. Advances m_input_tail if found. + std::optional GetInputIndex(const COutPoint& outpoint) const noexcept + { + // This assumes ConnectBlock accesses all inputs in the same order as they are added to m_inputs + // in StartFetching. Some outpoints are not accessed because they are created by the block, so we scan until we + // come across the requested input. The input will be cached after access, so we can advance the tail so + // future accesses won't have to scan previously accessed inputs. + for (const auto i : std::views::iota(m_input_tail, m_inputs.size())) [[likely]] { + if (m_inputs[i].outpoint == outpoint) { + m_input_tail = i + 1; + return i; + } + } + return std::nullopt; + } + + CCoinsMap::iterator FetchCoin(const COutPoint& outpoint) const override + { + const auto& [ret, inserted]{cacheCoins.try_emplace(outpoint)}; + if (!inserted) return ret; + + if (const auto i{GetInputIndex(outpoint)}) [[likely]] { + auto& input{m_inputs[*i]}; + // Check if the coin is ready to be read. We need to acquire to match the worker thread's release. + while (!input.ready.test(std::memory_order_acquire)) { + // Work instead of waiting if the coin is not ready + if (!ProcessInputInBackground()) { + // No more work, just wait + input.ready.wait(/*old=*/false, std::memory_order_acquire); + break; + } + } + if (input.coin) [[likely]] ret->second.coin = std::move(*input.coin); + } + + if (ret->second.coin.IsSpent()) [[unlikely]] { + // We will only get in here for BIP30 checks, txid collisions, or a block with missing or spent inputs. + if (auto coin{GetCoinWithoutMutating(outpoint)}) { + ret->second.coin = std::move(*coin); + } else { + cacheCoins.erase(ret); + return cacheCoins.end(); + } + } + + cachedCoinsUsage += ret->second.coin.DynamicMemoryUsage(); + return ret; + } + + std::vector m_worker_threads{}; + std::barrier<> m_barrier; + + //! Stop all worker threads. + void StopFetching() noexcept + { + if (m_inputs.empty()) return; + // Skip fetching the rest of the inputs by moving the head to the end. + m_input_head.store(m_inputs.size(), std::memory_order_relaxed); + // Wait for all threads to stop. + m_barrier.arrive_and_wait(); + m_inputs.clear(); + } + +public: + //! Start fetching all block inputs in parallel. + void StartFetching(const CBlock& block) noexcept + { + // Loop through the inputs of the block and set them in the queue. Also construct the set of txids to filter. + for (const auto& tx : block.vtx | std::views::drop(1)) [[likely]] { + for (const auto& input : tx->vin) [[likely]] m_inputs.emplace_back(input.prevout); + m_txids.emplace_back(tx->GetHash().ToUint256().GetUint64(0)); + } + // Don't start threads if there's nothing to fetch. + if (m_inputs.empty()) [[unlikely]] return; + // Sort txids so we can do binary search lookups. + std::ranges::sort(m_txids); + // Start workers by entering the barrier. + m_barrier.arrive_and_wait(); + } + + //! Stop fetching and reset state. Must be called before block is destroyed. + void Reset() noexcept + { + StopFetching(); + m_input_head.store(0, std::memory_order_relaxed); + m_input_tail = 0; + m_txids.clear(); + cacheCoins.clear(); + cachedCoinsUsage = 0; + hashBlock = uint256::ZERO; + } + + bool Flush() override + { + // We need to stop workers from accessing base before we mutate it. + StopFetching(); + auto cursor{CoinsViewCacheCursor(m_sentinel, cacheCoins, /*will_erase=*/true)}; + const auto ret{base->BatchWrite(cursor, hashBlock)}; + Reset(); + return ret; + } + + explicit CoinsViewCacheAsync(CCoinsViewCache& cache, const CCoinsView& db, + int32_t num_workers = WORKER_THREADS) noexcept + : CCoinsViewCache{&cache}, m_db{db}, m_barrier{num_workers + 1} + { + for (const auto n : std::views::iota(0, num_workers)) { + m_worker_threads.emplace_back([this, n] { + util::ThreadRename(strprintf("inputfetch.%i", n)); + while (true) { + m_barrier.arrive_and_wait(); + while (ProcessInputInBackground()) [[likely]] {} + if (m_inputs.empty()) [[unlikely]] return; + m_barrier.arrive_and_wait(); + } + }); + } + } + + ~CoinsViewCacheAsync() override + { + m_barrier.arrive_and_drop(); + for (auto& t : m_worker_threads) t.join(); + } +}; + +#endif // BITCOIN_COINSVIEWCACHEASYNC_H diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index 83cb989aa9b1..96c95cc4571c 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -32,6 +32,7 @@ add_executable(test_bitcoin cluster_linearize_tests.cpp coins_tests.cpp coinscachepair_tests.cpp + coinsviewcacheasync_tests.cpp coinstatsindex_tests.cpp common_url_tests.cpp compress_tests.cpp diff --git a/src/test/coinsviewcacheasync_tests.cpp b/src/test/coinsviewcacheasync_tests.cpp new file mode 100644 index 000000000000..0c625e0fc510 --- /dev/null +++ b/src/test/coinsviewcacheasync_tests.cpp @@ -0,0 +1,203 @@ +// Copyright (c) The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include + +BOOST_AUTO_TEST_SUITE(coinsviewcacheasync_tests) + +struct NoAccessCoinsView : CCoinsView { + std::optional GetCoin(const COutPoint&) const override { abort(); } +}; + +static CBlock CreateBlock() noexcept +{ + static constexpr auto NUM_TXS{100}; + CBlock block; + CMutableTransaction coinbase; + coinbase.vin.emplace_back(); + block.vtx.push_back(MakeTransactionRef(coinbase)); + + Txid prevhash{Txid::FromUint256(uint256{1})}; + + for (const auto i : std::views::iota(1, NUM_TXS)) { + CMutableTransaction tx; + Txid txid; + if (i % 3 == 0) { + // External input + txid = Txid::FromUint256(uint256(i)); + } else if (i % 3 == 1) { + // Internal spend (prev tx) + txid = prevhash; + } else { + // Test shortid collisions (looks internal, but is external) + uint256 u{}; + std::memcpy(u.begin(), prevhash.ToUint256().begin(), 8); + txid = Txid::FromUint256(u); + } + tx.vin.emplace_back(txid, 0); + prevhash = tx.GetHash(); + block.vtx.push_back(MakeTransactionRef(tx)); + } + + return block; +} + +void PopulateView(const CBlock& block, CCoinsView& view, bool spent = false) +{ + CCoinsViewCache cache{&view}; + cache.SetBestBlock(uint256::ONE); + + std::unordered_set txids{}; + txids.reserve(block.vtx.size() - 1); + for (const auto& tx : block.vtx | std::views::drop(1)) { + for (const auto& in : tx->vin) { + if (!txids.contains(in.prevout.hash)) { + Coin coin{}; + if (!spent) coin.out.nValue = 1; + cache.EmplaceCoinInternalDANGER(COutPoint{in.prevout}, std::move(coin)); + } + } + txids.emplace(tx->GetHash()); + } + + cache.Flush(); +} + +void CheckCache(const CBlock& block, const CCoinsViewCache& cache) +{ + uint32_t counter{0}; + std::unordered_set txids{}; + txids.reserve(block.vtx.size() - 1); + + for (const auto& tx : block.vtx) { + if (tx->IsCoinBase()) { + BOOST_CHECK(!cache.GetPossiblySpentCoinFromCache(tx->vin[0].prevout)); + } else { + for (const auto& in : tx->vin) { + const auto& outpoint{in.prevout}; + const auto& first{cache.AccessCoin(outpoint)}; + const auto& second{cache.AccessCoin(outpoint)}; + BOOST_CHECK_EQUAL(&first, &second); + const auto should_have{!txids.contains(outpoint.hash)}; + if (should_have) ++counter; + const auto have{cache.GetPossiblySpentCoinFromCache(outpoint)}; + BOOST_CHECK_EQUAL(should_have, !!have); + } + txids.emplace(tx->GetHash()); + } + } + BOOST_CHECK(cache.GetCacheSize() == counter); +} + + +BOOST_AUTO_TEST_CASE(fetch_inputs_from_db) +{ + const auto block{CreateBlock()}; + CCoinsViewDB db{{.path = "", .cache_bytes = 1_MiB, .memory_only = true}, {}}; + PopulateView(block, db); + NoAccessCoinsView no_access; + CCoinsViewCache main_cache{&no_access}; + CoinsViewCacheAsync view{main_cache, db}; + for (auto i{0}; i < 3; ++i) { + view.StartFetching(block); + CheckCache(block, view); + view.Reset(); + } +} + +BOOST_AUTO_TEST_CASE(fetch_inputs_from_cache) +{ + const auto block{CreateBlock()}; + const CCoinsViewDB db{{.path = "", .cache_bytes = 1_MiB, .memory_only = true}, {}}; + NoAccessCoinsView no_access; + CCoinsViewCache main_cache{&no_access}; + PopulateView(block, main_cache); + CoinsViewCacheAsync view{main_cache, db}; + for (auto i{0}; i < 3; ++i) { + view.StartFetching(block); + CheckCache(block, view); + view.Reset(); + } +} + +// Test for the case where a block spends coins that are spent in the cache, but +// the spentness has not been flushed to the db. +BOOST_AUTO_TEST_CASE(fetch_no_double_spend) +{ + const auto block{CreateBlock()}; + CCoinsViewDB db{{.path = "", .cache_bytes = 1_MiB, .memory_only = true}, {}}; + PopulateView(block, db); + NoAccessCoinsView no_access; + CCoinsViewCache main_cache{&no_access}; + // Add all inputs as spent already in cache + PopulateView(block, main_cache, /*spent=*/true); + CoinsViewCacheAsync view{main_cache, db}; + for (auto i{0}; i < 3; ++i) { + view.StartFetching(block); + for (const auto& tx : block.vtx) { + for (const auto& in : tx->vin) { + const auto& c{view.AccessCoin(in.prevout)}; + BOOST_CHECK(c.IsSpent()); + } + } + // Coins are not added to the view, even though they exist unspent in the parent db + BOOST_CHECK_EQUAL(view.GetCacheSize(), 0); + view.Reset(); + } +} + +BOOST_AUTO_TEST_CASE(fetch_no_inputs) +{ + const auto block{CreateBlock()}; + const CCoinsViewDB db{{.path = "", .cache_bytes = 1_MiB, .memory_only = true}, {}}; + NoAccessCoinsView no_access; + CCoinsViewCache main_cache{&no_access}; + CoinsViewCacheAsync view{main_cache, db}; + for (auto i{0}; i < 3; ++i) { + view.StartFetching(block); + for (const auto& tx : block.vtx) { + for (const auto& in : tx->vin) { + const auto& c{view.AccessCoin(in.prevout)}; + BOOST_CHECK(c.IsSpent()); + } + } + BOOST_CHECK_EQUAL(view.GetCacheSize(), 0); + view.Reset(); + } +} + +// Test that the main thread can make progress with no workers +BOOST_AUTO_TEST_CASE(fetch_main_thread) +{ + const auto block{CreateBlock()}; + const CCoinsViewDB db{{.path = "", .cache_bytes = 1_MiB, .memory_only = true}, {}}; + NoAccessCoinsView no_access; + CCoinsViewCache main_cache{&no_access}; + PopulateView(block, main_cache); + CoinsViewCacheAsync view{main_cache, db, /*num_workers=*/0}; + for (auto i{0}; i < 3; ++i) { + view.StartFetching(block); + CheckCache(block, view); + view.Reset(); + } +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/fuzz/CMakeLists.txt b/src/test/fuzz/CMakeLists.txt index 607723b978ae..2bbda8e88efc 100644 --- a/src/test/fuzz/CMakeLists.txt +++ b/src/test/fuzz/CMakeLists.txt @@ -27,6 +27,7 @@ add_executable(fuzz cluster_linearize.cpp coins_view.cpp coinscache_sim.cpp + coinsviewcacheasync.cpp connman.cpp crypto.cpp crypto_aes256.cpp diff --git a/src/test/fuzz/coinsviewcacheasync.cpp b/src/test/fuzz/coinsviewcacheasync.cpp new file mode 100644 index 000000000000..18ec10a9982a --- /dev/null +++ b/src/test/fuzz/coinsviewcacheasync.cpp @@ -0,0 +1,176 @@ +// Copyright (c) The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +std::optional g_async_cache{}; +std::optional g_db{}; + +static void setup_threadpool_test() +{ + LogInstance().DisableLogging(); + auto db_params = DBParams{ + .path = "", + .cache_bytes = 1_MiB, + .memory_only = true, + }; + g_db.emplace(std::move(db_params), CoinsViewOptions{}); + CCoinsViewCache cache{nullptr}; + g_async_cache.emplace(cache, *g_db); +} + +FUZZ_TARGET(coinsviewcacheasync, .init = setup_threadpool_test) +{ + SeedRandomStateForTest(SeedRand::ZEROS); + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) + { + CBlock block; + Txid prevhash{Txid::FromUint256(ConsumeUInt256(fuzzed_data_provider))}; + + std::map db_map{}; + std::map cache_map{}; + std::vector input_outpoints{}; + + CCoinsViewCache main_cache(&*g_db); + // Used for writing to the db and erasing between iterations + CCoinsViewCache dummy_cache(&*g_db); + dummy_cache.SetBestBlock(uint256::ONE); + + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) + { + CMutableTransaction tx; + + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) + { + Txid txid; + if (fuzzed_data_provider.ConsumeBool()) { + txid = Txid::FromUint256(ConsumeUInt256(fuzzed_data_provider)); + } else if (fuzzed_data_provider.ConsumeBool()) { + txid = prevhash; + } else { + // Test shortid collisions + uint256 u{ConsumeUInt256(fuzzed_data_provider)}; + std::memcpy(u.begin(), prevhash.ToUint256().begin(), 8); + txid = Txid::FromUint256(u); + } + const auto index{fuzzed_data_provider.ConsumeIntegral()}; + const COutPoint outpoint{txid, index}; + + tx.vin.emplace_back(outpoint); + + if (fuzzed_data_provider.ConsumeBool()) { + Coin coin{}; + coin.fCoinBase = fuzzed_data_provider.ConsumeBool(); + coin.nHeight = + fuzzed_data_provider.ConsumeIntegralInRange( + 0, std::numeric_limits::max()); + coin.out.nValue = ConsumeMoney(fuzzed_data_provider); + assert(!coin.IsSpent()); + db_map.try_emplace(outpoint, coin); + dummy_cache.EmplaceCoinInternalDANGER( + COutPoint(outpoint), + std::move(coin)); + } + + // Add a different coin to the cache + if (fuzzed_data_provider.ConsumeBool()) { + Coin coin{}; + coin.fCoinBase = fuzzed_data_provider.ConsumeBool(); + coin.nHeight = + fuzzed_data_provider.ConsumeIntegralInRange( + 0, std::numeric_limits::max()); + coin.out.nValue = + fuzzed_data_provider.ConsumeIntegralInRange( + -1, MAX_MONEY); + cache_map.try_emplace(outpoint, coin); + main_cache.EmplaceCoinInternalDANGER( + COutPoint(outpoint), + std::move(coin)); + } + + input_outpoints.emplace_back(outpoint); + } + + prevhash = tx.GetHash(); + block.vtx.push_back(MakeTransactionRef(tx)); + } + + (void)dummy_cache.Sync(); + CoinsViewCacheAsync& cache(*g_async_cache); + cache.SetBackend(main_cache); + cache.StartFetching(block); + + std::unordered_set outpoints_in_cache{}; + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), static_cast(input_outpoints.size() * 10)) + { + COutPoint outpoint; + if (fuzzed_data_provider.ConsumeBool()) { + const auto index{fuzzed_data_provider.ConsumeIntegralInRange(0, input_outpoints.size() - 1)}; + outpoint = input_outpoints[index]; + } else { + const auto txid{Txid::FromUint256(ConsumeUInt256(fuzzed_data_provider))}; + const auto index{fuzzed_data_provider.ConsumeIntegral()}; + outpoint = COutPoint(txid, index); + } + const auto& accessed_coin{cache.AccessCoin(outpoint)}; + const auto coin{cache.GetPossiblySpentCoinFromCache(outpoint)}; + if (coin) { + assert(!coin->IsSpent()); + assert(coin->fCoinBase == accessed_coin.fCoinBase); + assert(coin->nHeight == accessed_coin.nHeight); + assert(coin->out == accessed_coin.out); + outpoints_in_cache.emplace(outpoint); + } + const auto& db_it{db_map.find(outpoint)}; + const auto cache_it{cache_map.find(outpoint)}; + if (!coin) { + assert(accessed_coin.IsSpent()); + // If we don't have a coin, then it's either spent in cache or missing + const auto spent_cache_coin{cache_it != cache_map.end() && cache_it->second.IsSpent()}; + const auto no_coin{cache_it == cache_map.end() && db_it == db_map.end()}; + assert(spent_cache_coin || no_coin); + } else if (cache_it != cache_map.end()) { + // Make sure we have the main cache coin if it exists instead of db + const auto& cache_coin{cache_it->second}; + assert(!cache_coin.IsSpent()); + assert(coin->fCoinBase == cache_coin.fCoinBase); + assert(coin->nHeight == cache_coin.nHeight); + assert(coin->out == cache_coin.out); + } else { + assert(db_it != db_map.end()); + // Check any coins not in the main cache are the same as the db + const auto& db_coin{db_it->second}; + assert(coin->fCoinBase == db_coin.fCoinBase); + assert(coin->nHeight == db_coin.nHeight); + assert(coin->out == db_coin.out); + } + } + assert(cache.GetCacheSize() == outpoints_in_cache.size()); + fuzzed_data_provider.ConsumeBool() ? (void)cache.Flush() : cache.Reset(); + for (const auto& pair : db_map) { + dummy_cache.SpendCoin(pair.first); + } + (void)dummy_cache.Flush(); + } +} From 44628ae75ebac4e8c7fbe637d54258e5ff6812ab Mon Sep 17 00:00:00 2001 From: Andrew Toth Date: Sat, 22 Nov 2025 20:15:47 -0500 Subject: [PATCH 45/48] bench: add CoinsViewCacheAsync benchmark Add benchmark to measure CoinsViewCacheAsync performance. This helps validate performance improvements and identify potential regressions. --- src/bench/CMakeLists.txt | 1 + src/bench/coinsviewcacheasync.cpp | 51 +++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 src/bench/coinsviewcacheasync.cpp diff --git a/src/bench/CMakeLists.txt b/src/bench/CMakeLists.txt index e0e03b1df7cc..c82f48b6af0a 100644 --- a/src/bench/CMakeLists.txt +++ b/src/bench/CMakeLists.txt @@ -19,6 +19,7 @@ add_executable(bench_bitcoin checkblockindex.cpp checkqueue.cpp cluster_linearize.cpp + coinsviewcacheasync.cpp connectblock.cpp crypto_hash.cpp descriptors.cpp diff --git a/src/bench/coinsviewcacheasync.cpp b/src/bench/coinsviewcacheasync.cpp new file mode 100644 index 000000000000..a5c69161b435 --- /dev/null +++ b/src/bench/coinsviewcacheasync.cpp @@ -0,0 +1,51 @@ +// Copyright (c) The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +static void CoinsViewCacheAsyncBenchmark(benchmark::Bench& bench) +{ + CBlock block; + DataStream{benchmark::data::block413567} >> TX_WITH_WITNESS(block); + const auto testing_setup{MakeNoLogFileContext(ChainType::MAIN, { .coins_db_in_memory = false })}; + Chainstate& chainstate{testing_setup->m_node.chainman->ActiveChainstate()}; + auto& coins_tip{WITH_LOCK(testing_setup->m_node.chainman->GetMutex(), return chainstate.CoinsTip();)}; + + for (const auto& tx : block.vtx | std::views::drop(1)) { + for (const auto& in : tx->vin) { + Coin coin{}; + coin.out.nValue = 1; + coins_tip.EmplaceCoinInternalDANGER(COutPoint{in.prevout}, std::move(coin)); + } + } + chainstate.ForceFlushStateToDisk(); + const auto& coins_db{WITH_LOCK(testing_setup->m_node.chainman->GetMutex(), return chainstate.CoinsDB();)}; + CoinsViewCacheAsync async_cache{coins_tip, coins_db}; + + bench.run([&] { + async_cache.StartFetching(block); + for (const auto& tx : block.vtx | std::views::drop(1)) { + for (const auto& in : tx->vin) { + const auto have{async_cache.HaveCoin(in.prevout)}; + assert(have); + } + } + async_cache.Reset(); + }); +} + +BENCHMARK(CoinsViewCacheAsyncBenchmark, benchmark::PriorityLevel::HIGH); From 88ca6652d6b636519bc977327a7282ebe94f1e76 Mon Sep 17 00:00:00 2001 From: Andrew Toth Date: Sat, 22 Nov 2025 20:18:53 -0500 Subject: [PATCH 46/48] validation: fetch block inputs via CCoinsViewCacheAsync during connection Pass CCoinsViewCacheAsync as the view into ConnectBlock. Inputs are now fetched in parallel during block connection, speeding up connection by over 30%. --- src/validation.cpp | 5 ++++- src/validation.h | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/validation.cpp b/src/validation.cpp index 507329655817..490950730dc6 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1872,6 +1872,7 @@ void CoinsViews::InitCache() { AssertLockHeld(::cs_main); m_cacheview = std::make_unique(&m_catcherview); + m_connect_block_view = std::make_unique(*m_cacheview, m_catcherview); } Chainstate::Chainstate( @@ -3083,7 +3084,8 @@ bool Chainstate::ConnectTip( LogDebug(BCLog::BENCH, " - Load block from disk: %.2fms\n", Ticks(time_2 - time_1)); { - CCoinsViewCache view(&CoinsTip()); + auto& view{*m_coins_views->m_connect_block_view}; + view.StartFetching(*block_to_connect); bool rv = ConnectBlock(*block_to_connect, state, pindexNew, view); if (m_chainman.m_options.signals) { m_chainman.m_options.signals->BlockChecked(block_to_connect, state); @@ -3092,6 +3094,7 @@ bool Chainstate::ConnectTip( if (state.IsInvalid()) InvalidBlockFound(pindexNew, state); LogError("%s: ConnectBlock %s failed, %s\n", __func__, pindexNew->GetBlockHash().ToString(), state.ToString()); + view.Reset(); return false; } time_3 = SteadyClock::now(); diff --git a/src/validation.h b/src/validation.h index cd448f3ca9eb..efb0a8b52d25 100644 --- a/src/validation.h +++ b/src/validation.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -485,6 +486,10 @@ class CoinsViews { //! can fit per the dbcache setting. std::unique_ptr m_cacheview GUARDED_BY(cs_main); + //! Used as an empty view that is only passed into ConnectBlock to help speed up block validation, + //! as well as not pollute the underlying cache with newly created coins in case the block is invalid. + std::unique_ptr m_connect_block_view GUARDED_BY(cs_main); + //! This constructor initializes CCoinsViewDB and CCoinsViewErrorCatcher instances, but it //! *does not* create a CCoinsViewCache instance by default. This is done separately because the //! presence of the cache has implications on whether or not we're allowed to flush the cache's From 5df79c28dacd24567c7505ca6d29c2aa7a3b4b7a Mon Sep 17 00:00:00 2001 From: Andrew Toth Date: Tue, 9 Dec 2025 09:54:45 -0500 Subject: [PATCH 47/48] some fixups --- src/bench/coinsviewcacheasync.cpp | 2 ++ src/test/coinsviewcacheasync_tests.cpp | 26 ++++++++++++++++++++++---- src/test/fuzz/coinsviewcacheasync.cpp | 5 +++++ 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/src/bench/coinsviewcacheasync.cpp b/src/bench/coinsviewcacheasync.cpp index a5c69161b435..9d8fdcf703c5 100644 --- a/src/bench/coinsviewcacheasync.cpp +++ b/src/bench/coinsviewcacheasync.cpp @@ -10,12 +10,14 @@ #include #include #include +#include #include #include #include #include #include +#include static void CoinsViewCacheAsyncBenchmark(benchmark::Bench& bench) { diff --git a/src/test/coinsviewcacheasync_tests.cpp b/src/test/coinsviewcacheasync_tests.cpp index 0c625e0fc510..54425647a085 100644 --- a/src/test/coinsviewcacheasync_tests.cpp +++ b/src/test/coinsviewcacheasync_tests.cpp @@ -7,8 +7,6 @@ #include #include #include -#include -#include #include #include #include @@ -17,7 +15,7 @@ #include #include -#include +#include #include #include @@ -104,7 +102,7 @@ void CheckCache(const CBlock& block, const CCoinsViewCache& cache) txids.emplace(tx->GetHash()); } } - BOOST_CHECK(cache.GetCacheSize() == counter); + BOOST_CHECK_EQUAL(cache.GetCacheSize(), counter); } @@ -200,4 +198,24 @@ BOOST_AUTO_TEST_CASE(fetch_main_thread) } } +// Access coin that is not a block's input +BOOST_AUTO_TEST_CASE(access_non_input_coin) +{ + const auto block{CreateBlock()}; + const CCoinsViewDB db{{.path = "", .cache_bytes = 1_MiB, .memory_only = true}, {}}; + NoAccessCoinsView no_access; + CCoinsViewCache main_cache{&no_access}; + Coin coin{}; + coin.out.nValue = 1; + const COutPoint outpoint{Txid::FromUint256(uint256::ZERO), 0}; + main_cache.EmplaceCoinInternalDANGER(COutPoint{Txid::FromUint256(uint256::ZERO), 0}, std::move(coin)); + CoinsViewCacheAsync view{main_cache, db, /*num_workers=*/0}; + for (auto i{0}; i < 3; ++i) { + view.StartFetching(block); + const auto& accessed_coin{view.AccessCoin(outpoint)}; + BOOST_CHECK(!accessed_coin.IsSpent()); + view.Reset(); + } +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/fuzz/coinsviewcacheasync.cpp b/src/test/fuzz/coinsviewcacheasync.cpp index 18ec10a9982a..187d0e49d7a5 100644 --- a/src/test/fuzz/coinsviewcacheasync.cpp +++ b/src/test/fuzz/coinsviewcacheasync.cpp @@ -2,8 +2,11 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include #include #include +#include +#include #include #include #include @@ -17,6 +20,8 @@ #include #include +#include +#include #include #include #include From 33e2f7d6000554965b9fc401d53bb07c575ba25c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C5=91rinc?= Date: Mon, 1 Sep 2025 21:29:37 -0700 Subject: [PATCH 48/48] coins: reduce lookups in dbcache layer propagation Previously, when the parent coins cache had no entry and the child did, `BatchWrite` performed a find followed by `try_emplace`, which resulted in multiple `SipHash` computations and bucket traversals on the common insert path. This change uses a single leading `try_emplace` and branches on the returned `inserted` flag. In the `FRESH && SPENT` case (only exercised by tests), we erase the just-inserted placeholder (which is constant time with no rehash anyway). Semantics are unchanged for all valid parent/child state combinations. This change is a minimal version of https://github.com/bitcoin/bitcoin/pull/32128/commits/723c49b63bb10da843fbb6efc6928dca415cc47f and draws simplification ideas https://github.com/bitcoin/bitcoin/pull/30673/commits/ae76ec7bcff0a08a61f294882a71e46d177b009f. Added TODO versions for related pre-existing issues that should be fixed in follow-ups. Co-authored-by: Martin Ankerl Co-authored-by: Andrew Toth Co-authored-by: optout <13562139+optout21@users.noreply.github.com> --- src/coins.cpp | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/coins.cpp b/src/coins.cpp index f9ac6dc59862..d7bd45fb28a6 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -191,18 +191,16 @@ void CCoinsViewCache::SetBestBlock(const uint256 &hashBlockIn) { bool CCoinsViewCache::BatchWrite(CoinsViewCacheCursor& cursor, const uint256 &hashBlockIn) { for (auto it{cursor.Begin()}; it != cursor.End(); it = cursor.NextAndMaybeErase(*it)) { - // Ignore non-dirty entries (optimization). - if (!it->second.IsDirty()) { + if (!it->second.IsDirty()) { // TODO a cursor can only contain dirty entries continue; } - CCoinsMap::iterator itUs = cacheCoins.find(it->first); - if (itUs == cacheCoins.end()) { - // The parent cache does not have an entry, while the child cache does. - // We can ignore it if it's both spent and FRESH in the child - if (!(it->second.IsFresh() && it->second.coin.IsSpent())) { - // Create the coin in the parent cache, move the data up - // and mark it as dirty. - itUs = cacheCoins.try_emplace(it->first).first; + auto [itUs, inserted]{cacheCoins.try_emplace(it->first)}; + if (inserted) { + if (it->second.IsFresh() && it->second.coin.IsSpent()) { + cacheCoins.erase(itUs); // TODO fresh coins should have been removed at spend + } else { + // The parent cache does not have an entry, while the child cache does. + // Move the data up and mark it as dirty. CCoinsCacheEntry& entry{itUs->second}; assert(entry.coin.DynamicMemoryUsage() == 0); if (cursor.WillErase(*it)) {