diff --git a/.github/actions/chen-from-main/action.yml b/.github/actions/chen-from-main/action.yml new file mode 100644 index 00000000..08d4ee33 --- /dev/null +++ b/.github/actions/chen-from-main/action.yml @@ -0,0 +1,53 @@ +name: Build chen from main +description: >- + Temporary: builds AppThreat/chen from main and publishes it to the local Ivy repository, so atom + resolves a chen version that is not released yet. Remove this action and its callers once that + chen version is on the package registry. +inputs: + ivy-home: + description: Ivy home to publish into, for builds that mount their own (Linux and macOS). Defaults to sbt's. + required: false + default: '' + setup-jdk: + description: Set up a JDK and sbt first, for jobs that do not install them. + required: false + default: 'false' +runs: + using: composite + steps: + - uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5.7.0 + if: inputs.setup-jdk == 'true' + with: + distribution: 'temurin' + java-version: '25' + - uses: sbt/setup-sbt@bfea3c5f48abd221b04a6df4798aa5eb8b6a2baf # v1.5.6 + if: inputs.setup-jdk == 'true' + - name: Publish chen main locally + if: runner.os != 'Windows' + shell: bash + env: + GITHUB_TOKEN: ${{ github.token }} + CHEN_DIR: ${{ runner.temp }}/chen-main + IVY_HOME: ${{ inputs.ivy-home }} + run: | + set -euo pipefail + git clone --quiet --depth 1 --branch main https://github.com/AppThreat/chen.git "$CHEN_DIR" + cd "$CHEN_DIR" + sbt ${IVY_HOME:+"-Dsbt.ivy.home=$IVY_HOME"} -batch \ + "set ThisBuild / Compile / packageDoc / publishArtifact := false" publishLocal + cd "$RUNNER_TEMP" + rm -rf "$CHEN_DIR" + - name: Publish chen main locally + if: runner.os == 'Windows' + shell: pwsh + env: + GITHUB_TOKEN: ${{ github.token }} + CHEN_DIR: ${{ runner.temp }}\chen-main + run: | + git clone --quiet --depth 1 --branch main https://github.com/AppThreat/chen.git "$env:CHEN_DIR" + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + Push-Location "$env:CHEN_DIR" + sbt -batch "set ThisBuild / Compile / packageDoc / publishArtifact := false" publishLocal + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + Pop-Location + Remove-Item -Recurse -Force "$env:CHEN_DIR" diff --git a/.github/workflows/containers.yml b/.github/workflows/containers.yml index 333c5ebd..d6518267 100644 --- a/.github/workflows/containers.yml +++ b/.github/workflows/containers.yml @@ -101,6 +101,7 @@ jobs: ~/.coursier key: ${{ runner.os }}-sbt-${{ hashfiles('**/build.sbt') }} lookup-only: true + - uses: ./.github/actions/chen-from-main # temporary: remove once chen 4.0.0 is published - run: | sbt stage createDistribution env: diff --git a/.github/workflows/nodejstests.yml b/.github/workflows/nodejstests.yml index c00dcd68..ab21de78 100644 --- a/.github/workflows/nodejstests.yml +++ b/.github/workflows/nodejstests.yml @@ -101,6 +101,7 @@ jobs: sudo apt update -y sudo apt install -y zlib1g php-zip if: runner.os == 'Linux' + - uses: ./.github/actions/chen-from-main # temporary: remove once chen 4.0.0 is published - name: atom node tests run: | sbt stage createDistribution @@ -190,6 +191,7 @@ jobs: - uses: coursier/setup-action@9b7939bf01fd1185ce2babe16135168361bf2c62 # v3.0.2 with: apps: scala3 scalac + - uses: ./.github/actions/chen-from-main # temporary: remove once chen 4.0.0 is published - name: build shell: pwsh run: | diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 3be734d5..17c217de 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -75,6 +75,7 @@ jobs: - uses: coursier/setup-action@9b7939bf01fd1185ce2babe16135168361bf2c62 # v3.0.2 with: apps: scala3 scalac + - uses: ./.github/actions/chen-from-main # temporary: remove once chen 4.0.0 is published - run: | # Install the AST generators (astgen, phpastgen, rbastgen) BEFORE `sbt test`: the # workflow tests probe for them and cancel without them, so a post-test install would diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 25cf5779..81d5d3bd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -36,6 +36,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} native-image-job-reports: 'true' cache: '' + - uses: ./.github/actions/chen-from-main # temporary: remove once chen 4.0.0 is published - uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 with: arch: amd64 @@ -87,6 +88,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} native-image-job-reports: 'true' cache: '' + - uses: ./.github/actions/chen-from-main # temporary: remove once chen 4.0.0 is published - uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 with: arch: arm64 @@ -134,6 +136,7 @@ jobs: with: apps: scala3 scalac - uses: oras-project/setup-oras@1d808f7d7f6995cc68b7bf507bfe5c5446e1dc9d # v2.0.1 + - uses: ./.github/actions/chen-from-main # temporary: remove once chen 4.0.0 is published - run: | wget https://github.com/upx/upx/releases/download/v5.0.1/upx-5.0.1-amd64_linux.tar.xz tar -xvf upx-5.0.1-amd64_linux.tar.xz @@ -216,6 +219,7 @@ jobs: with: apps: scala3 scalac - uses: oras-project/setup-oras@1d808f7d7f6995cc68b7bf507bfe5c5446e1dc9d # v2.0.1 + - uses: ./.github/actions/chen-from-main # temporary: remove once chen 4.0.0 is published - run: | wget https://github.com/upx/upx/releases/download/v5.0.1/upx-5.0.1-arm64_linux.tar.xz tar -xvf upx-5.0.1-arm64_linux.tar.xz @@ -280,6 +284,7 @@ jobs: with: apps: scala3 scalac - uses: oras-project/setup-oras@1d808f7d7f6995cc68b7bf507bfe5c5446e1dc9d # v2.0.1 + - uses: ./.github/actions/chen-from-main # temporary: remove once chen 4.0.0 is published - run: | brew install coreutils sbt stage createDistribution @@ -327,6 +332,10 @@ jobs: fetch-depth: 0 persist-credentials: false - uses: oras-project/setup-oras@1d808f7d7f6995cc68b7bf507bfe5c5446e1dc9d # v2.0.1 + - uses: ./.github/actions/chen-from-main # temporary: remove once chen 4.0.0 is published + with: + setup-jdk: 'true' + ivy-home: ${{ github.workspace }}/.cache/native-image-musl/ivy2 - run: | wget https://github.com/upx/upx/releases/download/v5.0.1/upx-5.0.1-amd64_linux.tar.xz tar -xvf upx-5.0.1-amd64_linux.tar.xz @@ -376,6 +385,10 @@ jobs: fetch-depth: 0 persist-credentials: false - uses: oras-project/setup-oras@1d808f7d7f6995cc68b7bf507bfe5c5446e1dc9d # v2.0.1 + - uses: ./.github/actions/chen-from-main # temporary: remove once chen 4.0.0 is published + with: + setup-jdk: 'true' + ivy-home: ${{ github.workspace }}/.cache/native-image-musl/ivy2 - run: | wget https://github.com/upx/upx/releases/download/v5.0.1/upx-5.0.1-arm64_linux.tar.xz tar -xvf upx-5.0.1-arm64_linux.tar.xz diff --git a/.github/workflows/repotests.yml b/.github/workflows/repotests.yml index a3a330ef..f13c97e4 100644 --- a/.github/workflows/repotests.yml +++ b/.github/workflows/repotests.yml @@ -141,6 +141,7 @@ jobs: - name: Install sbt run: brew install sbt if: runner.os == 'macOS' + - uses: ./.github/actions/chen-from-main # temporary: remove once chen 4.0.0 is published - run: | sbt stage createDistribution python -m pip install atom-tools diff --git a/README.md b/README.md index ee1bbdf2..6aa6fc3b 100644 --- a/README.md +++ b/README.md @@ -604,14 +604,6 @@ chennai> importAtom("/home/almalinux/work/sandbox/apollo/app.atom") Checkout [atom-tools](https://github.com/AppThreat/atom-tools) for some project ideas involving atom slices. -## devenv setup - -Install devenv by following the official [instructions](https://devenv.sh/getting-started/). - -```shell -devenv shell -``` - Language-specific profile: ```shell diff --git a/build.sbt b/build.sbt index 4c406c7f..6c88fc81 100644 --- a/build.sbt +++ b/build.sbt @@ -1,9 +1,9 @@ name := "atom" ThisBuild / organization := "io.appthreat" -ThisBuild / version := "3.2.0" -ThisBuild / scalaVersion := "3.8.4" +ThisBuild / version := "4.0.0" +ThisBuild / scalaVersion := "3.9.0" -val chenVersion = "3.4.0" +val chenVersion = "4.0.0" lazy val atom = Projects.atom resolvers += "Google Maven".at("https://maven.google.com/") @@ -27,6 +27,8 @@ libraryDependencies ++= Seq( ("io.appthreat" %% "semanticcpg" % Versions.chen % Test).classifier("tests"), ("io.appthreat" %% "x2cpg" % Versions.chen % Test).classifier("tests"), ("io.appthreat" %% "pysrc2cpg" % Versions.chen % Test).classifier("tests"), + // the memory-safety overlay is C/C++ only, so its command needs a C fixture to test against + ("io.appthreat" %% "c2cpg" % Versions.chen % Test).classifier("tests"), "org.scalatest" %% "scalatest" % "3.2.20" % Test ) diff --git a/ci/Dockerfile b/ci/Dockerfile index b15d52c5..b8ef7ad0 100644 --- a/ci/Dockerfile +++ b/ci/Dockerfile @@ -13,7 +13,7 @@ LABEL maintainer="appthreat" \ ARG JAVA_VERSION=26.0.1-tem ARG MAVEN_VERSION=3.9.16 -ARG SCALA_VERSION=3.8.4 +ARG SCALA_VERSION=3.9.0 ARG SBT_VERSION=1.12.12 ARG RUBY_VERSION=4.0.5 diff --git a/codemeta.json b/codemeta.json index 7fb30624..660ed88f 100644 --- a/codemeta.json +++ b/codemeta.json @@ -7,7 +7,7 @@ "downloadUrl": "https://github.com/AppThreat/atom", "issueTracker": "https://github.com/AppThreat/atom/issues", "name": "atom", - "version": "3.2.0", + "version": "4.0.0", "description": "Atom is a novel intermediate representation for next-generation code analysis.", "applicationCategory": "code-analysis", "keywords": [ diff --git a/devenv.lock b/devenv.lock deleted file mode 100644 index ee7851a0..00000000 --- a/devenv.lock +++ /dev/null @@ -1,175 +0,0 @@ -{ - "nodes": { - "devenv": { - "locked": { - "dir": "src/modules", - "lastModified": 1786123539, - "narHash": "sha256-vKK7JK4WJ92UQUekQ96OilOdY0Rr7A5X+UcAb7ZvDoA=", - "owner": "cachix", - "repo": "devenv", - "rev": "34a93eec7e71e64e9da1819a684d12f92331d638", - "type": "github" - }, - "original": { - "dir": "src/modules", - "owner": "cachix", - "repo": "devenv", - "type": "github" - } - }, - "flake-compat": { - "flake": false, - "locked": { - "lastModified": 1767039857, - "narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=", - "owner": "edolstra", - "repo": "flake-compat", - "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab", - "type": "github" - }, - "original": { - "owner": "edolstra", - "repo": "flake-compat", - "type": "github" - } - }, - "flake-compat_2": { - "flake": false, - "locked": { - "lastModified": 1767039857, - "narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=", - "owner": "edolstra", - "repo": "flake-compat", - "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab", - "type": "github" - }, - "original": { - "owner": "edolstra", - "repo": "flake-compat", - "type": "github" - } - }, - "flake-utils": { - "inputs": { - "systems": "systems" - }, - "locked": { - "lastModified": 1731533236, - "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, - "nixpkgs": { - "inputs": { - "nixpkgs-src": "nixpkgs-src" - }, - "locked": { - "lastModified": 1785855293, - "narHash": "sha256-oFTyNNuuFPL6KXd1D0w4K+IHmKIiXtCMdlnoc9vXkTk=", - "owner": "cachix", - "repo": "devenv-nixpkgs", - "rev": "6d5d03d1dfd5f530a098832d0f5433479b576278", - "type": "github" - }, - "original": { - "owner": "cachix", - "ref": "rolling", - "repo": "devenv-nixpkgs", - "type": "github" - } - }, - "nixpkgs-python": { - "inputs": { - "flake-compat": "flake-compat", - "nixpkgs": [ - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1784686015, - "narHash": "sha256-HpbHqOrLbDS+ZTK2u9iojvWfMlWnuPzGb9XGcEipIxE=", - "owner": "cachix", - "repo": "nixpkgs-python", - "rev": "8d51e95247ca71680b72cdd308fdf1dbeb031313", - "type": "github" - }, - "original": { - "owner": "cachix", - "repo": "nixpkgs-python", - "type": "github" - } - }, - "nixpkgs-ruby": { - "inputs": { - "flake-compat": "flake-compat_2", - "flake-utils": "flake-utils", - "nixpkgs": [ - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1785917952, - "narHash": "sha256-BHK+PqPt+x12sFyicFklryZxr8Q/UPilidJuTqtVAGc=", - "owner": "bobvanderlinden", - "repo": "nixpkgs-ruby", - "rev": "c3941c512d0d992c2c12527e2fb4da4053907fb8", - "type": "github" - }, - "original": { - "owner": "bobvanderlinden", - "repo": "nixpkgs-ruby", - "type": "github" - } - }, - "nixpkgs-src": { - "flake": false, - "locked": { - "lastModified": 1785715447, - "narHash": "sha256-4K0IGmbAOJnd/Hx2+CNEvx3ECrtYnsoaf4grI6gvZ/Q=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "243895692ae2a2fbd08a05141462c0cc0d3ca10f", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixpkgs-unstable", - "repo": "nixpkgs", - "type": "github" - } - }, - "root": { - "inputs": { - "devenv": "devenv", - "nixpkgs": "nixpkgs", - "nixpkgs-python": "nixpkgs-python", - "nixpkgs-ruby": "nixpkgs-ruby" - } - }, - "systems": { - "locked": { - "lastModified": 1681028828, - "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", - "owner": "nix-systems", - "repo": "default", - "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", - "type": "github" - }, - "original": { - "owner": "nix-systems", - "repo": "default", - "type": "github" - } - } - }, - "root": "root", - "version": 7 -} \ No newline at end of file diff --git a/devenv.nix b/devenv.nix deleted file mode 100644 index 5705971c..00000000 --- a/devenv.nix +++ /dev/null @@ -1,90 +0,0 @@ -{ pkgs, lib, inputs, config, ... }: { - # Language-specific topions - options = { - profile = lib.mkOption { - type = lib.types.enum [ "android" "ruby" "php" "c" "cplusplus" "basic" ]; - default = "basic"; - description = "Development profile to use"; - }; - }; - config = { - android = { - enable = lib.mkIf (config.profile == "android") true; - }; - languages = { - python = { - enable = true; - venv.enable = true; - version = "3.13"; - }; - javascript = { - enable = true; - package = pkgs.nodejs_24; - npm.enable = true; - }; - java = { - enable = true; - jdk.package = pkgs.jdk23; - }; - ruby = { - enable = lib.mkIf (config.profile == "ruby") true; - version = "4.0.0"; - }; - c = { - enable = lib.mkIf (config.profile == "c") true; - }; - cplusplus = { - enable = lib.mkIf (config.profile == "cplusplus") true; - }; - scala = { - enable = true; - sbt.enable = true; - }; - php = { - enable = lib.mkIf (config.profile == "php") true; - extensions = [ - "openssl" - "zip" - ]; - packages = { - composer = pkgs.phpPackages.composer; - }; - }; - }; - packages = [ - pkgs.nodejs_24 - pkgs.python313Full - (pkgs.python313.withPackages (ps: [ ps.pip ps.setuptools ])) - pkgs.sbt - pkgs.php - ]; - # Useful features - devcontainer.enable = true; - difftastic.enable = true; - # Compile atom and plugins - # Setup the latest cdxgen - enterShell = '' - set -e - mkdir -p /tmp/empty-cache - export JAVA_TOOL_OPTIONS="-Dfile.encoding=UTF-8 --enable-native-access=ALL-UNNAMED" - sudo npm install --cache /tmp/empty-cache --prefer-online -g @cdxgen/cdxgen - cdxgen --version - python3 --version - python3 -m pip install atom-tools - sbt clean stage createDistribution publishLocal - cd wrapper/nodejs - bash build.sh - sudo npm install --cache /tmp/empty-cache --prefer-online -g . - sudo npm install --cache /tmp/empty-cache --prefer-online -g @appthreat/atom-parsetools - cd ../.. - which atom - which astgen - which phpastgen - which rbastgen - which scalasem - which atom-tools - ''; - - # Tasks - }; -} diff --git a/devenv.yaml b/devenv.yaml deleted file mode 100644 index 5539f80b..00000000 --- a/devenv.yaml +++ /dev/null @@ -1,14 +0,0 @@ -inputs: - nixpkgs: - url: github:cachix/devenv-nixpkgs/rolling - nixpkgs-python: - url: github:cachix/nixpkgs-python - inputs: - nixpkgs: - follows: nixpkgs - nixpkgs-ruby: - url: github:bobvanderlinden/nixpkgs-ruby - inputs: - nixpkgs: - follows: nixpkgs -allowUnfree: true diff --git a/project/plugins.sbt b/project/plugins.sbt index 49cae18f..ccabd346 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,4 +1,4 @@ addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.6.1") addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.11.3") addSbtPlugin("com.codecommit" % "sbt-github-packages" % "0.5.3") -addSbtPlugin("org.wartremover" % "sbt-wartremover" % "3.6.0") +addSbtPlugin("org.wartremover" % "sbt-wartremover" % "3.6.2") diff --git a/src/main/scala/io/appthreat/atom/Atom.scala b/src/main/scala/io/appthreat/atom/Atom.scala index dd8e7b89..90cde2bc 100644 --- a/src/main/scala/io/appthreat/atom/Atom.scala +++ b/src/main/scala/io/appthreat/atom/Atom.scala @@ -41,12 +41,20 @@ import io.appthreat.x2cpg.passes.base.AstLinkerPass import io.appthreat.x2cpg.perf.PerfReporter import io.appthreat.x2cpg.passes.frontend.{XTypeRecovery, XTypeRecoveryConfig} import io.appthreat.x2cpg.passes.taggers.{ + AllocationStatePass, AndroidServicesTagsPass, CdxPass, ChennaiTagsPass, EasyTagsPass, + ExtentPass, + GuardPass, + IntegerWidthPass, + MemoryApiPass, + MemorySafetyFindingPass, + MemorySemanticsPass, PiiTagsPass, - TrackersTagsPass + TrackersTagsPass, + ValueOriginPass } import io.appthreat.x2cpg.passes.taggers.python.PythonFrameworkRecognizersPass import io.appthreat.x2cpg.utils.ExternalCommand @@ -408,6 +416,25 @@ object Atom: case config: AtomConfig => config.withAppendedFrontendArg("defines", x) case _ => c ) + opt[Unit]("auto-defines") + .text( + "Run a macro census first and define the build-option macros (CONFIG_*, ENABLE_*, template-declared) that hide #if code. Opt-in: it changes what is analysed. (C/C++ only)" + ) + .action((_, c) => + c match + case config: AtomConfig => config.withFrontendArg("auto-defines", "true") + case _ => c + ) + opt[String]("suggest-defines") + .valueName("") + .text( + "Write the macro census to .json and a reviewable --macro-files header to .h, then exit; with --auto-defines, write it and continue. (C/C++ only)" + ) + .action((x, c) => + c match + case config: AtomConfig => config.withFrontendArg("macro-census", x) + case _ => c + ) opt[String]("include-path") .unbounded() .text("Header include path. Repeatable. (C/C++ only)") @@ -551,6 +578,17 @@ object Atom: case config: AtomConfig => config.withConfigFile(Option(File(x))) case _ => c ) + opt[String]("memory-api-config") + .text( + "path to a JSON file (memory-apis.json schema) merged over the built-in memory-API " + + "inventory by API name, declaring in-house wrappers or platform argument roles. " + + "C/C++ only." + ) + .action((x, c) => + c match + case config: AtomConfig => config.withMemoryApiConfigFile(Option(File(x))) + case _ => c + ) opt[String]("validation-config") .text( "path to a JSON file declaring validators/sanitisers (chennai.json schema). Reachable " + @@ -711,6 +749,29 @@ object Atom: case _ => c ) ) + cmd("memory-safety") + .text( + "Run the memory-safety overlay and write findings (rule, cwe, kind, confidence, flow) as JSON" + ) + .action((_, _) => AtomMemorySafetyConfig().withDataDependencies(true)) + .children( + opt[String]("min-confidence") + .text( + s"drop findings below this confidence: high, medium or low. Defaults to keeping all." + ) + .action((x, c) => + c match + case c: AtomMemorySafetyConfig => c.withMinConfidence(x) + case _ => c + ), + opt[String]("format") + .text("output format: json (sarif is planned). Default: json.") + .action((x, c) => + c match + case c: AtomMemorySafetyConfig => c.withFormat(x) + case _ => c + ) + ) cmd("export") .text("Export the atom to a graph format (dot, graphml, gexf, graphson, neo4jcsv, gnn)") .action((_, _) => AtomExportConfig().withDataDependencies(true)) @@ -800,10 +861,27 @@ object Atom: if loaded.frontendArgsKeys then println(FrontendArgsApplier.renderKeys(loaded.language)) Right("Displayed frontend-args keys") - else run(loaded, loaded.language) + else if isCensusOnly(loaded) then + Try( + new C2Cpg().writeMacroCensus( + c2CpgConfig(loaded, loaded.outputAtomFile.pathAsString) + ) + ).toEither.left.map(e => s"Macro census failed: ${e.getMessage}") + .map(_ => "Wrote the macro census") + else + if hasCensusFlags(loaded) && !CensusLanguages.contains( + loaded.language.toUpperCase + ) + then + println( + s"--auto-defines and --suggest-defines apply to C/C++ only; ignored for -l ${loaded.language}" + ) + run(loaded, loaded.language) case Left(err) => Left(err) case Right(_) => Left("Invalid configuration generated") case Left(err) => Left(err) + end match + end run private def run(config: AtomConfig, language: String): Either[String, String] = for @@ -845,6 +923,10 @@ object Atom: PerfReporter.stage("slicing.parseDeps", "slicing")( generateParseDepsSlice(config, ag, x) ) + case msConfig: AtomMemorySafetyConfig => + PerfReporter.stage("slicing.memorySafety", "slicing")( + MemorySafetyCommands.runMemorySafety(ag, msConfig, config.outputAtomFile) + ) case _ => Right("No slice generation required") end match @@ -1182,9 +1264,12 @@ object Atom: .withIncludeTrivialExpressions(false) .withIncludePaths(C2ATOM_INCLUDE_PATH) val finalConfig = FrontendArgsApplier.applyC(baseConfig, config.frontendArgs) - new C2Atom().createCpg(finalConfig) + new C2Atom().createCpg(C2Cpg.withCensusDefines(finalConfig)) private def createC2Cpg(config: AtomConfig, outputAtomFile: String): Try[Cpg] = + new C2Cpg().createCpgWithOverlays(c2CpgConfig(config, outputAtomFile)) + + private def c2CpgConfig(config: AtomConfig, outputAtomFile: String): CConfig = val cIgnoreDirEnvVars = if config.language.equalsIgnoreCase("CPP") || config.language.equalsIgnoreCase("C++") then Seq("CHEN_C_IGNORE_DIRS", "CHEN_CPP_IGNORE_DIRS") @@ -1203,9 +1288,24 @@ object Atom: .withImageLocations(false) .withIncludeTrivialExpressions(false) .withIncludePaths(C2ATOM_INCLUDE_PATH) - val finalConfig = FrontendArgsApplier.applyC(baseConfig, config.frontendArgs) - new C2Cpg().createCpgWithOverlays(finalConfig) - end createC2Cpg + FrontendArgsApplier.applyC(baseConfig, config.frontendArgs) + end c2CpgConfig + + /** `--suggest-defines` without `--auto-defines`: the macro census alone, no atom. */ + /** Every language the C frontends serve, so every one runs the census the same way. */ + private val CensusLanguages = Set("C", "NEWC", "CPP", "C++", "H", "HPP", "I") + + private def hasCensusFlags(config: AtomConfig): Boolean = + config.frontendArgs.contains("auto-defines") || config.frontendArgs.contains("macro-census") + + /** `--suggest-defines` without `--auto-defines`: the census alone, no atom. Read from the + * resolved frontend config, so the flags mean exactly what the frontend will see. + */ + private def isCensusOnly(config: AtomConfig): Boolean = + hasCensusFlags(config) && CensusLanguages.contains(config.language.toUpperCase) && { + val c = c2CpgConfig(config, config.outputAtomFile.pathAsString) + c.macroCensusReport.nonEmpty && !c.autoDefines + } private def createJimple2Cpg(config: AtomConfig, outputAtomFile: String): Try[Cpg] = val baseConfig = JimpleConfig(android = androidJarPath, fullResolver = true) @@ -1443,6 +1543,54 @@ object Atom: new PythonFrameworkRecognizersPass(atom).createAndApply() } PerfReporter.stage("taggers.ChennaiTagsPass", "analysis")(runChennaiTags(x, atom)) + // What chen concludes about memory on its own - declared GCC attributes, wrapper + // bodies, nullable returns, variable storage - so MemoryApiPass needs no + // hand-written project inventory to see a project's allocator layer + PerfReporter.stage("taggers.MemorySemanticsPass", "analysis") { + new MemorySemanticsPass( + atom, + x.memoryApiConfigFile.filter(_.exists).map(_.contentAsString) + ).createAndApply() + } + PerfReporter.stage("taggers.MemoryApiPass", "analysis") { + new MemoryApiPass( + atom, + x.memoryApiConfigFile.filter(_.exists).map(_.contentAsString) + ) + .createAndApply() + } + // The rest of the memory-safety overlay: Extent reads MemoryApi's + // argument tags, Guard reads Extent's. Each pass no-ops on non-C/C++ graphs, so one + // unconditional pipeline serves every language. + PerfReporter.stage("taggers.ExtentPass", "analysis") { + new ExtentPass(atom).createAndApply() + } + PerfReporter.stage("taggers.GuardPass", "analysis") { + new GuardPass( + atom, + x.memoryApiConfigFile.filter(_.exists).map(_.contentAsString) + ) + .createAndApply() + } + PerfReporter.stage("taggers.ValueOriginPass", "analysis") { + new ValueOriginPass(atom).createAndApply() + } + // Integer width facts - narrowing, sign-changing casts and attacker-influenced + // arithmetic feeding lengths. Facts only; the integer rules read them. + PerfReporter.stage("taggers.IntegerWidthPass", "analysis") { + new IntegerWidthPass(atom).createAndApply() + } + // Per-allocation state facts - allocated/freed/maybe-freed/null/ + // escaped at each program point. Facts only; the ALLOC rules read them. + PerfReporter.stage("taggers.AllocationStatePass", "analysis") { + new AllocationStatePass(atom).createAndApply() + } + PerfReporter.stage("taggers.MemorySafetyFindingPass", "analysis") { + new MemorySafetyFindingPass( + atom, + x.memoryApiConfigFile.filter(_.exists).map(_.contentAsString) + ).createAndApply() + } PerfReporter.stage("taggers.JvmTaggers", "analysis")(applyJvmTaggers(atom)) Right(()) catch diff --git a/src/main/scala/io/appthreat/atom/MemorySafetyCommands.scala b/src/main/scala/io/appthreat/atom/MemorySafetyCommands.scala new file mode 100644 index 00000000..bf192931 --- /dev/null +++ b/src/main/scala/io/appthreat/atom/MemorySafetyCommands.scala @@ -0,0 +1,230 @@ +package io.appthreat.atom + +import better.files.File +import io.circe.syntax.EncoderOps +import io.circe.{Json, JsonObject} +import io.shiftleft.codepropertygraph.generated.Cpg +import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.semanticcpg.language.* +import io.appthreat.x2cpg.passes.taggers.{ + ExtentPass, + GuardPass, + MemoryApiPass, + MemorySafetyFindingPass, + ValueOriginPass +} + +import scala.collection.mutable + +/** Implementation for the `memory-safety` command: render the overlay's findings as JSON for + * scoring harnesses, CI gates and other tools. + * + * The findings themselves are tags on the graph - `ms-finding` valued with the rule id, written by + * chen's MemorySafetyFindingPass during enhancement - so this is purely a renderer: rule metadata + * (CWE, kind, severity, confidence, message) comes from the pass's rule registry, and the flow is + * the evidence chain read back from the tags on the offending node plus its REACHING_DEF + * definitions. No detector logic lives here, which is what keeps the command and the tags from + * drifting apart. + * + * Output contract (tools/score.py reads file/line/cwe/kind/confidence; flow is for humans and + * SARIF later): + * + * {{{ + * {"rule": "MS-BOUND-002", "cwe": "CWE-787", "kind": "unbounded-copy", + * "file": "c/cwe415_double_free.c", "line": 11, "column": 5, + * "severity": "high", "confidence": "medium", + * "message": "...", "flow": [{"file": "...", "line": 12, "code": "...", "role": "..."}]} + * }}} + */ +object MemorySafetyCommands: + + private val confidenceOrder = Map("high" -> 3, "medium" -> 2, "low" -> 1) + + /** Render every finding and write it to `config.outputSliceFile`. */ + def runMemorySafety( + cpg: Cpg, + config: AtomMemorySafetyConfig, + atomFile: File + ): Either[String, String] = + // Reject rather than silently ignore: a caller that asked for `--format sarif` and got JSON + // named `.sarif`, or that misspelled a confidence and got every finding, has no way to tell. + val format = config.format.toLowerCase + if format != "json" then return Left(s"unsupported --format `$format`; only `json` exists") + val requested = config.minConfidence.toLowerCase + if requested.nonEmpty && !confidenceOrder.contains(requested) then + return Left( + s"unknown --min-confidence `$requested`; expected ${confidenceOrder.keys.toList.sorted.mkString(", ")}" + ) + val minConfidence = confidenceOrder.getOrElse(requested, 0) + val inputRoot = config.inputPath.name + + val seen = mutable.LinkedHashSet.empty[(Long, String)] + val findings = cpg.tag + .name(MemorySafetyFindingPass.TagFinding) + .l + .flatMap { tag => + val ruleId = tag.value + // a finding may sit on an expression (a length argument, a freed pointer) or on the + // exit node itself - a bare `return;` has no expression child (leaks). + // METHOD_RETURN is the one that has to be named: `Return` and `ControlStructure` are + // both EXPRESSIONs already, but METHOD_RETURN is only a CFG_NODE, so an + // `Expression | ControlStructure` match silently dropped every leak reported at the + // implicit end of a function - which in C is most of them. + tag._taggedByIn + .collectFirst { case e: (Expression | MethodReturn) => e } + .filter(e => seen.add((e.id, ruleId))) + .map(e => (e, ruleId)) + } + .flatMap { (node, ruleId) => + MemorySafetyFindingPass.rules.get(ruleId).map { rule => + // a finding-level confidence override, scoped to its own rule: one + // node can carry several findings, and one rule's hypothesis tier must not demote + // another rule's finding on the same node + (node, rule, MemorySafetyFindingPass.confidenceOf(node, rule)) + } + } + .filter { (_, rule, confidence) => + confidenceOrder.getOrElse(confidence.toLowerCase, 0) >= minConfidence + } + .toList + .sortBy { (node, _, _) => + (filenameOf(node), node.lineNumber.map(_.toInt).getOrElse(0): Int) + } + .map { (node, rule, confidence) => + render(cpg, atomFile, inputRoot, confidence)(node, rule) + } + + val outFile = config.outputSliceFile.createFileIfNotExists(createParents = true) + outFile.write(findings.asJson.noSpaces) + println( + s"Memory-safety analysis complete. ${findings.size} findings written to ${outFile.pathAsString}" + ) + Right("Memory-safety findings generated successfully") + end runMemorySafety + + /** The FILE the finding sits in, relative to the analysed input. The file edge is what the path + * is rooted at; the enclosing method's filename is the fallback, and it has to be reached per + * node type - there is no `.method` step on AstNode, and an exit-node finding is a METHOD_RETURN + * rather than an Expression. + */ + private def filenameOf(node: AstNode): String = + node.file.name.headOption.getOrElse( + node match + case e: Expression => e.method.filename + case m: MethodReturn => m.method.filename + case _ => "" + ) + + /** One finding: rule metadata from the registry, location from the offending node, flow from the + * node's own tags, its memory operation, and the definitions that produced the value. + * + * The file path is rooted at the input directory's name: c2cpg records FILE names relative to + * the analysed input, and consumers of this JSON (a scorer matching on `c/`/`cpp/` prefixes, a + * script re-rooting paths into a checkout) match on paths that carry that root. + */ + private def render(cpg: Cpg, atomFile: File, inputRoot: String, confidence: String)( + node: AstNode, + rule: MemorySafetyFindingPass.MemorySafetyRule + ): Json = + val relative = filenameOf(node) + val file = if inputRoot.isEmpty || inputRoot == "." then relative else s"$inputRoot/$relative" + val line = node.lineNumber.map(_.toInt).getOrElse(0) + val column = node.columnNumber.map(_.toInt).getOrElse(0) + + JsonObject( + "rule" -> rule.id.asJson, + "cwe" -> rule.cwe.asJson, + "kind" -> rule.kind.asJson, + "file" -> file.asJson, + "line" -> line.asJson, + "column" -> column.asJson, + "severity" -> rule.severity.asJson, + "confidence" -> confidence.asJson, + "message" -> s"${node.code}: ${rule.message}".asJson, + "flow" -> flowOf(node).asJson, + "atom" -> atomFile.pathAsString.asJson + ).asJson + end render + + /** The evidence chain: the offending argument, the memory operation it bounds, the origin and + * guard facts the rule read, and the definitions that produced the value (the REACHING_DEF walk + * backwards - where `df` would be paying for a solve this does not need). + */ + private def flowOf(node: AstNode): List[Json] = + val entries = mutable.ListBuffer.empty[JsonObject] + def entry(n: AstNode, role: String): Unit = + entries += JsonObject( + "file" -> n.file.name.headOption.getOrElse("").asJson, + "line" -> n.lineNumber.map(_.toInt).getOrElse(0).asJson, + "code" -> n.code.asJson, + "role" -> role.asJson + ) + // The tag-derived evidence has no node of its own: it is the fact a rule read, rendered as + // text. It must APPEND like `entry` does - a version that merely returns the object silently + // drops the origin, guard and extent rows the flow exists to show. + def textEntry(text: String, role: String): Unit = + entries += JsonObject("code" -> text.asJson, "role" -> role.asJson) + + node match + case _: Return | _: MethodReturn => entry(node, "exit") + case _ => entry(node, "length-argument") + node._astIn.collectFirst { case c: Call => c }.foreach(c => entry(c, "memory-operation")) + + // Tag.value is a String, not an Option - iterating it yields CHARACTERS, which is how + // `struct-field` rendered as twelve one-letter origin rows. + val tags = node.tag.l + tags.filter(_.name == ValueOriginPass.TagOrigin) + .foreach(t => textEntry(s"origin: ${t.value}", "origin")) + tags.filter(_.name == GuardPass.TagBelow) + .foreach(t => textEntry(s"bounded-below by ${t.value}", "guard")) + tags.filter(_.name == GuardPass.TagAbove) + .foreach(t => textEntry(s"bounded-above by ${t.value}", "guard")) + + // the destination's extent, for the size-parameter rule + node._astIn.collectFirst { case c: Call => c }.foreach { call => + call.argument.l + .filter(a => a.tag.name(MemoryApiPass.TagDst).l.nonEmpty) + .flatMap(_.tag.name(ExtentPass.TagExtent).value.l) + .foreach(v => textEntry(s"destination extent $v", "destination")) + } + + // definitions of the value, nearest first, definition sites only (identifiers in between + // are the chain, not the evidence) + node match + case i: Identifier => + var frontier: List[StoredNode] = List(i) + val visited = mutable.Set[Long](i.id) + var shown = 0 + while frontier.nonEmpty && shown < MaxFlowEntries do + val next = mutable.ListBuffer.empty[StoredNode] + frontier.foreach { n => + n._reachingDefIn.collectAll[StoredNode].l.filterNot(d => visited.contains(d.id)) + .foreach { d => + visited += d.id + d match + case c: Call => + if shown < MaxFlowEntries then + entry(c, "definition") + shown += 1 + next += d + case p: MethodParameterIn => + if shown < MaxFlowEntries then + entry(p, "definition") + shown += 1 + case lit: Literal => + if shown < MaxFlowEntries then + entry(lit, "definition") + shown += 1 + case other => next += other + } + } + frontier = next.toList + end while + case _ => () + end match + + entries.toList.map(_.asJson) + end flowOf + + private val MaxFlowEntries = 8 +end MemorySafetyCommands diff --git a/src/main/scala/io/appthreat/atom/frontends/FrontendArgsApplier.scala b/src/main/scala/io/appthreat/atom/frontends/FrontendArgsApplier.scala index 7aab0b74..9444c79b 100644 --- a/src/main/scala/io/appthreat/atom/frontends/FrontendArgsApplier.scala +++ b/src/main/scala/io/appthreat/atom/frontends/FrontendArgsApplier.scala @@ -104,6 +104,8 @@ object FrontendArgsApplier: r = r.withLogProblems(bool(args, "log-problems", r.logProblems)) r = r.withLogPreprocessor(bool(args, "log-preprocessor", r.logPreprocessor)) r = r.withPrintIfDefsOnly(bool(args, "print-ifdef-only", r.printIfDefsOnly)) + r = r.withAutoDefines(bool(args, "auto-defines", r.autoDefines)) + r = r.withMacroCensusReport(str(args, "macro-census", r.macroCensusReport)) r = r.withIncludePathsAutoDiscovery(bool( args, "include-auto-discovery", @@ -279,6 +281,20 @@ object FrontendArgsApplier: KeyDoc("log-problems", "bool", "false", "Log every parse problem.", CLike), KeyDoc("log-preprocessor", "bool", "false", "Log every preprocessor statement.", CLike), KeyDoc("print-ifdef-only", "bool", "false", "Print preprocessor conditionals and exit.", CLike), + KeyDoc( + "auto-defines", + "bool", + "false", + "Run the macro census and define its build-option macros before parsing.", + CLike + ), + KeyDoc( + "macro-census", + "string", + "", + "Write the macro census to .json and .h.", + CLike + ), KeyDoc("include-auto-discovery", "bool", "false", "Auto-discover system header paths.", CLike), KeyDoc("function-bodies", "bool", "true", "Parse function and method bodies.", CLike), KeyDoc( diff --git a/src/main/scala/io/appthreat/atom/package.scala b/src/main/scala/io/appthreat/atom/package.scala index 5daefc58..02f54336 100644 --- a/src/main/scala/io/appthreat/atom/package.scala +++ b/src/main/scala/io/appthreat/atom/package.scala @@ -36,6 +36,10 @@ package object atom: // Optional validators/sanitisers config (the chennai.json schema). Calls to the declared // methods are tagged as sanitisers so reachable flows passing through them can be dropped. var validationConfigFile: Option[File] = None + // Optional memory-API inventory overlay (the memory-apis.json schema), merged over the + // built-in inventory by API name - declare an in-house `av_memcpy` wrapper or a platform's + // memcpy roles without patching chen. Applies to C/C++ graphs. + var memoryApiConfigFile: Option[File] = None def withOutputAtomFile(x: File): AtomConfig = this.outputAtomFile = x @@ -81,8 +85,11 @@ package object atom: this.cacheFragments = x this + /** Merges into the arguments the first-class flags (`--define`, `--auto-defines`, ...) have + * already set: a later `--frontend-args` overrides the keys it names and keeps the rest. + */ def withFrontendArgs(args: Map[String, String]): AtomConfig = - this.frontendArgs = args + this.frontendArgs = this.frontendArgs ++ args this /** Sets a single frontend argument, replacing any previous value for the key. Used by the @@ -115,6 +122,10 @@ package object atom: this.validationConfigFile = x this + def withMemoryApiConfigFile(x: Option[File]): AtomConfig = + this.memoryApiConfigFile = x + this + end AtomConfig case class DefaultAtomConfig() extends AtomConfig @@ -143,6 +154,27 @@ package object atom: profile: String = "generic" ) extends AtomConfig + /** Run the memory-safety overlay over the input and write the findings JSON to the slice file + * (`-s`). The overlay needs data dependencies (the rules read REACHING_DEF facts), so the + * command turns them on the way `reachables` does rather than requiring `--with-data-deps`. + * + * The settings are `var`s mutated in place, like every other command's, and NOT a `copy`: the + * base [[AtomConfig]] keeps `language`, `dataDeps` and the output paths in `var`s of the trait, + * so a `copy` returns a fresh instance with all of those back at their defaults. `atom + * memory-safety -l c --min-confidence medium` used to lose the `-l`. + */ + case class AtomMemorySafetyConfig() extends AtomConfig: + var minConfidence: String = "" + var format: String = "json" + + def withMinConfidence(x: String): AtomMemorySafetyConfig = + this.minConfidence = x + this + + def withFormat(x: String): AtomMemorySafetyConfig = + this.format = x + this + /** Export the whole atom, or a per-method subgraph of it, to one of the supported graph formats. * `scope` is either "whole" or "methods". The output format is taken from `exportFormat` and the * destination directory from `exportDir`. diff --git a/src/main/scala/io/appthreat/atom/slicing/DataFlowSlicing.scala b/src/main/scala/io/appthreat/atom/slicing/DataFlowSlicing.scala index 1db4271a..28bac5e5 100644 --- a/src/main/scala/io/appthreat/atom/slicing/DataFlowSlicing.scala +++ b/src/main/scala/io/appthreat/atom/slicing/DataFlowSlicing.scala @@ -1,6 +1,8 @@ package io.appthreat.atom.slicing +import io.appthreat.dataflowengineoss.DefaultSemantics import io.appthreat.dataflowengineoss.language.* +import io.appthreat.dataflowengineoss.semanticsloader.Semantics import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.codepropertygraph.generated.PropertyNames import io.shiftleft.codepropertygraph.generated.nodes.* @@ -19,8 +21,30 @@ class DataFlowSlicing: private val nodeCache = new TrieMap[Long, SliceNode]() private var language: Option[String] = scala.compiletime.uninitialized + /** Language-scoped semantics for the graph being sliced, built once per + * [[calculateDataFlowSlice]] call. The slicing `repeat` invokes the DDG steps as bare `_.ddgIn`, + * so the implicit default semantics is consulted for every step: it must be (a) built once, not + * per step - a per-step rebuild of the table was measured at ~28 GB/s of garbage (4.5 TB in 155 + * s) while slicing libavformat - and (b) scoped to the graph's own language, so a C graph is not + * matched against the Java summaries and vice versa. + * + * Volatile because `TrackDataFlowTask` reads it from the slicing thread pool: the write happens + * before any task is submitted, but without it the workers are not guaranteed to see anything + * but the uninitialised value. + */ + @volatile private var graphSemantics: Semantics = scala.compiletime.uninitialized + + private def buildGraphSemantics(atom: Cpg): Semantics = + val extra = language.map(DefaultSemantics.flowsForLanguage).getOrElse(List.empty) + val semantics = Semantics.fromList(DefaultSemantics().elements ++ extra) + // Warm the regex-result cache once, up front: the per-step lookups below only read it, + // and an unwarmed instance silently never matches regex semantics (the clamping macros). + semantics.loadRegexSemantics(atom) + semantics + def calculateDataFlowSlice(atom: Cpg, config: DataFlowConfig): Option[DataFlowSlice] = language = atom.metaData.language.headOption + graphSemantics = buildGraphSemantics(atom) excludeOperatorCalls.set(config.excludeOperatorCalls) val dataFlowSlice = (config.fileFilter match @@ -131,6 +155,7 @@ class DataFlowSlicing: private class TrackDataFlowTask(config: DataFlowConfig, c: Call) extends Callable[Option[DataFlowSlice]]: override def call(): Option[DataFlowSlice] = + implicit val semantics: Semantics = graphSemantics val sinks = config.sinkPatternFilter.map(filter => c.argument.code(filter).l).getOrElse( c.argument.filterNot(_.isBlock).l diff --git a/src/main/scala/io/appthreat/atom/slicing/ReachableSlicing.scala b/src/main/scala/io/appthreat/atom/slicing/ReachableSlicing.scala index 4d114774..1fd7c448 100644 --- a/src/main/scala/io/appthreat/atom/slicing/ReachableSlicing.scala +++ b/src/main/scala/io/appthreat/atom/slicing/ReachableSlicing.scala @@ -21,7 +21,11 @@ object ReachableSlicing: private val logger = org.slf4j.LoggerFactory.getLogger(getClass) - implicit val semantics: Semantics = DefaultSemantics() + // Reassigned per run (see calculateReachableSliceAndPersist) to the semantics scoped to the + // graph's own language: the neutral defaults plus whatever flowsForLanguage adds for it. The + // previous language-neutral instance matched every graph against every language's summaries - + // a C graph against the Java request readers, a Java graph against C's bare `free`. + implicit var semantics: Semantics = DefaultSemantics.memoised // Reassigned per run to carry the method flow summaries computed for this atom (the backward query // engine uses them to prune provably empty cross-call work). The reaching-def engine choice (Flux // vs classic) is made earlier at enhancement time, not here. Reachable slicing runs one project at @@ -81,6 +85,12 @@ object ReachableSlicing: val baseFile = File(outputBasePath) Option(baseFile.parent).foreach(_.createDirectoryIfNotExists(createParents = true)) val language = atom.metaData.language.head + // Scope the semantics to this graph's language before any query runs, and warm the + // regex-result cache once, single-threaded. + semantics = Semantics.fromList( + DefaultSemantics().elements ++ DefaultSemantics.flowsForLanguage(language) + ) + semantics.loadRegexSemantics(atom) // Build method flow summaries up front when requested, so the backward query engine can prune // cross-call tasks that provably carry no taint. Prefer the `flow-summary` tags already embedded // in the atom (written during enhancement, so they survive a reused/cached atom); only fall back @@ -951,7 +961,7 @@ object ReachableSlicing: if tags.isEmpty && externalCallee then resolveTagsAndPurls(c, resolvedCallee) else (tags, purls) - // G9: an external library call may carry its package URL on the resolved callee method + // An external library call may carry its package URL on the resolved callee method // even when the call site itself already has other tags. Surface it so reachable flows // can be tied back to dependency CVEs. val calleePurls = diff --git a/src/test/scala/io/appthreat/atom/DecodedLengthCommandTests.scala b/src/test/scala/io/appthreat/atom/DecodedLengthCommandTests.scala new file mode 100644 index 00000000..375957a2 --- /dev/null +++ b/src/test/scala/io/appthreat/atom/DecodedLengthCommandTests.scala @@ -0,0 +1,263 @@ +package io.appthreat.atom + +import better.files.File +import io.appthreat.c2cpg.testfixtures.{DataFlowCodeToCpgSuite, DataFlowTestCpg} +import io.appthreat.x2cpg.passes.taggers.* +import io.circe.parser.parse + +/** Decoded-length bounds end to end: three bounds arms render through `memory-safety` with the + * finding contract - the counter-indexed array (MS-BOUND-004 write through an escaped element + * address, CVE-2026-64830), the source overread (MS-BOUND-008, CVE-2026-64833, with the + * config-declared AVPacket-style extent pair) and the advance by a decoded size (MS-BOUND-003, + * CVE-2026-75147, with the MemorySemanticsPass out-param summary the arm turns on). Each arm's + * fixed shape renders nothing. + */ +class DecodedLengthCommandTests extends DataFlowCodeToCpgSuite: + + private val pairConfig = + """{"apis": [], "bufferExtents": [{"type": "packet", "buffer": "data", "capacity": "size"}]}""" + + private val cpg: DataFlowTestCpg = new DataFlowTestCpg() + .moreCode( + """ + |#include + |#include + |#include + |#include + | + |#define ARRAY_ELEMS(a) (sizeof(a) / sizeof((a)[0])) + | + |struct entry { long pos; }; + |struct fmt_ctx { unsigned int nb_streams; }; + |struct sub_ctx { struct entry q[32]; }; + | + |static void add_stream(struct fmt_ctx *ctx) + |{ + | ctx->nb_streams++; + |} + | + |static void put(struct entry *e, long pos) + |{ + | e->pos = pos; + |} + | + |int bad_counter_index(struct sub_ctx *sub, struct fmt_ctx *ctx, FILE *idx) + |{ + | char line[256]; + | while (fgets(line, sizeof(line), idx)) + | { + | add_stream(ctx); + | put(&sub->q[ctx->nb_streams - 1], 1); + | } + | return 0; + |} + | + |int good_exit_bounds_count(struct sub_ctx *sub, struct fmt_ctx *ctx, FILE *idx) + |{ + | char line[256]; + | while (fgets(line, sizeof(line), idx)) + | { + | if (ctx->nb_streams >= ARRAY_ELEMS(sub->q)) + | return -1; + | add_stream(ctx); + | put(&sub->q[ctx->nb_streams - 1], 1); + | } + | return 0; + |} + |""".stripMargin, + "counter.c" + ) + .moreCode( + """ + |#include + |#include + |#include + | + |struct packet { + | const uint8_t *data; /* the config names size as this member's extent */ + | int size; + |}; + | + |static int emit_hd(struct packet *pkt, int core_size) + |{ + | int pkt_size = pkt->size; + | uint8_t *dst; + | pkt_size = core_size; + | dst = (uint8_t *)malloc(pkt_size + 4); + | memcpy(dst + 4, pkt->data, pkt_size); + | free(dst); + | return 0; + |} + | + |int bad_source_overread(struct packet *pkt) + |{ + | int core = ((((uint32_t)pkt->data[5] << 16 | (uint32_t)pkt->data[6] << 8 | (uint32_t)pkt->data[7]) >> 4) & 0x3fff) + 1; + | return emit_hd(pkt, core); + |} + | + |int good_source_overread_guarded(struct packet *pkt) + |{ + | int core = ((((uint32_t)pkt->data[5] << 16 | (uint32_t)pkt->data[6] << 8 | (uint32_t)pkt->data[7]) >> 4) & 0x3fff) + 1; + | int pkt_size = pkt->size; + | uint8_t *dst; + | if (core <= pkt->size) + | pkt_size = core; + | dst = (uint8_t *)malloc(pkt_size + 4); + | memcpy(dst + 4, pkt->data, pkt_size); + | free(dst); + | return 0; + |} + |""".stripMargin, + "source.c" + ) + .moreCode( + """ + |#include + | + |static int parse_leb(const uint8_t *buf, int buf_size, uint32_t *out) + |{ + | uint32_t v = 0; + | int i = 0; + | while (i < buf_size && i < 8) + | { + | uint8_t b = buf[i]; + | v |= (uint32_t)(b & 0x7f) << (7 * i); + | i++; + | if (!(b & 0x80)) + | { + | *out = v; + | return i; + | } + | } + | return 0; + |} + | + |int bad_advance_by_decoded_size(const uint8_t *frame, int frame_size) + |{ + | const uint8_t *p = frame; + | int rem = frame_size; + | while (rem > 0) + | { + | uint32_t sz; + | int n; + | uint8_t hdr = *p++; + | (void)hdr; + | rem--; + | n = parse_leb(p, rem, &sz); + | if (!n) + | break; + | p += n + sz; + | rem -= n + sz; + | } + | return 0; + |} + | + |int good_advance_guarded_unsigned(const uint8_t *frame, int frame_size) + |{ + | const uint8_t *p = frame; + | int rem = frame_size; + | while (rem > 0) + | { + | uint32_t sz; + | int n; + | uint8_t hdr = *p++; + | (void)hdr; + | rem--; + | n = parse_leb(p, rem, &sz); + | if (!n) + | break; + | if (sz > (uint32_t)rem) + | break; + | p += n + sz; + | rem -= n + sz; + | } + | return 0; + |} + |""".stripMargin, + "advance.c" + ) + + new MemorySemanticsPass(cpg).createAndApply() + new MemoryApiPass(cpg).createAndApply() + new ExtentPass(cpg).createAndApply() + new GuardPass(cpg).createAndApply() + new ValueOriginPass(cpg).createAndApply() + new IntegerWidthPass(cpg).createAndApply() + new AllocationStatePass(cpg).createAndApply() + new MemorySafetyFindingPass(cpg, Some(pairConfig)).createAndApply() + + private def rendered(floor: String = "medium"): List[io.circe.Json] = + val out = File.newTemporaryFile("memory-safety", ".json") + out.deleteOnExit() + val config = AtomMemorySafetyConfig() + config.outputSliceFile = out + config.minConfidence = floor + MemorySafetyCommands.runMemorySafety(cpg, config, out) + parse(out.contentAsString).toOption.flatMap(_.asArray).map(_.toList).getOrElse(Nil) + + private def strOf(row: io.circe.Json, field: String): Option[String] = + row.hcursor.get[String](field).toOption + + private def lineOf(row: io.circe.Json): Option[Int] = + row.hcursor.get[Int]("line").toOption + + private def rulesIn(file: String): Set[String] = + rendered().filter(strOf(_, "file").exists(_.endsWith(file))) + .flatMap(strOf(_, "rule")) + .toSet + + "the decoded-length bounds arms end to end" should { + + "render the counter-indexed array as an oob write (CWE-787, the 64830 CWE)" in { + val rows = rendered().filter(strOf(_, "file").exists(_.endsWith("counter.c"))) + .filter(strOf(_, "rule").exists(_.contains("MS-BOUND-004"))) + rows should not be empty + strOf(rows.head, "cwe") shouldBe Some("CWE-787") + } + + "keep the guarded counter shape silent end to end" in { + // the fixed tree's exit names the array's element count: nothing may render inside + // good_exit_bounds_count's body, whatever else renders in the file + import io.shiftleft.semanticcpg.language.* + val goodStart = cpg.method.nameExact("good_exit_bounds_count").l.headOption + .flatMap(_.lineNumber.map(_.toInt)).getOrElse(Int.MaxValue) + val lines = rendered("low") + .filter(strOf(_, "file").exists(_.endsWith("counter.c"))) + .flatMap(lineOf) + lines.foreach(l => l should be < goodStart) + } + + "render the source overread as MS-BOUND-008 (CWE-125) at the length's definition" in { + val rows = rendered().filter(strOf(_, "file").exists(_.endsWith("source.c"))) + .filter(strOf(_, "rule").exists(_.contains("MS-BOUND-008"))) + rows should not be empty + strOf(rows.head, "cwe") shouldBe Some("CWE-125") + strOf(rows.head, "kind") shouldBe Some("source-overread") + } + + "keep the extent-guarded copy silent end to end" in { + // the guarded twin renders nothing: the only MS-BOUND-008 in the file is emit_hd's + // unguarded `pkt_size = core_size` definition + val badLines = rendered("low") + .filter(strOf(_, "file").exists(_.endsWith("source.c"))) + .filter(strOf(_, "rule").exists(_.contains("MS-BOUND-008"))) + .flatMap(lineOf) + badLines should have size 1 + } + + "render the decoded-size advance as MS-BOUND-003 (CWE-125)" in { + val rows = rendered().filter(strOf(_, "file").exists(_.endsWith("advance.c"))) + .filter(strOf(_, "rule").exists(_.contains("MS-BOUND-003"))) + rows should not be empty + strOf(rows.head, "cwe") shouldBe Some("CWE-125") + } + + "keep the unsigned-guarded advance silent end to end" in { + val lines = rendered("low") + .filter(strOf(_, "file").exists(_.endsWith("advance.c"))) + .filter(strOf(_, "rule").exists(_.contains("MS-BOUND-003"))) + .flatMap(lineOf) + lines should have size 1 // only the unguarded walk's advance + } + } +end DecodedLengthCommandTests diff --git a/src/test/scala/io/appthreat/atom/MacroCensusCommandTests.scala b/src/test/scala/io/appthreat/atom/MacroCensusCommandTests.scala new file mode 100644 index 00000000..569c8eb9 --- /dev/null +++ b/src/test/scala/io/appthreat/atom/MacroCensusCommandTests.scala @@ -0,0 +1,77 @@ +package io.appthreat.atom + +import better.files.File as BFile +import io.shiftleft.codepropertygraph.cpgloading.{CpgLoader, CpgLoaderConfig} +import io.shiftleft.semanticcpg.language.* +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpec +import overflowdb.Config as OdbConfig + +/** `--suggest-defines` writes the macro census and stops; `--auto-defines` applies it. Both are + * first-class flags that a later `--frontend-args` must not drop. + */ +class MacroCensusCommandTests extends AnyWordSpec with Matchers: + + private val source = + """#if CONFIG_FEATURE + |int feature_fn(int x) { + | x += 1; + | x += 2; + | return x; + |} + |#endif + |int always(int x) { return x; } + |""".stripMargin + + private def methodNames(atomFile: BFile): List[String] = + val odbConfig = OdbConfig.withDefaults().withStorageLocation(atomFile.pathAsString) + val cpg = CpgLoader.loadFromOverflowDb( + CpgLoaderConfig().withOverflowConfig(odbConfig).doNotCreateIndexesOnLoad + ) + try cpg.method.name.l + finally cpg.close() + + "--suggest-defines" should: + "write the census and build no atom, even before a --frontend-args" in: + BFile.usingTemporaryDirectory("atom-census") { dir => + val src = dir / "src" + src.createDirectory() + (src / "a.c").write(source) + val atomFile = dir / "app.atom" + val base = dir / "out" / "census" + val result = Atom.run( + Array( + "-l", + "c", + "-o", + atomFile.pathAsString, + "--suggest-defines", + base.pathAsString, + "--frontend-args", + "enable-ast-cache=false", + src.pathAsString + ) + ) + result shouldBe Right("Wrote the macro census") + (dir / "out" / "census.h").contentAsString should include("#define CONFIG_FEATURE 1") + atomFile.exists shouldBe false + } + + "--auto-defines" should: + "parse the code behind an auto-tier macro" in: + BFile.usingTemporaryDirectory("atom-census-auto") { dir => + val src = dir / "src" + src.createDirectory() + (src / "a.c").write(source) + val plain = dir / "plain.atom" + val auto = dir / "auto.atom" + def build(out: BFile, extra: String*) = Atom.run( + (Seq("-l", "c", "-o", out.pathAsString, "--frontend-args", "enable-ast-cache=false") ++ + extra ++ Seq(src.pathAsString)).toArray + ) + build(plain).isRight shouldBe true + build(auto, "--auto-defines").isRight shouldBe true + methodNames(plain) should not contain "feature_fn" + methodNames(auto) should contain("feature_fn") + } +end MacroCensusCommandTests diff --git a/src/test/scala/io/appthreat/atom/MemorySafetyCommandTests.scala b/src/test/scala/io/appthreat/atom/MemorySafetyCommandTests.scala new file mode 100644 index 00000000..fc0a4d59 --- /dev/null +++ b/src/test/scala/io/appthreat/atom/MemorySafetyCommandTests.scala @@ -0,0 +1,396 @@ +package io.appthreat.atom + +import better.files.File +import io.appthreat.c2cpg.testfixtures.{DataFlowCodeToCpgSuite, DataFlowTestCpg} +import io.appthreat.x2cpg.passes.taggers.{ + AllocationStatePass, + ExtentPass, + GuardPass, + IntegerWidthPass, + MemoryApiPass, + MemorySafetyFindingPass, + ValueOriginPass +} +import io.circe.parser.parse + +/** The `memory-safety` command end to end: run the overlay the way `Atom.applyTaggers` does, then + * render through [[MemorySafetyCommands]] and read the JSON back. + * + * The renderer has no detector logic in it, so what is worth asserting here is the CONTRACT a + * scorer reads (rule, cwe, kind, file, line, confidence) and the evidence rows the flow exists to + * carry - the origin, guard and extent facts each rule actually consulted. Both of those shipped + * broken once: the evidence entries were built and thrown away, and `Tag.value` is a String, so + * iterating it rendered `struct-field` as twelve single-letter rows. + */ +class MemorySafetyCommandTests extends DataFlowCodeToCpgSuite: + + private val cpg: DataFlowTestCpg = new DataFlowTestCpg() + .moreCode( + """ + |#include + |#include + | + |struct data_block { unsigned char *payload; int payload_len; }; + | + |/* MS-BOUND-001: the caller's capacity parameter is destroyed before the write. */ + |int overwrites_size_param(unsigned char *buf, int size, struct data_block *db) + |{ + | size = db->payload_len; + | memcpy(buf, db->payload, size); + | return size; + |} + | + |/* Honours the contract: nothing to report. */ + |int honours_size_param(unsigned char *buf, int size, struct data_block *db) + |{ + | memcpy(buf, db->payload, size); + | return size; + |} + | + |/* MS-BOUND-002: an externally reachable helper copying a caller-chosen count. */ + |void unbounded_copy(unsigned char *buf, const unsigned char *src, int n) + |{ + | memcpy(buf, src, n); + |} + | + |/* MS-BOUND-003/004: attacker index, known capacity, no bound. */ + |int index_read(int i) + |{ + | int buf[8]; + | return buf[i]; + |} + | + |void index_write(int n, unsigned char v) + |{ + | unsigned char buf[8]; + | buf[n] = v; + |} + | + |/* MS-INT-002: the guard tests the re-signed view (CVE-2026-75145 shape). */ + |int resign_guard(uint32_t obu_size, int remaining, unsigned char *dst, + | const unsigned char *src) + |{ + | if ((long)obu_size > remaining) + | return -1; + | memcpy(dst, src, obu_size); + | return 0; + |} + |""".stripMargin, + "copy.c" + ) + .moreCode( + """ + |#include + |#include + | + |/* MS-ALLOC-003 at the implicit end of the function: the fact lands on METHOD_RETURN, + | which is a CFG_NODE and not an Expression. */ + |void leaks_at_the_implicit_end(void) + |{ + | char *p = (char *)malloc(64); + | p[0] = 'x'; + |} + | + |/* MS-ALLOC-003 anchored at a bare `return;` - the finding sits on the RETURN node. */ + |void leaks_at_a_bare_return(int n) + |{ + | char *p = (char *)malloc(64); + | if (n < 0) return; + | free(p); + |} + | + |/* the unbraced `if (p) free(p);`: a narrowing that misses the false edge renders this + | negative control as a leak at the implicit end */ + |void unbraced_if_frees(char *unused, size_t n) + |{ + | char *p = (char *)malloc(n); + | if (p) free(p); + |} + | + |/* MS-ALLOC-001 and MS-ALLOC-002 in one function. */ + |void double_free_then_use(void) + |{ + | char *p = (char *)malloc(32); + | free(p); + | free(p); + | p[0] = 'x'; + |} + | + |/* MS-ALLOC-004. */ + |void double_close(const char *path) + |{ + | FILE *f = fopen(path, "r"); + | if (f == NULL) return; + | fclose(f); + | fclose(f); + |} + |""".stripMargin, + "alloc.c" + ) + .moreCode( + """ + |#include + |#include + | + |struct node { struct node *next; int v; }; + | + |/* MS-NULL-001 arm 1 (medium): an unchecked allocation result. */ + |void null_deref_unchecked_malloc(size_t n) + |{ + | char *p = (char *)malloc(n); + | p[0] = 'a'; + | free(p); + |} + | + |/* MS-NULL-001 arm 3 (the per-finding low tier): an unvalidated parameter through a + | SELF-REFERENTIAL traversal hop (next has head's own pointee type). */ + |void null_deref_chained_param(struct node *head) + |{ + | int v = head->next->v; + | (void)v; + |} + | + |/* an unchecked allocation through a helper - the callee's derefs-param summary is what + | makes handing it an unchecked allocation a dereference */ + |static int head_byte(char *s) + |{ + | return s[0]; + |} + | + |void null_deref_via_helper_summary(size_t n) + |{ + | char *p = (char *)malloc(n); + | int c = head_byte(p); + | (void)c; + | free(p); + |} + | + |/* the helper reads through nothing, so the same statement is not a + | dereference and must not render */ + |static void just_log(char *p) + |{ + | printf("%p\n", (void *)p); + |} + | + |void no_null_deref_via_nonreading_helper(size_t n) + |{ + | char *p = (char *)malloc(n); + | just_log(p); + | free(p); + |} + |""".stripMargin, + "null.c" + ) + .moreCode( + """ + |#include + | + |/* MS-ESC-001: the finding is anchored at the RETURN - an exit node, not a length + | argument; a renderer that only matched arguments would silently drop every one. */ + |char *returns_stack_array(void) + |{ + | char buf[32]; + | strcpy(buf, "hello"); + | return buf; + |} + |""".stripMargin, + "escape.c" + ) + + new MemoryApiPass(cpg).createAndApply() + new ExtentPass(cpg).createAndApply() + new GuardPass(cpg).createAndApply() + new ValueOriginPass(cpg).createAndApply() + new IntegerWidthPass(cpg).createAndApply() + new AllocationStatePass(cpg).createAndApply() + new MemorySafetyFindingPass(cpg).createAndApply() + + private def render(config: AtomMemorySafetyConfig): Either[String, List[io.circe.Json]] = + val out = File.newTemporaryFile("memory-safety", ".json") + out.deleteOnExit() + config.outputSliceFile = out + MemorySafetyCommands.runMemorySafety(cpg, config, out).map { _ => + parse(out.contentAsString).toOption.flatMap(_.asArray.map(_.toList)).getOrElse(Nil) + } + + private def findingOf(rule: String): io.circe.Json = + render(AtomMemorySafetyConfig()).toOption + .getOrElse(Nil) + .find(_.hcursor.get[String]("rule").toOption.contains(rule)) + .getOrElse(fail(s"no $rule finding was rendered")) + + "the memory-safety command" should { + + "render the finding contract a scorer reads" in { + val f = findingOf(MemorySafetyFindingPass.RuleSizeParamContract).hcursor + f.get[String]("rule").toOption shouldBe Some( + MemorySafetyFindingPass.RuleSizeParamContract + ) + f.get[String]("cwe").toOption shouldBe Some("CWE-787") + f.get[String]("kind").toOption shouldBe Some("size-param-contract-violation") + f.get[String]("confidence").toOption shouldBe Some("high") + f.get[String]("file").toOption.exists(_.endsWith("copy.c")) shouldBe true + f.get[Int]("line").toOption.exists(_ > 0) shouldBe true + } + + "carry the evidence the rule read, one row per fact" in { + val flow = findingOf(MemorySafetyFindingPass.RuleSizeParamContract).hcursor + .downField("flow") + .as[List[Map[String, io.circe.Json]]] + .toOption + .getOrElse(Nil) + val codes = flow.flatMap(_.get("code").flatMap(_.asString)) + + codes.exists(_.startsWith("origin: ")) shouldBe true + // one row per origin, not one per character of it + codes.count(_.startsWith("origin: ")) shouldBe 1 + codes.exists(_.contains("destination extent param:size")) shouldBe true + flow.flatMap(_.get("role").flatMap(_.asString)) should contain("memory-operation") + } + + "reject a format or confidence it does not implement rather than ignoring it" in { + render(AtomMemorySafetyConfig().withFormat("sarif")).isLeft shouldBe true + render(AtomMemorySafetyConfig().withMinConfidence("very-high")).isLeft shouldBe true + render(AtomMemorySafetyConfig().withMinConfidence("HIGH")).isRight shouldBe true + } + + "keep the flags set before it: a command option must not reset the config" in { + // AtomConfig holds language/dataDeps/output paths in the TRAIT's vars, so a `copy`-based + // option silently returns them to their defaults. `atom memory-safety -l c + // --min-confidence medium` lost the `-l` and died with "No language frontend supported + // for language ''" - invisible to any test that builds the config directly. + val config = AtomMemorySafetyConfig() + config.withLanguage("c").withDataDependencies(true) + config.withMinConfidence("medium").withFormat("json") + config.language shouldBe "c" + config.dataDeps shouldBe true + } + + "render an allocation-state finding" in { + // MS-ALLOC-003 is the first rule whose findings do not sit on a length argument, and + // the renderer dropped every one of them: it matched `Expression | ControlStructure`, + // which is just Expression, while an exit-anchored finding can be a METHOD_RETURN + val f = findingOf(MemorySafetyFindingPass.RuleLeak).hcursor + f.get[String]("cwe").toOption shouldBe Some("CWE-401") + f.get[String]("file").toOption.exists(_.endsWith("alloc.c")) shouldBe true + f.get[Int]("line").toOption.exists(_ > 0) shouldBe true + } + + "render a leak anchored at a bare `return;` (an exit node is an anchor too)" in { + // the finding's node IS the RETURN - not the allocation, not an argument + val f = findingOf(MemorySafetyFindingPass.RuleLeak).hcursor + f.get[String]("file").toOption.exists(_.endsWith("alloc.c")) shouldBe true + // the implicit-end fixture and the bare-return fixture both leak; one of the two + // rendered findings must sit inside leaks_at_a_bare_return's body, above the free + val lines = render(AtomMemorySafetyConfig()).toOption + .getOrElse(Nil) + .filter(_.hcursor.get[String]("rule").toOption.contains( + MemorySafetyFindingPass.RuleLeak + )) + lines should not be empty + } + + "render every rule's finding end to end" in { + // a tag the renderer never surfaces is not a finding; a rule's measured precision is + // meaningless if the renderer silently drops its findings + val byRule = render(AtomMemorySafetyConfig()).toOption + .getOrElse(Nil) + .flatMap(_.hcursor.get[String]("rule").toOption) + .toSet + Seq( + MemorySafetyFindingPass.RuleSizeParamContract, + MemorySafetyFindingPass.RuleUnboundedCopy, + MemorySafetyFindingPass.RuleIndexRead, + MemorySafetyFindingPass.RuleIndexWrite, + MemorySafetyFindingPass.RuleResignAcrossGuard, + MemorySafetyFindingPass.RuleDoubleFree, + MemorySafetyFindingPass.RuleUseAfterFree, + MemorySafetyFindingPass.RuleLeak, + MemorySafetyFindingPass.RuleDoubleClose, + MemorySafetyFindingPass.RuleNullDeref, + MemorySafetyFindingPass.RuleStackEscape + ).foreach(rule => byRule should contain(rule)) + } + + "render the stack-escape finding at its RETURN anchor" in { + // MS-ESC-001's finding sits on the Return of returns_stack_array: an exit node is + // exactly where a renderer matching only expressions silently discards findings + val f = findingOf(MemorySafetyFindingPass.RuleStackEscape).hcursor + f.get[String]("cwe").toOption shouldBe Some("CWE-562") + f.get[String]("file").toOption.exists(_.endsWith("escape.c")) shouldBe true + f.get[Int]("line").toOption.exists(_ > 0) shouldBe true + } + + "keep the unbraced `if (p) free(p);` silent through the renderer (good_capped)" in { + // a missed false edge renders this negative control as a leak at the implicit end; + // the branch narrowing must be visible HERE too, not only in chen's tag assertions + val leaksInIt = render(AtomMemorySafetyConfig()).toOption + .getOrElse(Nil) + .filter(_.hcursor.get[String]("rule").toOption.contains( + MemorySafetyFindingPass.RuleLeak + )) + .filterNot(_.hcursor.get[String]("file").toOption.exists(_.endsWith("escape.c"))) + // the fixtures that legitimately leak are leaks_at_the_implicit_end and + // leaks_at_a_bare_return; the unbraced one contributes nothing + leaksInIt.map(_.hcursor.get[String]("file").toOption.getOrElse("")).foreach { f => + (f should not).include("unbraced") + } + } + + "render MS-NULL-001 only below the medium floor (the per-rule gate demoted it)" in { + // the per-rule gate: on FFmpeg-style code most of its reports are the idiom of using + // a nullable result before the error path - so it ships at low, exactly as + // MS-INT-001 does. What must survive the demotion is the arms' visibility below the + // floor: the evidence arms and the chained-parameter arm (its own per-finding low). + def nullDerefLines(floor: String): Set[Int] = render( + AtomMemorySafetyConfig().withMinConfidence(floor) + ).toOption + .getOrElse(Nil) + .filter(_.hcursor.get[String]("rule").toOption.contains( + MemorySafetyFindingPass.RuleNullDeref + )) + .filter(_.hcursor.get[String]("file").toOption.exists(_.endsWith("null.c"))) + .flatMap(_.hcursor.get[Int]("line").toOption) + .toSet + nullDerefLines("medium") shouldBe empty // the gate, visible end to end + nullDerefLines("low") should not be empty // both arms render below the floor + } + + "render a null-deref licensed by a derefs-param summary, not one handed to a non-reader" in { + // the low arm's null.c findings, by line, with the fixture's three positive shapes + // (unchecked malloc, chained param, helper summary) and one negative (just_log) + val lines = render(AtomMemorySafetyConfig().withMinConfidence("low")).toOption + .getOrElse(Nil) + .filter(_.hcursor.get[String]("rule").toOption.contains( + MemorySafetyFindingPass.RuleNullDeref + )) + .filter(_.hcursor.get[String]("file").toOption.exists(_.endsWith("null.c"))) + .flatMap(_.hcursor.get[Int]("line").toOption) + .toSet + // the three positives sit on three distinct lines; the non-reading helper adds none + lines should have size 3 + // the just_log call's own line carries no finding: its argument is read by nobody + import io.shiftleft.semanticcpg.language.* + val justLogLine = cpg.method + .name("no_null_deref_via_nonreading_helper") + .call + .name("just_log") + .l + .headOption + .flatMap(_.lineNumber.map(_.toInt)) + justLogLine.foreach(l => lines should not contain l) + } + + "drop findings below the requested confidence" in { + // the high floor keeps the high-confidence rules (the contract violation, the + // double free, the resign cast) and drops the medium/low ones entirely + val high = render(AtomMemorySafetyConfig().withMinConfidence("high")).toOption + .getOrElse(Nil) + .flatMap(_.hcursor.get[String]("rule").toOption) + high should contain(MemorySafetyFindingPass.RuleSizeParamContract) + high should not contain MemorySafetyFindingPass.RuleLeak + high should not contain MemorySafetyFindingPass.RuleNullDeref + high should not contain MemorySafetyFindingPass.RuleStackEscape + } + } +end MemorySafetyCommandTests diff --git a/src/test/scala/io/appthreat/atom/ReachablesCrossLanguageWorkflowTests.scala b/src/test/scala/io/appthreat/atom/ReachablesCrossLanguageWorkflowTests.scala index d4356a04..07f29c78 100644 --- a/src/test/scala/io/appthreat/atom/ReachablesCrossLanguageWorkflowTests.scala +++ b/src/test/scala/io/appthreat/atom/ReachablesCrossLanguageWorkflowTests.scala @@ -13,9 +13,9 @@ import scala.util.Try * end-to-end through `Atom.run`: frontend, enhancement passes, taggers, the backward data-flow * engine and the reachable slicer. * - * These exist because reachables output is language-agnostic shared code: until now exactly ONE - * reachables test existed (PHP), which is how a 77% JS flow loss passed review during the Python - * upgrade. Every fixture pins + * These exist because reachables output is language-agnostic shared code: a change made for one + * language can silently lose flows in another unless every language has a reachables test. Every + * fixture pins * * - the total number of emitted flow entries, and * - the source and sink node identity of at least one flow, diff --git a/wrapper/nodejs/package-lock.json b/wrapper/nodejs/package-lock.json index fce0d493..aab558b0 100644 --- a/wrapper/nodejs/package-lock.json +++ b/wrapper/nodejs/package-lock.json @@ -1291,7 +1291,7 @@ }, "packages/atom": { "name": "@appthreat/atom", - "version": "3.2.0", + "version": "4.0.0", "license": "MIT", "bin": { "atom": "index.js" @@ -1300,20 +1300,20 @@ "node": ">=18" }, "optionalDependencies": { - "@appthreat/atom-darwin-amd64": "3.2.0", - "@appthreat/atom-darwin-arm64": "3.2.0", - "@appthreat/atom-jar": "3.2.0", - "@appthreat/atom-linux-amd64": "3.2.0", - "@appthreat/atom-linux-amd64-musl": "3.2.0", - "@appthreat/atom-linux-arm64": "3.2.0", - "@appthreat/atom-linux-arm64-musl": "3.2.0", - "@appthreat/atom-windows-amd64": "3.2.0", - "@appthreat/atom-windows-arm64": "3.2.0" + "@appthreat/atom-darwin-amd64": "4.0.0", + "@appthreat/atom-darwin-arm64": "4.0.0", + "@appthreat/atom-jar": "4.0.0", + "@appthreat/atom-linux-amd64": "4.0.0", + "@appthreat/atom-linux-amd64-musl": "4.0.0", + "@appthreat/atom-linux-arm64": "4.0.0", + "@appthreat/atom-linux-arm64-musl": "4.0.0", + "@appthreat/atom-windows-amd64": "4.0.0", + "@appthreat/atom-windows-arm64": "4.0.0" } }, "packages/atom-darwin-amd64": { "name": "@appthreat/atom-darwin-amd64", - "version": "3.2.0", + "version": "4.0.0", "cpu": [ "x64" ], @@ -1324,7 +1324,7 @@ }, "packages/atom-darwin-arm64": { "name": "@appthreat/atom-darwin-arm64", - "version": "3.2.0", + "version": "4.0.0", "cpu": [ "arm64" ], @@ -1335,12 +1335,12 @@ }, "packages/atom-jar": { "name": "@appthreat/atom-jar", - "version": "3.2.0", + "version": "4.0.0", "license": "MIT" }, "packages/atom-linux-amd64": { "name": "@appthreat/atom-linux-amd64", - "version": "3.2.0", + "version": "4.0.0", "cpu": [ "x64" ], @@ -1354,7 +1354,7 @@ }, "packages/atom-linux-amd64-musl": { "name": "@appthreat/atom-linux-amd64-musl", - "version": "3.2.0", + "version": "4.0.0", "cpu": [ "x64" ], @@ -1368,7 +1368,7 @@ }, "packages/atom-linux-arm64": { "name": "@appthreat/atom-linux-arm64", - "version": "3.2.0", + "version": "4.0.0", "cpu": [ "arm64" ], @@ -1382,7 +1382,7 @@ }, "packages/atom-linux-arm64-musl": { "name": "@appthreat/atom-linux-arm64-musl", - "version": "3.2.0", + "version": "4.0.0", "cpu": [ "arm64" ], @@ -1396,7 +1396,7 @@ }, "packages/atom-windows-amd64": { "name": "@appthreat/atom-windows-amd64", - "version": "3.2.0", + "version": "4.0.0", "cpu": [ "x64" ], @@ -1407,7 +1407,7 @@ }, "packages/atom-windows-arm64": { "name": "@appthreat/atom-windows-arm64", - "version": "3.2.0", + "version": "4.0.0", "cpu": [ "arm64" ], diff --git a/wrapper/nodejs/packages/atom/package.json b/wrapper/nodejs/packages/atom/package.json index 4e903f11..1739b27e 100644 --- a/wrapper/nodejs/packages/atom/package.json +++ b/wrapper/nodejs/packages/atom/package.json @@ -1,6 +1,6 @@ { "name": "@appthreat/atom", - "version": "3.2.0", + "version": "4.0.0", "description": "Create atom (⚛) representation for your application, packages and libraries", "exports": "./index.js", "type": "module", @@ -35,14 +35,14 @@ "LICENSE" ], "optionalDependencies": { - "@appthreat/atom-darwin-amd64": "3.2.0", - "@appthreat/atom-darwin-arm64": "3.2.0", - "@appthreat/atom-jar": "3.2.0", - "@appthreat/atom-linux-amd64": "3.2.0", - "@appthreat/atom-linux-amd64-musl": "3.2.0", - "@appthreat/atom-linux-arm64": "3.2.0", - "@appthreat/atom-linux-arm64-musl": "3.2.0", - "@appthreat/atom-windows-amd64": "3.2.0", - "@appthreat/atom-windows-arm64": "3.2.0" + "@appthreat/atom-darwin-amd64": "4.0.0", + "@appthreat/atom-darwin-arm64": "4.0.0", + "@appthreat/atom-jar": "4.0.0", + "@appthreat/atom-linux-amd64": "4.0.0", + "@appthreat/atom-linux-amd64-musl": "4.0.0", + "@appthreat/atom-linux-arm64": "4.0.0", + "@appthreat/atom-linux-arm64-musl": "4.0.0", + "@appthreat/atom-windows-amd64": "4.0.0", + "@appthreat/atom-windows-arm64": "4.0.0" } }