From 94a0cd5cd7360490ebb8816e7e5a097802a3f5f4 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Sun, 12 Jul 2026 13:18:12 +0200 Subject: [PATCH 1/5] refactor: move WebPageTest execution to a separate workflow triggered by artifact metadata Signed-off-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> --- .github/workflows/lint.yml | 56 ++++++++++++++++++++++ .github/workflows/test.yml | 95 -------------------------------------- .github/workflows/wpt.yml | 81 ++++++++++++++++++++++++++++++++ 3 files changed, 137 insertions(+), 95 deletions(-) delete mode 100644 .github/workflows/test.yml create mode 100644 .github/workflows/wpt.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index db29d87ad..e79e034a2 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -23,6 +23,62 @@ jobs: - name: Validate run: npm run validate + - name: Check path filters + uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + wpt: + - 'src/technologies/*.json' + - 'src/categories.json' + - 'src/groups.json' + - 'tests/*.js' + - '.github/workflows/wpt.yml' + + - name: Parse WebPageTest URLs + if: steps.filter.outputs.wpt == 'true' + id: wpt-urls + env: + PR_BODY: ${{ github.event.pull_request.body }} + run: | + lines=() + while IFS= read -r line; do + lines+=("${line//[$'\r\n']}") + done <<< "$PR_BODY" + + start_index=-1 + for ((i=0; i<${#lines[@]}; i++)); do + if [[ "${lines[$i]}" == *"**Test websites**:"* ]]; then + start_index=$((i + 1)) + break + fi + done + + URLS="" + if [ $start_index -gt -1 ]; then + url_pattern="(https:\/\/[^[:space:]]+)" + for ((i=start_index; i<${#lines[@]}; i++)); do + if [[ ${lines[$i]} =~ $url_pattern ]]; then + URLS+="${BASH_REMATCH[1]} " + fi + done + fi + echo "urls=$URLS" >> $GITHUB_OUTPUT + + - name: Save PR info + if: steps.filter.outputs.wpt == 'true' + run: | + mkdir -p pr-metadata + echo "${{ github.event.pull_request.number }}" > pr-metadata/pr_number + echo "${{ steps.wpt-urls.outputs.urls }}" > pr-metadata/urls + + - name: Upload PR Metadata + if: steps.filter.outputs.wpt == 'true' + uses: actions/upload-artifact@v4 + with: + name: pr-metadata + path: pr-metadata/ + dependabot: name: Dependabot auto-merge runs-on: ubuntu-slim diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 5afd86bb1..000000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,95 +0,0 @@ -name: Tests - -on: - pull_request_target: - branches: - - main - paths: - - "src/technologies/*.json" - - "src/categories.json" - - "src/groups.json" - - "tests/*.js" - - ".github/workflows/test.yml" - workflow_dispatch: - -jobs: - test: - name: WebPageTest Test Cases - runs-on: ubuntu-slim - permissions: - contents: read - pull-requests: write - if: | - ${{ github.repository == 'HTTPArchive/wappalyzer' || - github.event.pull_request.base.repo.full_name == 'HTTPArchive/wappalyzer' }} - steps: - - name: Checkout - uses: actions/checkout@v7 - with: - ref: ${{ github.event.pull_request.head.sha }} - - - name: Install dependencies - run: npm install - - - name: Validate - run: npm run validate - - - name: Run WebPageTest with unit tests - id: unit-test - env: - WPT_SERVER: "webpagetest.httparchive.org" - WPT_API_KEY: ${{ secrets.HA_API_KEY }} - PR_NUMBER: ${{ github.event.pull_request.number }} - run: npm run test - - - name: Run WebPageTest for more websites - id: wpt-test - env: - WPT_SERVER: "webpagetest.httparchive.org" - WPT_API_KEY: ${{ secrets.HA_API_KEY }} - PR_NUMBER: ${{ github.event.pull_request.number }} - PR_BODY: ${{ github.event.pull_request.body }} - run: | - # Read PR body into an array, removing line breaks and carriage returns - lines=() - while IFS= read -r line; do - lines+=("${line//[$'\r\n']}") - done <<< "$PR_BODY" - - # Find the index of the line after "**Test websites**:" - start_index=-1 - for ((i=0; i<${#lines[@]}; i++)); do - if [[ "${lines[$i]}" == *"**Test websites**:"* ]]; then - start_index=$((i + 1)) - break - fi - done - - # If the index is valid, then parse the URLs - if [ $start_index -gt -1 ]; then - # Initialize an array for URLs - URLS=() - url_pattern="(https:\/\/[^[:space:]]+)" - - for ((i=start_index; i<${#lines[@]}; i++)); do - if [[ ${lines[$i]} =~ $url_pattern ]]; then - URLS+=("${BASH_REMATCH[1]}") - fi - done - - # Run WebPageTest for each URL - for TEST_WEBSITE in "${URLS[@]}"; do - echo "::group::Running WPT test for $TEST_WEBSITE" - node tests/wpt.js "$TEST_WEBSITE" - echo "::endgroup::" - done - else - echo "No websites found." - fi - - - name: Add comment with results - uses: mshick/add-pr-comment@v3 - if: steps.unit-test.outcome == 'success' || steps.wpt-test.outcome == 'success' - with: - refresh-message-position: true - message-path: test-results.md diff --git a/.github/workflows/wpt.yml b/.github/workflows/wpt.yml new file mode 100644 index 000000000..56d1f6ebc --- /dev/null +++ b/.github/workflows/wpt.yml @@ -0,0 +1,81 @@ +name: WPT Run & Comment + +on: + workflow_run: + workflows: ["Lint, validate & Dependabot auto-merge"] + types: [completed] + +jobs: + wpt: + name: Run WebPageTest + runs-on: ubuntu-slim + if: ${{ github.event.workflow_run.conclusion == 'success' }} + permissions: + contents: read + pull-requests: write + actions: read + steps: + - name: Checkout Trusted Code + uses: actions/checkout@v7 + + - name: Download PR Metadata + uses: actions/download-artifact@v4 + id: download-artifact + continue-on-error: true + with: + name: pr-metadata + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + path: pr-metadata + + - name: Check if WPT run is needed + id: check-run + run: | + if [ "${{ steps.download-artifact.outcome }}" = "success" ] && [ -f pr-metadata/pr_number ]; then + echo "run_wpt=true" >> $GITHUB_OUTPUT + echo "PR_NUMBER=$(cat pr-metadata/pr_number)" >> $GITHUB_ENV + echo "WPT_URLS=$(cat pr-metadata/urls)" >> $GITHUB_ENV + echo "Starting WPT run." + else + echo "run_wpt=false" >> $GITHUB_OUTPUT + echo "No WPT run needed." + fi + + - name: Install dependencies + if: steps.check-run.outputs.run_wpt == 'true' + run: npm install + + - name: Run WebPageTest with unit tests + if: steps.check-run.outputs.run_wpt == 'true' + id: unit-test + env: + WPT_SERVER: "webpagetest.httparchive.org" + WPT_API_KEY: ${{ secrets.HA_API_KEY }} + PR_NUMBER: ${{ env.PR_NUMBER }} + run: npm run test + + - name: Run WebPageTest for more websites + if: steps.check-run.outputs.run_wpt == 'true' + id: wpt-test + env: + WPT_SERVER: "webpagetest.httparchive.org" + WPT_API_KEY: ${{ secrets.HA_API_KEY }} + PR_NUMBER: ${{ env.PR_NUMBER }} + run: | + if [ -n "${{ env.WPT_URLS }}" ]; then + for TEST_WEBSITE in ${{ env.WPT_URLS }}; do + echo "::group::Running WPT test for $TEST_WEBSITE" + node tests/wpt.js "$TEST_WEBSITE" + echo "::endgroup::" + done + else + echo "No websites found." + fi + + - name: Add comment with results + if: steps.check-run.outputs.run_wpt == 'true' && (steps.unit-test.outcome == 'success' || steps.wpt-test.outcome == 'success') + uses: mshick/add-pr-comment@v3 + with: + refresh-message-position: true + message-path: test-results.md + issue: ${{ env.PR_NUMBER }} From bf35bbd2f96022860d625547e28b378db94e927e Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Sun, 12 Jul 2026 13:23:13 +0200 Subject: [PATCH 2/5] Potential fix for pull request finding 'CodeQL / Environment variable built from user-controlled sources' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- .github/workflows/wpt.yml | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/.github/workflows/wpt.yml b/.github/workflows/wpt.yml index 56d1f6ebc..685c8ac04 100644 --- a/.github/workflows/wpt.yml +++ b/.github/workflows/wpt.yml @@ -32,12 +32,27 @@ jobs: id: check-run run: | if [ "${{ steps.download-artifact.outcome }}" = "success" ] && [ -f pr-metadata/pr_number ]; then - echo "run_wpt=true" >> $GITHUB_OUTPUT - echo "PR_NUMBER=$(cat pr-metadata/pr_number)" >> $GITHUB_ENV - echo "WPT_URLS=$(cat pr-metadata/urls)" >> $GITHUB_ENV + PR_NUMBER_RAW="$(cat pr-metadata/pr_number)" + if ! printf '%s' "$PR_NUMBER_RAW" | grep -Eq '^[0-9]+$'; then + echo "run_wpt=false" >> "$GITHUB_OUTPUT" + echo "Invalid PR number metadata." + exit 0 + fi + + WPT_URLS_RAW="" + if [ -f pr-metadata/urls ]; then + WPT_URLS_RAW="$(cat pr-metadata/urls)" + fi + + # Prevent environment-file injection by stripping CR/LF from untrusted artifact content. + WPT_URLS_SAFE="$(printf '%s' "$WPT_URLS_RAW" | tr '\r\n' ' ')" + + echo "run_wpt=true" >> "$GITHUB_OUTPUT" + printf 'PR_NUMBER=%s\n' "$PR_NUMBER_RAW" >> "$GITHUB_ENV" + printf 'WPT_URLS=%s\n' "$WPT_URLS_SAFE" >> "$GITHUB_ENV" echo "Starting WPT run." else - echo "run_wpt=false" >> $GITHUB_OUTPUT + echo "run_wpt=false" >> "$GITHUB_OUTPUT" echo "No WPT run needed." fi From 8add0206edfccf567434c1bbac19d5614d5f706a Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Sun, 12 Jul 2026 13:24:50 +0200 Subject: [PATCH 3/5] Potential fix for pull request finding 'CodeQL / Artifact poisoning' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- .github/workflows/wpt.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/wpt.yml b/.github/workflows/wpt.yml index 685c8ac04..f7913b7ef 100644 --- a/.github/workflows/wpt.yml +++ b/.github/workflows/wpt.yml @@ -26,13 +26,13 @@ jobs: name: pr-metadata run-id: ${{ github.event.workflow_run.id }} github-token: ${{ secrets.GITHUB_TOKEN }} - path: pr-metadata + path: ${{ runner.temp }}/pr-metadata - name: Check if WPT run is needed id: check-run run: | - if [ "${{ steps.download-artifact.outcome }}" = "success" ] && [ -f pr-metadata/pr_number ]; then - PR_NUMBER_RAW="$(cat pr-metadata/pr_number)" + if [ "${{ steps.download-artifact.outcome }}" = "success" ] && [ -f "${{ runner.temp }}/pr-metadata/pr_number" ]; then + PR_NUMBER_RAW="$(cat "${{ runner.temp }}/pr-metadata/pr_number")" if ! printf '%s' "$PR_NUMBER_RAW" | grep -Eq '^[0-9]+$'; then echo "run_wpt=false" >> "$GITHUB_OUTPUT" echo "Invalid PR number metadata." @@ -40,8 +40,8 @@ jobs: fi WPT_URLS_RAW="" - if [ -f pr-metadata/urls ]; then - WPT_URLS_RAW="$(cat pr-metadata/urls)" + if [ -f "${{ runner.temp }}/pr-metadata/urls" ]; then + WPT_URLS_RAW="$(cat "${{ runner.temp }}/pr-metadata/urls")" fi # Prevent environment-file injection by stripping CR/LF from untrusted artifact content. From fb424b283ef158452d6ba3aa429307bd47f6d981 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Sun, 12 Jul 2026 13:25:19 +0200 Subject: [PATCH 4/5] Potential fix for pull request finding 'CodeQL / Code injection' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- .github/workflows/wpt.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wpt.yml b/.github/workflows/wpt.yml index f7913b7ef..9d7d6c034 100644 --- a/.github/workflows/wpt.yml +++ b/.github/workflows/wpt.yml @@ -76,9 +76,10 @@ jobs: WPT_SERVER: "webpagetest.httparchive.org" WPT_API_KEY: ${{ secrets.HA_API_KEY }} PR_NUMBER: ${{ env.PR_NUMBER }} + WPT_URLS: ${{ env.WPT_URLS }} run: | - if [ -n "${{ env.WPT_URLS }}" ]; then - for TEST_WEBSITE in ${{ env.WPT_URLS }}; do + if [ -n "$WPT_URLS" ]; then + for TEST_WEBSITE in $WPT_URLS; do echo "::group::Running WPT test for $TEST_WEBSITE" node tests/wpt.js "$TEST_WEBSITE" echo "::endgroup::" From 85283a7917c97c1eb1d9daaf38dc69b9b3f2349a Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Sun, 12 Jul 2026 13:27:47 +0200 Subject: [PATCH 5/5] Potential fix for pull request finding 'CodeQL / Environment variable built from user-controlled sources' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- .github/workflows/wpt.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/wpt.yml b/.github/workflows/wpt.yml index 9d7d6c034..326facbdf 100644 --- a/.github/workflows/wpt.yml +++ b/.github/workflows/wpt.yml @@ -44,12 +44,12 @@ jobs: WPT_URLS_RAW="$(cat "${{ runner.temp }}/pr-metadata/urls")" fi - # Prevent environment-file injection by stripping CR/LF from untrusted artifact content. + # Prevent output/env-file injection by stripping CR/LF from untrusted artifact content. WPT_URLS_SAFE="$(printf '%s' "$WPT_URLS_RAW" | tr '\r\n' ' ')" echo "run_wpt=true" >> "$GITHUB_OUTPUT" - printf 'PR_NUMBER=%s\n' "$PR_NUMBER_RAW" >> "$GITHUB_ENV" - printf 'WPT_URLS=%s\n' "$WPT_URLS_SAFE" >> "$GITHUB_ENV" + printf 'pr_number=%s\n' "$PR_NUMBER_RAW" >> "$GITHUB_OUTPUT" + printf 'wpt_urls=%s\n' "$WPT_URLS_SAFE" >> "$GITHUB_OUTPUT" echo "Starting WPT run." else echo "run_wpt=false" >> "$GITHUB_OUTPUT" @@ -66,7 +66,7 @@ jobs: env: WPT_SERVER: "webpagetest.httparchive.org" WPT_API_KEY: ${{ secrets.HA_API_KEY }} - PR_NUMBER: ${{ env.PR_NUMBER }} + PR_NUMBER: ${{ steps.check-run.outputs.pr_number }} run: npm run test - name: Run WebPageTest for more websites @@ -75,8 +75,8 @@ jobs: env: WPT_SERVER: "webpagetest.httparchive.org" WPT_API_KEY: ${{ secrets.HA_API_KEY }} - PR_NUMBER: ${{ env.PR_NUMBER }} - WPT_URLS: ${{ env.WPT_URLS }} + PR_NUMBER: ${{ steps.check-run.outputs.pr_number }} + WPT_URLS: ${{ steps.check-run.outputs.wpt_urls }} run: | if [ -n "$WPT_URLS" ]; then for TEST_WEBSITE in $WPT_URLS; do @@ -94,4 +94,4 @@ jobs: with: refresh-message-position: true message-path: test-results.md - issue: ${{ env.PR_NUMBER }} + issue: ${{ steps.check-run.outputs.pr_number }}