Merge pull request #45091 from github/repo-sync #20
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Sync llms.txt | |
| # **What it does**: Generates docs.github.com/llms.txt and github.com/llms.txt | |
| # from the page catalog and popularity data, then opens PRs to update both. | |
| # **Why we have it**: Agents discover docs through llms.txt; the page list keeps | |
| # pace with what's actually popular without writers updating it by hand. | |
| # **Who does it impact**: Docs consumers via agents, and anyone landing on | |
| # github.com/llms.txt or docs.github.com/llms.txt. | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - main | |
| paths: | |
| - '.github/workflows/sync-llms-txt.yml' | |
| - 'data/llms-txt/**' | |
| - 'src/workflows/generate-llms-txt.ts' | |
| schedule: | |
| - cron: '20 16 * * 1' # Run every Monday at 16:20 UTC / 9:20 PDT / 8:20 PST | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' | |
| cancel-in-progress: true | |
| jobs: | |
| sync: | |
| name: Sync llms.txt | |
| if: github.repository == 'github/docs-internal' | |
| runs-on: ubuntu-latest | |
| env: | |
| BRANCH: sync-llms-txt | |
| steps: | |
| - name: Checkout docs-internal | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - name: Generate GitHub App token | |
| id: app-token | |
| uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 | |
| with: | |
| app-id: ${{ secrets.DOCS_BOT_APP_ID }} | |
| private-key: ${{ secrets.DOCS_BOT_APP_PRIVATE_KEY }} | |
| owner: github | |
| repositories: github,docs-internal,docs-engineering,docs-internal-data | |
| - uses: ./.github/actions/node-npm-setup | |
| - name: Generate llms.txt for docs.github.com | |
| env: | |
| DOCS_BOT_PAT_BASE: ${{ steps.app-token.outputs.token }} | |
| run: | | |
| npm run generate-llms-txt --silent -- \ | |
| --config data/llms-txt/config-docs.yml \ | |
| --output /tmp/docs-llms.txt | |
| echo "Generated docs llms.txt ($(wc -l < /tmp/docs-llms.txt) lines, $(wc -c < /tmp/docs-llms.txt) bytes)" | |
| - name: Generate llms.txt for github.com | |
| env: | |
| DOCS_BOT_PAT_BASE: ${{ steps.app-token.outputs.token }} | |
| run: | | |
| npm run generate-llms-txt --silent -- \ | |
| --config data/llms-txt/config-monolith.yml \ | |
| --output /tmp/monolith-llms.txt | |
| echo "Generated monolith llms.txt ($(wc -l < /tmp/monolith-llms.txt) lines, $(wc -c < /tmp/monolith-llms.txt) bytes)" | |
| # ---------- PR to docs-internal: update data/llms-txt/docs.md ---------- | |
| - name: Diff docs llms.txt against committed copy | |
| id: diff_docs | |
| run: | | |
| if diff -q /tmp/docs-llms.txt data/llms-txt/docs.md > /dev/null 2>&1; then | |
| echo "No docs changes, skipping" | |
| echo "changed=false" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "Docs changes detected" | |
| echo "changed=true" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Ensure sync branch exists in docs-internal | |
| if: steps.diff_docs.outputs.changed == 'true' | |
| env: | |
| GH_TOKEN: ${{ steps.app-token.outputs.token }} | |
| run: | | |
| REPO="github/docs-internal" | |
| if gh api "repos/$REPO/git/ref/heads/$BRANCH" --jq '.object.sha' > /dev/null 2>&1; then | |
| echo "Branch $BRANCH exists, fetching" | |
| git -c url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf="https://github.com/" \ | |
| fetch origin "$BRANCH" | |
| git checkout "$BRANCH" | |
| else | |
| echo "Branch $BRANCH does not exist, creating from main" | |
| git checkout -b "$BRANCH" | |
| fi | |
| - name: Commit and push docs.md | |
| if: steps.diff_docs.outputs.changed == 'true' | |
| env: | |
| GH_TOKEN: ${{ steps.app-token.outputs.token }} | |
| run: | | |
| cp /tmp/docs-llms.txt data/llms-txt/docs.md | |
| git config user.name "docs-bot" | |
| git config user.email "77750099+docs-bot@users.noreply.github.com" | |
| git add data/llms-txt/docs.md | |
| # diff_docs compares against main, but the sync branch may already | |
| # exist with this exact content (open PR from a prior run). In that | |
| # case there is nothing new to stage, and `git commit` would exit 1 | |
| # and fail the whole workflow. Skip the commit and push when the | |
| # branch is already up to date. | |
| if git diff --cached --quiet; then | |
| echo "Sync branch already has the latest generated docs.md; nothing to commit." | |
| else | |
| git commit -m "Update data/llms-txt/docs.md from popularity data" | |
| git push "https://x-access-token:${GH_TOKEN}@github.com/github/docs-internal.git" "$BRANCH" | |
| fi | |
| - name: Create or update docs-internal PR | |
| if: steps.diff_docs.outputs.changed == 'true' | |
| env: | |
| GH_TOKEN: ${{ steps.app-token.outputs.token }} | |
| run: | | |
| REPO="github/docs-internal" | |
| if EXISTING_PR=$(gh pr list --repo "$REPO" --head "$BRANCH" \ | |
| --json number --jq '.[0].number' 2>/dev/null) && [ -n "$EXISTING_PR" ]; then | |
| echo "Docs PR #$EXISTING_PR already exists, updated with new commit" | |
| exit 0 | |
| fi | |
| RUN_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" | |
| PR_BODY="The [sync-llms-txt workflow]($RUN_URL) generated this PR. | |
| Updates \`data/llms-txt/docs.md\`, served at https://docs.github.com/llms.txt. Built from the page catalog and popularity data using \`data/llms-txt/config-default.yml\` + \`config-docs.yml\`." | |
| gh pr create \ | |
| --repo "$REPO" \ | |
| --title "Update data/llms-txt/docs.md" \ | |
| --body "$PR_BODY" \ | |
| --head "$BRANCH" \ | |
| --base main \ | |
| --draft \ | |
| --label "llm-generated" | |
| # ---------- PR to github/github: update public/llms.txt ---------- | |
| - name: Fetch current public/llms.txt from github/github | |
| id: fetch_monolith | |
| env: | |
| GH_TOKEN: ${{ steps.app-token.outputs.token }} | |
| run: | | |
| if gh api repos/github/github/contents/public/llms.txt \ | |
| --jq '.content' 2>/dev/null | base64 -d > /tmp/monolith-current.txt; then | |
| echo "Fetched current ($(wc -l < /tmp/monolith-current.txt) lines)" | |
| else | |
| rm -f /tmp/monolith-current.txt | |
| fi | |
| - name: Diff monolith llms.txt | |
| id: diff_monolith | |
| run: | | |
| if [ -f /tmp/monolith-current.txt ] && diff -q /tmp/monolith-llms.txt /tmp/monolith-current.txt > /dev/null 2>&1; then | |
| echo "No monolith changes, skipping" | |
| echo "changed=false" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "Monolith changes detected" | |
| echo "changed=true" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Ensure sync branch exists in github/github | |
| if: steps.diff_monolith.outputs.changed == 'true' | |
| env: | |
| GH_TOKEN: ${{ steps.app-token.outputs.token }} | |
| run: | | |
| REPO="github/github" | |
| if gh api "repos/$REPO/git/ref/heads/$BRANCH" --jq '.object.sha' > /dev/null 2>&1; then | |
| echo "Branch $BRANCH exists" | |
| else | |
| DEFAULT_BRANCH=$(gh api "repos/$REPO" --jq '.default_branch') | |
| BASE_SHA=$(gh api "repos/$REPO/git/ref/heads/$DEFAULT_BRANCH" --jq '.object.sha') | |
| gh api "repos/$REPO/git/refs" \ | |
| --method POST \ | |
| -f ref="refs/heads/$BRANCH" \ | |
| -f sha="$BASE_SHA" | |
| echo "Created branch $BRANCH from $DEFAULT_BRANCH at $BASE_SHA" | |
| fi | |
| - name: Commit monolith llms.txt to github/github | |
| if: steps.diff_monolith.outputs.changed == 'true' | |
| env: | |
| GH_TOKEN: ${{ steps.app-token.outputs.token }} | |
| run: | | |
| REPO="github/github" | |
| CONTENT=$(base64 -w 0 /tmp/monolith-llms.txt) | |
| if EXISTING_SHA=$(gh api "repos/$REPO/contents/public/llms.txt?ref=$BRANCH" \ | |
| --jq '.sha' 2>/dev/null); then | |
| echo "Existing file SHA: $EXISTING_SHA" | |
| else | |
| EXISTING_SHA="" | |
| fi | |
| COMMIT_ARGS=(-f "message=Sync llms.txt from docs.github.com" | |
| -f "content=$CONTENT" | |
| -f "branch=$BRANCH") | |
| if [ -n "$EXISTING_SHA" ]; then | |
| COMMIT_ARGS+=(-f "sha=$EXISTING_SHA") | |
| fi | |
| gh api "repos/$REPO/contents/public/llms.txt" \ | |
| --method PUT \ | |
| "${COMMIT_ARGS[@]}" --jq '.commit.sha' | |
| - name: Create or update github/github PR | |
| if: steps.diff_monolith.outputs.changed == 'true' | |
| env: | |
| GH_TOKEN: ${{ steps.app-token.outputs.token }} | |
| run: | | |
| REPO="github/github" | |
| if EXISTING_PR=$(gh pr list --repo "$REPO" --head "$BRANCH" \ | |
| --json number --jq '.[0].number' 2>/dev/null) && [ -n "$EXISTING_PR" ]; then | |
| echo "Monolith PR #$EXISTING_PR already exists, updated with new commit" | |
| exit 0 | |
| fi | |
| RUN_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" | |
| DEFAULT_BRANCH=$(gh api "repos/$REPO" --jq '.default_branch') | |
| PR_BODY="The [sync-llms-txt workflow]($RUN_URL) generated this PR. | |
| Updates \`public/llms.txt\`, served at https://github.com/llms.txt. Built in docs-internal from the page catalog and popularity data using \`data/llms-txt/config-default.yml\` + \`config-monolith.yml\`. | |
| No feature flags. Static file in \`public/\`, no code changes. | |
| <!-- | |
| Labels for github/github PR template automation: | |
| (\`environment:production-dotcom\`) | |
| (\`risk:low\`) | |
| (\`validate:other\`) | |
| (\`mitigate:rollback\`) | |
| (\`backend/rails/api-only\`) | |
| pull_request_template_version=2 | |
| -->" | |
| gh pr create \ | |
| --repo "$REPO" \ | |
| --title "Sync llms.txt from docs.github.com" \ | |
| --body "$PR_BODY" \ | |
| --head "$BRANCH" \ | |
| --base "$DEFAULT_BRANCH" \ | |
| --label "docs" | |
| - uses: ./.github/actions/slack-alert | |
| if: ${{ failure() && github.event_name != 'workflow_dispatch' }} | |
| with: | |
| slack_token: ${{ secrets.SLACK_DOCS_BOT_TOKEN }} | |
| - uses: ./.github/actions/create-workflow-failure-issue | |
| if: ${{ failure() && github.event_name != 'workflow_dispatch' }} | |
| with: | |
| token: ${{ steps.app-token.outputs.token }} |