diff --git a/.github/workflows/comment_bot.yml b/.github/workflows/comment_bot.yml index 044c761aa636..3b4f6aeb53b2 100644 --- a/.github/workflows/comment_bot.yml +++ b/.github/workflows/comment_bot.yml @@ -56,3 +56,36 @@ jobs: archery --debug trigger-bot \ --event-name "${GITHUB_EVENT_NAME}" \ --event-payload "${GITHUB_EVENT_PATH}" + + docs-preview: + name: Docs Preview + # Like the Archery bot, only accept requests from the Apache org: the + # preview build runs the pull request's code on the default branch. + if: >- + github.event.action == 'created' && + github.event.issue.pull_request && + startsWith(github.event.comment.body, '@github-actions docs-preview') && + contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), + github.event.comment.author_association) + runs-on: ubuntu-latest + permissions: + actions: write + pull-requests: write + steps: + - name: Request Docs Preview + env: + GH_TOKEN: ${{ github.token }} + COMMENT_ID: ${{ github.event.comment.id }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + PR_NUMBER: ${{ github.event.issue.number }} + run: | + # Pin the commit that is the head now, see docs.yml + head_sha=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq '.head.sha') + gh workflow run docs.yml \ + --repo "${GITHUB_REPOSITORY}" \ + --ref "${DEFAULT_BRANCH}" \ + -f pr_number="${PR_NUMBER}" \ + -f head_sha="${head_sha}" + gh api --method POST \ + "repos/${GITHUB_REPOSITORY}/issues/comments/${COMMENT_ID}/reactions" \ + -f content='+1' > /dev/null diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index d176095739fb..bb12e28957ca 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -17,6 +17,13 @@ name: Docs +# docs_preview.yml reads the pull request and commit of a requested preview +# from this name +run-name: >- + ${{ github.event_name == 'workflow_dispatch' && + format('Docs preview for #{0} at {1}', inputs.pr_number, inputs.head_sha) || + '' }} + on: push: branches: @@ -29,6 +36,20 @@ on: - '.github/workflows/docs.yml' - 'cpp/apidoc/**' - 'docs/**' + # Builds a docs preview for any pull request, requested with the + # "@github-actions docs-preview" comment, see comment_bot.yml. These runs + # build pull request code on the default branch: they must not use secrets + # and must not save caches. + workflow_dispatch: + inputs: + pr_number: + description: Pull request to build a docs preview for + required: true + type: string + head_sha: + description: Head commit of the pull request + required: true + type: string permissions: contents: read @@ -53,15 +74,36 @@ jobs: with: persist-credentials: false fetch-depth: 0 + ref: >- + ${{ github.event_name == 'workflow_dispatch' && + format('refs/pull/{0}/head', inputs.pr_number) || '' }} + - name: Check Requested Commit + if: github.event_name == 'workflow_dispatch' + env: + HEAD_SHA: ${{ inputs.head_sha }} + run: | + # Only build the commit the preview was requested for + if [ "$(git rev-parse HEAD)" != "${HEAD_SHA}" ]; then + echo "::error::Pull request head is no longer ${HEAD_SHA}" + exit 1 + fi - name: Free up disk space run: | ci/scripts/util_free_space.sh - name: Cache Docker Volumes + if: github.event_name != 'workflow_dispatch' uses: actions/cache@v6 with: path: .docker key: debian-docs-${{ hashFiles('cpp/**') }} restore-keys: debian-docs- + - name: Restore Docker Volumes + if: github.event_name == 'workflow_dispatch' + uses: actions/cache/restore@v6 + with: + path: .docker + key: debian-docs-${{ hashFiles('cpp/**') }} + restore-keys: debian-docs- - name: Setup Python uses: actions/setup-python@v7 with: @@ -70,10 +112,31 @@ jobs: run: pip install -e dev/archery[docker] - name: Execute Docker Build env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} JDK: 17 - run: archery docker run debian-docs + run: | + # Keep the rendered docs on the runner so they can be uploaded below + mkdir -p "${RUNNER_TEMP}/docs" + archery docker run -v "${RUNNER_TEMP}/docs:/build/docs" debian-docs + - name: Prepare Docs Preview + if: >- + github.event_name == 'pull_request' || + github.event_name == 'workflow_dispatch' + run: | + # build files are created by the docker user + sudo chown -R "${USER}:" "${RUNNER_TEMP}/docs" + # These are only used while generating the page + rm -rf "${RUNNER_TEMP}/docs/.doctrees" + # Published by docs_preview.yml, which has the credentials this job lacks + - name: Upload Docs Preview + if: >- + github.event_name == 'pull_request' || + github.event_name == 'workflow_dispatch' + uses: actions/upload-artifact@v7 + with: + name: docs-preview + path: ${{ runner.temp }}/docs + retention-days: 3 + if-no-files-found: error - name: Docker Push if: >- success() && diff --git a/.github/workflows/docs_light.yml b/.github/workflows/docs_light.yml index 9e62aacfa741..796f8b0cc154 100644 --- a/.github/workflows/docs_light.yml +++ b/.github/workflows/docs_light.yml @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -name: Docs +name: Docs Light on: pull_request: @@ -67,7 +67,4 @@ jobs: - name: Setup Archery run: pip install -e dev/archery[docker] - name: Execute Docker Build - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} run: archery docker run conda-python-docs diff --git a/.github/workflows/docs_preview.yml b/.github/workflows/docs_preview.yml new file mode 100644 index 000000000000..d2d8ef1d38a0 --- /dev/null +++ b/.github/workflows/docs_preview.yml @@ -0,0 +1,168 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Publishes the docs built for a pull request by the "Docs" workflow to +# nightlies.apache.org/arrow/pr_docs///. +# +# "Docs" builds pull request code, so it has no upload credentials and only +# stores the docs as an artifact. This workflow has the credentials: it must +# never check out or run pull request code, and the artifact is untrusted. + +name: Docs Preview + +on: + workflow_run: + workflows: + - Docs + types: + - completed + +permissions: + contents: read + +env: + PREVIEW_URL: https://nightlies.apache.org/arrow/pr_docs + # Previews are removed this many days after they were published + RETENTION_DAYS: 14 + # Upper bound on what a pull request can publish, the docs are about 250 MB + MAX_SIZE_MB: 1024 + +jobs: + + publish: + name: Publish + if: >- + github.repository == 'apache/arrow' && + github.event.workflow_run.conclusion == 'success' && + contains(fromJSON('["pull_request", "workflow_dispatch"]'), + github.event.workflow_run.event) + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + actions: read + contents: read + pull-requests: write + env: + GH_TOKEN: ${{ github.token }} + steps: + - name: Find Pull Request + id: pr + env: + EVENT: ${{ github.event.workflow_run.event }} + HEAD_OWNER: ${{ github.event.workflow_run.head_repository.owner.login }} + HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} + HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + TITLE: ${{ github.event.workflow_run.display_title }} + run: | + if [ "${EVENT}" = "workflow_dispatch" ]; then + # Requested with a comment: docs.yml names the run after the pull + # request and commit, pull request code can't change that. + pattern='^Docs preview for #([0-9]+) at ([0-9a-f]{40})$' + [[ "${TITLE}" =~ ${pattern} ]] || exit 0 + number=${BASH_REMATCH[1]} + sha=${BASH_REMATCH[2]} + else + # The event doesn't list fork pull requests: find the open pull + # request whose head is the branch and commit that were built. + sha=${HEAD_SHA} + number=$( + gh api --method GET "repos/${GITHUB_REPOSITORY}/pulls" \ + -f head="${HEAD_OWNER}:${HEAD_BRANCH}" \ + -f per_page=100 \ + --jq "[.[] | select(.head.sha == \"${sha}\")][0].number // empty" + ) + fi + echo "number=${number}" >> "${GITHUB_OUTPUT}" + echo "sha=${sha}" >> "${GITHUB_OUTPUT}" + - name: Download Docs Preview + id: preview + if: steps.pr.outputs.number + env: + PR_NUMBER: ${{ steps.pr.outputs.number }} + RUN_ID: ${{ github.event.workflow_run.id }} + run: | + date=$(date -u +%F) + path="${date}/${PR_NUMBER}" + gh run download "${RUN_ID}" \ + --repo "${GITHUB_REPOSITORY}" \ + --name docs-preview \ + --dir "stage/${path}" + # Only publish plain files and directories, no dotfiles such as + # .htaccess, and only a plausible amount of them. + find stage \( ! -type f ! -type d -o -name '.*' \) -prune -exec rm -rf {} + + size_mb=$(du -sm stage | cut -f1) + if [ "${size_mb}" -gt "${MAX_SIZE_MB}" ] || [ ! -f "stage/${path}/index.html" ]; then + echo "::error::Docs preview is ${size_mb} MB (limit ${MAX_SIZE_MB}) or has no index.html" + exit 1 + fi + # rsync filters, the first match wins: replace this preview, keep + # the other previews of today, of the last RETENTION_DAYS days and + # of tomorrow, in case another run publishes after midnight. + # Everything else on the remote is not matched and gets deleted, + # which is what removes old previews. + filters="--include=/${path}/*** --exclude=/${date}/*" + for days in +1 $(seq -1 -1 $((1 - RETENTION_DAYS))); do + filters+=" --exclude=/$(date -u -d "${date} ${days} days" +%F)/" + done + echo "path=${path}" >> "${GITHUB_OUTPUT}" + echo "filters=${filters}" >> "${GITHUB_OUTPUT}" + # The default branch, only for the sync-nightlies action + - name: Checkout Arrow + if: steps.preview.outputs.filters + uses: actions/checkout@v7 + with: + persist-credentials: false + path: arrow + sparse-checkout: .github/actions + - name: Sync to Remote + # This deletes on the remote, never run it without the filters + if: steps.preview.outputs.filters + uses: ./arrow/.github/actions/sync-nightlies + with: + upload: true + # Symlinks are not transferred + switches: -rtzh --delete --stats ${{ steps.preview.outputs.filters }} + local_path: stage + remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH }}/arrow/pr_docs + remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }} + remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }} + remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }} + remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }} + remote_host_key: ${{ secrets.NIGHTLIES_RSYNC_HOST_KEY }} + - name: Comment on Pull Request + if: steps.preview.outputs.filters + env: + PR_NUMBER: ${{ steps.pr.outputs.number }} + PREVIEW_PATH: ${{ steps.preview.outputs.path }} + SHA: ${{ steps.pr.outputs.sha }} + run: | + marker="" + body="${marker} + :open_book: Docs preview for ${SHA}: ${PREVIEW_URL}/${PREVIEW_PATH}/index.html + It is removed after ${RETENTION_DAYS} days." + # Keep a single comment per pull request up to date + comments="repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" + comment_id=$( + gh api --paginate "${comments}" \ + --jq ".[] | select(.user.login == \"github-actions[bot]\") | select(.body | startswith(\"${marker}\")) | .id" | + awk 'NR == 1' + ) + if [ -n "${comment_id}" ]; then + gh api --method PATCH "repos/${GITHUB_REPOSITORY}/issues/comments/${comment_id}" -f body="${body}" + else + gh api --method POST "${comments}" -f body="${body}" + fi > /dev/null diff --git a/dev/tasks/docs/github.linux.yml b/dev/tasks/docs/github.linux.yml index 9c1667ebf782..4c4a214bdcc8 100644 --- a/dev/tasks/docs/github.linux.yml +++ b/dev/tasks/docs/github.linux.yml @@ -16,7 +16,6 @@ # under the License. {% import 'macros.jinja' as macros with context %} -{% set pr_number = pr_number|replace("Unset", arrow.branch) %} {{ macros.github_header() }} jobs: @@ -48,26 +47,6 @@ jobs: ref: {{ default_branch|default("main") }} path: crossbow fetch-depth: 1 - {% if publish %} - - name: Prepare Docs Preview - run: | - # build files are created by the docker user - sudo chown -R ${USER}: build - # These are only used while generating the page - rm -rf build/docs/.doctrees - - name: Upload preview to S3 - env: - {%- raw %} - AWS_ACCESS_KEY_ID: ${{ secrets.CROSSBOW_DOCS_ARROW_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.CROSSBOW_DOCS_ARROW_AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.CROSSBOW_DOCS_ARROW_S3_BUCKET_REGION }} - BUCKET: ${{ secrets.CROSSBOW_DOCS_ARROW_S3_BUCKET }} - {% endraw %} - run: | - aws s3 cp build/docs/ $BUCKET/pr_docs/{{ pr_number }}/ --recursive - echo ":open_book: You can find the preview here: https://s3.amazonaws.com/arrow-data/pr_docs/{{ pr_number }}/index.html" >> $GITHUB_STEP_SUMMARY - - {% endif %} - name: Prepare Docs artifacts run: | cd build diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index c3ddbc91f9a0..5e259055dfd2 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -689,9 +689,7 @@ tasks: params: env: JDK: 17 - pr_number: Unset image: debian-docs - publish: false artifacts: - docs.tar.gz @@ -797,14 +795,3 @@ tasks: params: type: tutorial_examples run: tutorial - -############################## Utility tasks ############################ - preview-docs: - ci: github - template: docs/github.linux.yml - params: - env: - JDK: 17 - pr_number: Unset - image: debian-docs - publish: true diff --git a/docs/source/developers/documentation.rst b/docs/source/developers/documentation.rst index 3cdb2f23b00d..a898c7c0fa64 100644 --- a/docs/source/developers/documentation.rst +++ b/docs/source/developers/documentation.rst @@ -133,26 +133,21 @@ The final output is located under the ``${PWD}/docs`` directory. Building a docs preview in a Pull Request ----------------------------------------- -You can build and preview the documentation within a GitHub pull request you are working on. - -To do so, post the comment ``@github-actions crossbow submit preview-docs`` -to the pull request. The rendered documentation will then be available within the -GitHub Actions response, where you need to click on the Crossbow build badge: - -.. figure:: ./images/docs_preview_1.jpeg - :scale: 70 % - :alt: GitHub Actions response with the crossbow build status. - - Crossbow build status - -and then in the summary of the workflow you can find the link to the Docs Preview -summary at the bottom of the page: - -.. figure:: ./images/docs_preview_2.jpeg - :scale: 70 % - :alt: Crossbow workflow page with the Docs Preview summary section. - - Docs Preview summary section +You can preview the documentation built from a GitHub pull request you are +working on. + +Pull requests that change files under ``docs/`` or ``cpp/apidoc/`` build the +complete documentation automatically. For any other pull request, for example +one that only changes docstrings, a committer can request the same build by +posting the comment ``@github-actions docs-preview`` to the pull request. The +preview is built from the commit that is the head of the pull request when +the comment is posted, comment again to rebuild after new commits have been +pushed. The automatic build is skipped while the pull request title contains +``WIP``. + +Once the build has finished, a comment with a link to the rendered +documentation is posted to the pull request. The comment is updated on each +new build and a preview is removed about two weeks after it was published. Building for dev purposes ------------------------- diff --git a/docs/source/developers/images/docs_preview_1.jpeg b/docs/source/developers/images/docs_preview_1.jpeg deleted file mode 100644 index 6111f34b1386..000000000000 Binary files a/docs/source/developers/images/docs_preview_1.jpeg and /dev/null differ diff --git a/docs/source/developers/images/docs_preview_2.jpeg b/docs/source/developers/images/docs_preview_2.jpeg deleted file mode 100644 index 3bf4f381855b..000000000000 Binary files a/docs/source/developers/images/docs_preview_2.jpeg and /dev/null differ