Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 33 additions & 0 deletions .github/workflows/comment_bot.yml
Original file line number Diff line number Diff line change
Expand Up @@ -56,3 +56,36 @@ jobs:
archery --debug trigger-bot \
--event-name "${GITHUB_EVENT_NAME}" \
--event-payload "${GITHUB_EVENT_PATH}"

docs-preview:
name: Docs Preview
# Like the Archery bot, only accept requests from the Apache org: the
# preview build runs the pull request's code on the default branch.
if: >-
github.event.action == 'created' &&
github.event.issue.pull_request &&
startsWith(github.event.comment.body, '@github-actions docs-preview') &&
contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'),
github.event.comment.author_association)
runs-on: ubuntu-latest
permissions:
actions: write
pull-requests: write
steps:
- name: Request Docs Preview
env:
GH_TOKEN: ${{ github.token }}
COMMENT_ID: ${{ github.event.comment.id }}
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
PR_NUMBER: ${{ github.event.issue.number }}
run: |
# Pin the commit that is the head now, see docs.yml
head_sha=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq '.head.sha')
gh workflow run docs.yml \
--repo "${GITHUB_REPOSITORY}" \
--ref "${DEFAULT_BRANCH}" \
-f pr_number="${PR_NUMBER}" \
-f head_sha="${head_sha}"
gh api --method POST \
"repos/${GITHUB_REPOSITORY}/issues/comments/${COMMENT_ID}/reactions" \
-f content='+1' > /dev/null
69 changes: 66 additions & 3 deletions .github/workflows/docs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,13 @@

name: Docs

# docs_preview.yml reads the pull request and commit of a requested preview
# from this name
run-name: >-
${{ github.event_name == 'workflow_dispatch' &&
format('Docs preview for #{0} at {1}', inputs.pr_number, inputs.head_sha) ||
'' }}

on:
push:
branches:
Expand All @@ -29,6 +36,20 @@ on:
- '.github/workflows/docs.yml'
- 'cpp/apidoc/**'
- 'docs/**'
# Builds a docs preview for any pull request, requested with the
# "@github-actions docs-preview" comment, see comment_bot.yml. These runs
# build pull request code on the default branch: they must not use secrets
# and must not save caches.
workflow_dispatch:
inputs:
pr_number:
description: Pull request to build a docs preview for
required: true
type: string
head_sha:
description: Head commit of the pull request
required: true
type: string

permissions:
contents: read
Expand All @@ -53,15 +74,36 @@ jobs:
with:
persist-credentials: false
fetch-depth: 0
ref: >-
${{ github.event_name == 'workflow_dispatch' &&
format('refs/pull/{0}/head', inputs.pr_number) || '' }}
- name: Check Requested Commit
if: github.event_name == 'workflow_dispatch'
env:
HEAD_SHA: ${{ inputs.head_sha }}
run: |
# Only build the commit the preview was requested for
if [ "$(git rev-parse HEAD)" != "${HEAD_SHA}" ]; then
echo "::error::Pull request head is no longer ${HEAD_SHA}"
exit 1
fi
- name: Free up disk space
run: |
ci/scripts/util_free_space.sh
- name: Cache Docker Volumes
if: github.event_name != 'workflow_dispatch'
uses: actions/cache@v6
with:
path: .docker
key: debian-docs-${{ hashFiles('cpp/**') }}
restore-keys: debian-docs-
- name: Restore Docker Volumes
if: github.event_name == 'workflow_dispatch'
uses: actions/cache/restore@v6
with:
path: .docker
key: debian-docs-${{ hashFiles('cpp/**') }}
restore-keys: debian-docs-
- name: Setup Python
uses: actions/setup-python@v7
with:
Expand All @@ -70,10 +112,31 @@ jobs:
run: pip install -e dev/archery[docker]
- name: Execute Docker Build
env:
ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
JDK: 17
run: archery docker run debian-docs
run: |
# Keep the rendered docs on the runner so they can be uploaded below
mkdir -p "${RUNNER_TEMP}/docs"
archery docker run -v "${RUNNER_TEMP}/docs:/build/docs" debian-docs
- name: Prepare Docs Preview
if: >-
github.event_name == 'pull_request' ||
github.event_name == 'workflow_dispatch'
run: |
# build files are created by the docker user
sudo chown -R "${USER}:" "${RUNNER_TEMP}/docs"
# These are only used while generating the page
rm -rf "${RUNNER_TEMP}/docs/.doctrees"
# Published by docs_preview.yml, which has the credentials this job lacks
- name: Upload Docs Preview
if: >-
github.event_name == 'pull_request' ||
github.event_name == 'workflow_dispatch'
uses: actions/upload-artifact@v7
with:
name: docs-preview
path: ${{ runner.temp }}/docs
retention-days: 3
if-no-files-found: error
- name: Docker Push
if: >-
success() &&
Expand Down
5 changes: 1 addition & 4 deletions .github/workflows/docs_light.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.

name: Docs
name: Docs Light

on:
pull_request:
Expand Down Expand Up @@ -67,7 +67,4 @@ jobs:
- name: Setup Archery
run: pip install -e dev/archery[docker]
- name: Execute Docker Build
env:
ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
run: archery docker run conda-python-docs
168 changes: 168 additions & 0 deletions .github/workflows/docs_preview.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,168 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

# Publishes the docs built for a pull request by the "Docs" workflow to
# nightlies.apache.org/arrow/pr_docs/<date>/<pull request>/.
#
# "Docs" builds pull request code, so it has no upload credentials and only
# stores the docs as an artifact. This workflow has the credentials: it must
# never check out or run pull request code, and the artifact is untrusted.

name: Docs Preview

on:
workflow_run:
workflows:
- Docs
types:
- completed

permissions:
contents: read

env:
PREVIEW_URL: https://nightlies.apache.org/arrow/pr_docs
# Previews are removed this many days after they were published
RETENTION_DAYS: 14
# Upper bound on what a pull request can publish, the docs are about 250 MB
MAX_SIZE_MB: 1024

jobs:

publish:
name: Publish
if: >-
github.repository == 'apache/arrow' &&
github.event.workflow_run.conclusion == 'success' &&
contains(fromJSON('["pull_request", "workflow_dispatch"]'),
github.event.workflow_run.event)
runs-on: ubuntu-latest
timeout-minutes: 30
permissions:
actions: read
contents: read
pull-requests: write
env:
GH_TOKEN: ${{ github.token }}
steps:
- name: Find Pull Request
id: pr
env:
EVENT: ${{ github.event.workflow_run.event }}
HEAD_OWNER: ${{ github.event.workflow_run.head_repository.owner.login }}
HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }}
HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
TITLE: ${{ github.event.workflow_run.display_title }}
run: |
if [ "${EVENT}" = "workflow_dispatch" ]; then
# Requested with a comment: docs.yml names the run after the pull
# request and commit, pull request code can't change that.
pattern='^Docs preview for #([0-9]+) at ([0-9a-f]{40})$'
[[ "${TITLE}" =~ ${pattern} ]] || exit 0
number=${BASH_REMATCH[1]}
sha=${BASH_REMATCH[2]}
else
# The event doesn't list fork pull requests: find the open pull
# request whose head is the branch and commit that were built.
sha=${HEAD_SHA}
number=$(
gh api --method GET "repos/${GITHUB_REPOSITORY}/pulls" \
-f head="${HEAD_OWNER}:${HEAD_BRANCH}" \
-f per_page=100 \
--jq "[.[] | select(.head.sha == \"${sha}\")][0].number // empty"
)
fi
echo "number=${number}" >> "${GITHUB_OUTPUT}"
echo "sha=${sha}" >> "${GITHUB_OUTPUT}"
- name: Download Docs Preview
id: preview
if: steps.pr.outputs.number
env:
PR_NUMBER: ${{ steps.pr.outputs.number }}
RUN_ID: ${{ github.event.workflow_run.id }}
run: |
date=$(date -u +%F)
path="${date}/${PR_NUMBER}"
gh run download "${RUN_ID}" \
--repo "${GITHUB_REPOSITORY}" \
--name docs-preview \
--dir "stage/${path}"
# Only publish plain files and directories, no dotfiles such as
# .htaccess, and only a plausible amount of them.
find stage \( ! -type f ! -type d -o -name '.*' \) -prune -exec rm -rf {} +
size_mb=$(du -sm stage | cut -f1)
if [ "${size_mb}" -gt "${MAX_SIZE_MB}" ] || [ ! -f "stage/${path}/index.html" ]; then
echo "::error::Docs preview is ${size_mb} MB (limit ${MAX_SIZE_MB}) or has no index.html"
exit 1
fi
# rsync filters, the first match wins: replace this preview, keep
# the other previews of today, of the last RETENTION_DAYS days and
# of tomorrow, in case another run publishes after midnight.
# Everything else on the remote is not matched and gets deleted,
# which is what removes old previews.
filters="--include=/${path}/*** --exclude=/${date}/*"
for days in +1 $(seq -1 -1 $((1 - RETENTION_DAYS))); do
filters+=" --exclude=/$(date -u -d "${date} ${days} days" +%F)/"
done
echo "path=${path}" >> "${GITHUB_OUTPUT}"
echo "filters=${filters}" >> "${GITHUB_OUTPUT}"
# The default branch, only for the sync-nightlies action
- name: Checkout Arrow
if: steps.preview.outputs.filters
uses: actions/checkout@v7
with:
persist-credentials: false
path: arrow
sparse-checkout: .github/actions
- name: Sync to Remote
# This deletes on the remote, never run it without the filters
if: steps.preview.outputs.filters
uses: ./arrow/.github/actions/sync-nightlies
with:
upload: true
# Symlinks are not transferred
switches: -rtzh --delete --stats ${{ steps.preview.outputs.filters }}
local_path: stage
remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH }}/arrow/pr_docs
remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }}
remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }}
remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }}
remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }}
remote_host_key: ${{ secrets.NIGHTLIES_RSYNC_HOST_KEY }}
- name: Comment on Pull Request
if: steps.preview.outputs.filters
env:
PR_NUMBER: ${{ steps.pr.outputs.number }}
PREVIEW_PATH: ${{ steps.preview.outputs.path }}
SHA: ${{ steps.pr.outputs.sha }}
run: |
marker="<!-- docs-preview -->"
body="${marker}
:open_book: Docs preview for ${SHA}: ${PREVIEW_URL}/${PREVIEW_PATH}/index.html
It is removed after ${RETENTION_DAYS} days."
# Keep a single comment per pull request up to date
comments="repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments"
comment_id=$(
gh api --paginate "${comments}" \
--jq ".[] | select(.user.login == \"github-actions[bot]\") | select(.body | startswith(\"${marker}\")) | .id" |
awk 'NR == 1'
)
if [ -n "${comment_id}" ]; then
gh api --method PATCH "repos/${GITHUB_REPOSITORY}/issues/comments/${comment_id}" -f body="${body}"
else
gh api --method POST "${comments}" -f body="${body}"
fi > /dev/null
21 changes: 0 additions & 21 deletions dev/tasks/docs/github.linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
# under the License.

{% import 'macros.jinja' as macros with context %}
{% set pr_number = pr_number|replace("Unset", arrow.branch) %}
{{ macros.github_header() }}

jobs:
Expand Down Expand Up @@ -48,26 +47,6 @@ jobs:
ref: {{ default_branch|default("main") }}
path: crossbow
fetch-depth: 1
{% if publish %}
- name: Prepare Docs Preview
run: |
# build files are created by the docker user
sudo chown -R ${USER}: build
# These are only used while generating the page
rm -rf build/docs/.doctrees
- name: Upload preview to S3
env:
{%- raw %}
AWS_ACCESS_KEY_ID: ${{ secrets.CROSSBOW_DOCS_ARROW_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.CROSSBOW_DOCS_ARROW_AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.CROSSBOW_DOCS_ARROW_S3_BUCKET_REGION }}
BUCKET: ${{ secrets.CROSSBOW_DOCS_ARROW_S3_BUCKET }}
{% endraw %}
run: |
aws s3 cp build/docs/ $BUCKET/pr_docs/{{ pr_number }}/ --recursive
echo ":open_book: You can find the preview here: https://s3.amazonaws.com/arrow-data/pr_docs/{{ pr_number }}/index.html" >> $GITHUB_STEP_SUMMARY

{% endif %}
- name: Prepare Docs artifacts
run: |
cd build
Expand Down
13 changes: 0 additions & 13 deletions dev/tasks/tasks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -689,9 +689,7 @@ tasks:
params:
env:
JDK: 17
pr_number: Unset
image: debian-docs
publish: false
artifacts:
- docs.tar.gz

Expand Down Expand Up @@ -797,14 +795,3 @@ tasks:
params:
type: tutorial_examples
run: tutorial

############################## Utility tasks ############################
preview-docs:
ci: github
template: docs/github.linux.yml
params:
env:
JDK: 17
pr_number: Unset
image: debian-docs
publish: true
Loading
Loading