fetchkit/.github/workflows/ci.yml at main · everruns/fetchkit · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
name: CI

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]
  workflow_call:

# Cancel in-progress runs for same PR/branch
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

permissions:
  contents: read

env:
  CARGO_TERM_COLOR: always
  RUST_BACKTRACE: 1

jobs:
  lint:
    name: Lint
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6
      - uses: dtolnay/rust-toolchain@stable
        with:
          components: rustfmt, clippy
      - uses: Swatinem/rust-cache@v2
      - name: Check formatting
        run: cargo fmt --all -- --check
      - name: Clippy
        run: cargo clippy --workspace --all-targets -- -D warnings

  test:
    name: Test
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6
      - uses: dtolnay/rust-toolchain@stable
      - uses: Swatinem/rust-cache@v2
      - name: Test
        run: cargo test --workspace
      - name: Test (network-dependent)
        run: cargo test --workspace -- --ignored

  build:
    name: Build
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6
      - uses: dtolnay/rust-toolchain@stable
      - uses: Swatinem/rust-cache@v2
      - name: Build release
        run: cargo build --workspace --exclude fetchkit-python --release
      - name: Upload CLI artifact
        uses: actions/upload-artifact@v4
        with:
          name: fetchkit-cli-linux
          path: target/release/fetchkit

  doc:
    name: Documentation
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6
      - uses: dtolnay/rust-toolchain@stable
      - uses: Swatinem/rust-cache@v2
      - name: Check documentation
        run: cargo doc --workspace --no-deps
        env:
          RUSTDOCFLAGS: -D warnings

  examples:
    name: Examples
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6
      - uses: dtolnay/rust-toolchain@stable
      - uses: Swatinem/rust-cache@v2
      - name: Run fetch_urls example
        run: cargo run -p fetchkit --example fetch_urls
        timeout-minutes: 2
      - name: Run save_to_file example
        run: cargo run -p fetchkit --example save_to_file
        timeout-minutes: 2

  python:
    name: Python Example
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6
      - uses: dtolnay/rust-toolchain@stable
      - uses: Swatinem/rust-cache@v2
      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - name: Set up virtualenv and install maturin
        run: |
          python -m venv .venv
          . .venv/bin/activate
          pip install maturin
      - name: Build Python bindings
        run: |
          . .venv/bin/activate
          maturin develop -m crates/fetchkit-python/Cargo.toml
      - name: Run Python example
        run: |
          . .venv/bin/activate
          python examples/python_fetchkit.py
        timeout-minutes: 2

  live-tests:
    name: Live Fetcher Tests
    if: github.event_name == 'pull_request'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0
      - uses: dtolnay/rust-toolchain@stable
      - uses: Swatinem/rust-cache@v2
      - name: Detect changed fetchers
        id: detect
        run: |
          FILTERS=$(bash scripts/changed-fetcher-tests.sh "origin/${{ github.base_ref }}")
          echo "filters=$FILTERS" >> "$GITHUB_OUTPUT"
          if [ -z "$FILTERS" ]; then
            echo "skip=true" >> "$GITHUB_OUTPUT"
          else
            echo "skip=false" >> "$GITHUB_OUTPUT"
          fi
      - name: Run live tests
        if: steps.detect.outputs.skip == 'false'
        run: |
          for filter in ${{ steps.detect.outputs.filters }}; do
            echo "::group::Running $filter"
            cargo test --features live-tests -p fetchkit "$filter" -- --nocapture
            echo "::endgroup::"
          done
        timeout-minutes: 5
        continue-on-error: true

  # Gate job for branch protection.
  check:
    name: Check
    if: always()
    needs: [lint, test, build, doc, examples, python]
    runs-on: ubuntu-latest
    steps:
      - name: Verify all jobs passed
        run: |
          if [[ "${{ needs.lint.result }}" != "success" ]] || \
             [[ "${{ needs.test.result }}" != "success" ]] || \
             [[ "${{ needs.build.result }}" != "success" ]] || \
             [[ "${{ needs.doc.result }}" != "success" ]] || \
             [[ "${{ needs.examples.result }}" != "success" ]] || \
             [[ "${{ needs.python.result }}" != "success" ]]; then
            echo "One or more required jobs failed"
            exit 1
          fi