From 5a94fdb9f871c79c9e6b7e45d2d449a67b5c2d85 Mon Sep 17 00:00:00 2001 From: irfanuddinahmad Date: Mon, 24 Aug 2026 20:07:07 +0500 Subject: [PATCH] feat: pip to uv migration --- .coveragerc | 53 - .dockerignore | 3 - .github/CODEOWNERS | 3 - .../check-consistent-dependencies.yml | 12 +- .../workflows/check_python_dependencies.yml | 19 +- .github/workflows/ci-static-analysis.yml | 21 +- .../workflows/compile-python-requirements.yml | 13 +- .github/workflows/js-tests.yml | 21 +- .github/workflows/lint-imports.yml | 22 +- .github/workflows/migrations-check.yml | 41 +- .github/workflows/pylint-checks.yml | 26 +- .github/workflows/quality-checks.yml | 33 +- .github/workflows/semgrep.yml | 10 +- .github/workflows/static-assets-check.yml | 40 +- .github/workflows/unit-tests.yml | 71 +- .../units-test-scripts-structures-pruning.yml | 13 +- .../units-test-scripts-user-retirement.yml | 13 +- .../upgrade-one-python-dependency.yml | 14 +- .gitignore | 3 - .readthedocs.yaml | 17 +- Makefile | 202 +- README.rst | 6 +- docs/concepts/testing/testing.rst | 3 +- docs/references/static-assets.rst | 4 +- package.json | 4 +- pyproject.toml | 462 +- requirements/README.rst | 59 +- requirements/common_constraints.txt | 30 - requirements/constraints.txt | 139 - requirements/edx-sandbox/README.rst | 29 +- requirements/edx-sandbox/base.in | 14 - requirements/edx-sandbox/base.txt | 59 +- requirements/edx-sandbox/pyproject.toml | 60 + requirements/edx-sandbox/uv.lock | 954 ++ requirements/edx/assets.in | 11 - requirements/edx/assets.txt | 19 +- requirements/edx/base.in | 2 - requirements/edx/base.txt | 378 +- requirements/edx/bundled.in | 46 - requirements/edx/coverage.in | 16 - requirements/edx/coverage.txt | 20 - requirements/edx/development.in | 26 - requirements/edx/development.txt | 1431 +-- requirements/edx/doc.in | 13 - requirements/edx/doc.txt | 1673 ---- requirements/edx/github.in | 89 - requirements/edx/kernel.in | 161 - requirements/edx/openstack.txt | 6 - requirements/edx/private.readme | 15 - requirements/edx/semgrep.in | 13 - requirements/edx/semgrep.txt | 213 - requirements/edx/testing.in | 49 - requirements/edx/testing.txt | 1753 ---- requirements/pip-tools.in | 12 - requirements/pip-tools.txt | 30 - scripts/compile_sass.py | 7 +- scripts/semgrep/README.rst | 16 + scripts/semgrep/pyproject.toml | 31 + scripts/semgrep/uv.lock | 1257 +++ scripts/structures_pruning/README.md | 22 +- scripts/structures_pruning/pyproject.toml | 41 + .../structures_pruning/requirements/base.in | 6 - .../structures_pruning/requirements/base.txt | 26 +- .../requirements/testing.in | 4 - .../requirements/testing.txt | 41 +- scripts/structures_pruning/uv.lock | 188 + scripts/update_uv_constraint_version.py | 43 + scripts/user_retirement/README.rst | 34 +- scripts/user_retirement/pyproject.toml | 52 + scripts/user_retirement/requirements/base.in | 13 - scripts/user_retirement/requirements/base.txt | 81 +- .../user_retirement/requirements/testing.in | 9 - .../user_retirement/requirements/testing.txt | 231 +- scripts/user_retirement/uv.lock | 1204 +++ scripts/watch_sass.sh | 2 +- scripts/xblock/pyproject.toml | 24 + scripts/xblock/requirements.in | 3 - scripts/xblock/requirements.txt | 17 +- scripts/xblock/uv.lock | 138 + scripts/xsslint_config.py | 1 + tox.ini | 14 +- uv.lock | 7975 +++++++++++++++++ 82 files changed, 13508 insertions(+), 6421 deletions(-) delete mode 100644 .coveragerc delete mode 100644 requirements/common_constraints.txt delete mode 100644 requirements/constraints.txt delete mode 100644 requirements/edx-sandbox/base.in create mode 100644 requirements/edx-sandbox/pyproject.toml create mode 100644 requirements/edx-sandbox/uv.lock delete mode 100644 requirements/edx/assets.in delete mode 100644 requirements/edx/base.in delete mode 100644 requirements/edx/bundled.in delete mode 100644 requirements/edx/coverage.in delete mode 100644 requirements/edx/coverage.txt delete mode 100644 requirements/edx/development.in delete mode 100644 requirements/edx/doc.in delete mode 100644 requirements/edx/doc.txt delete mode 100644 requirements/edx/github.in delete mode 100644 requirements/edx/kernel.in delete mode 100644 requirements/edx/openstack.txt delete mode 100644 requirements/edx/private.readme delete mode 100644 requirements/edx/semgrep.in delete mode 100644 requirements/edx/semgrep.txt delete mode 100644 requirements/edx/testing.in delete mode 100644 requirements/edx/testing.txt delete mode 100644 requirements/pip-tools.in delete mode 100644 requirements/pip-tools.txt create mode 100644 scripts/semgrep/README.rst create mode 100644 scripts/semgrep/pyproject.toml create mode 100644 scripts/semgrep/uv.lock create mode 100644 scripts/structures_pruning/pyproject.toml delete mode 100644 scripts/structures_pruning/requirements/base.in delete mode 100644 scripts/structures_pruning/requirements/testing.in create mode 100644 scripts/structures_pruning/uv.lock create mode 100644 scripts/update_uv_constraint_version.py create mode 100644 scripts/user_retirement/pyproject.toml delete mode 100644 scripts/user_retirement/requirements/base.in delete mode 100644 scripts/user_retirement/requirements/testing.in create mode 100644 scripts/user_retirement/uv.lock create mode 100644 scripts/xblock/pyproject.toml delete mode 100644 scripts/xblock/requirements.in create mode 100644 scripts/xblock/uv.lock create mode 100644 uv.lock diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 7d8fe1c26987..000000000000 --- a/.coveragerc +++ /dev/null @@ -1,53 +0,0 @@ -# .coveragerc for edx-platform -[run] -branch=true -data_file = reports/.coverage -source = - cms - common/djangoapps - lms - openedx - pavelib - scripts - xmodule - -omit = - cms/envs/* - cms/manage.py - cms/djangoapps/contentstore/views/dev.py - cms/djangoapps/*/migrations/* - cms/djangoapps/*/features/* - cms/lib/*/migrations/* - lms/debug/* - lms/envs/* - lms/djangoapps/*/migrations/* - lms/djangoapps/*/features/* - common/djangoapps/*/migrations/* - openedx/core/djangoapps/*/migrations/* - openedx/core/djangoapps/debug/* - openedx/envs/* - openedx/features/*/migrations/* - -concurrency=multiprocessing -parallel = true -relative_files = true - -[report] -ignore_errors = True - -exclude_lines = - pragma: no cover - raise NotImplementedError - -[html] -title = edx-platform Python Test Coverage Report -directory = reports/cover - -[xml] -output = reports/coverage.xml - -[paths] -jenkins_source = - /home/jenkins/workspace/$JOB_NAME - /home/jenkins/workspace/$SUBSET_JOB - /home/jenkins/edx-platform diff --git a/.dockerignore b/.dockerignore index 160c48565fce..68c84538fa68 100644 --- a/.dockerignore +++ b/.dockerignore @@ -11,9 +11,6 @@ # Files that should be git-ignored, but are hand-edited or otherwise valued, # and so should not be destroyed by "make clean". # start-noclean -requirements/private.txt -requirements/edx/private.in -requirements/edx/private.txt lms/envs/private.py cms/envs/private.py # end-noclean diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 5e217a7c09e7..e25ed5024d00 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -63,9 +63,6 @@ lms/envs/minimal.yml @feanil lms/envs/production.py @feanil @kdmccormick cms/envs/production.py @feanil @kdmccormick -# Ensure that this file is only used when strictly necessary -requirements/edx/github.in @feanil @kdmccormick - # Review GitHub Actions workflow changes (incl. Dependabot PRs). # Replaces the deprecated `reviewers:` key in dependabot.yml. /.github/workflows/ @openedx/wg-maintenance-openedx-platform-oncall diff --git a/.github/workflows/check-consistent-dependencies.yml b/.github/workflows/check-consistent-dependencies.yml index 89210e854f07..d24fcfbf2519 100644 --- a/.github/workflows/check-consistent-dependencies.yml +++ b/.github/workflows/check-consistent-dependencies.yml @@ -39,13 +39,21 @@ jobs: git fetch origin "$BASE_SHA" # The ^"? is because git may quote weird file paths - if git diff --name-only "$BASE_SHA" | grep -P '^"?((requirements/)|(scripts/.*?/requirements/))'; then + # + # This still watches requirements/ because requirements/edx-sandbox + # (its own uv project) and the requirements/edx/*.txt compatibility + # exports still live there. A planned follow-up will remove the + # requirements/ folder entirely (see public-engineering#543), at + # which point this pattern will need updating again. + if git diff --name-only "$BASE_SHA" | grep -P '^"?((requirements/)|(scripts/.*?/requirements/)|(scripts/[^/]+/pyproject\.toml)|(scripts/[^/]+/uv\.lock)|(pyproject\.toml)|(uv\.lock))'; then echo "RELEVANT=true" >> "$GITHUB_ENV" fi - - uses: actions/setup-python@v6 + - name: Install uv if: ${{ env.RELEVANT == 'true' }} + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 with: + enable-cache: true python-version: '3.12' - name: "Recompile requirements" diff --git a/.github/workflows/check_python_dependencies.yml b/.github/workflows/check_python_dependencies.yml index 664d50f0e545..a6930d79d7da 100644 --- a/.github/workflows/check_python_dependencies.yml +++ b/.github/workflows/check_python_dependencies.yml @@ -10,27 +10,22 @@ jobs: strategy: matrix: - python-version: ["3.12"] + python-version: + - "3.12" steps: - name: Checkout Repository uses: actions/checkout@v7 - - name: Set up Python - uses: actions/setup-python@v6 + - name: Install uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 with: + enable-cache: true python-version: ${{ matrix.python-version }} - - name: Install repo-tools - run: pip install edx-repo-tools[find_dependencies] - - - name: Install setuptool - run: pip install setuptools - - name: Run Python script run: | - find_python_dependencies \ - --req-file requirements/edx/base.txt \ - --req-file requirements/edx/testing.txt \ + uvx --from 'edx-repo-tools[find_dependencies]' find_python_dependencies \ + --req-file uv.lock \ --ignore https://github.com/mitodl/edx-sga \ --ignore https://github.com/open-craft/xblock-poll diff --git a/.github/workflows/ci-static-analysis.yml b/.github/workflows/ci-static-analysis.yml index db004ee9b495..5e310ac223a6 100644 --- a/.github/workflows/ci-static-analysis.yml +++ b/.github/workflows/ci-static-analysis.yml @@ -16,28 +16,15 @@ jobs: steps: - uses: actions/checkout@v7 - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: ${{ matrix.python-version }} - name: Install system requirements run: sudo apt update && sudo apt install -y libxmlsec1-dev - - name: Install pip - run: make pre-requirements - - - name: Get pip cache dir - id: pip-cache-dir - run: echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT - - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v6 + - name: Install uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }} - restore-keys: ${{ runner.os }}-pip- + enable-cache: true + python-version: ${{ matrix.python-version }} - name: Install python dependencies run: make dev-requirements diff --git a/.github/workflows/compile-python-requirements.yml b/.github/workflows/compile-python-requirements.yml index c9ce6a808bed..a3baf983b701 100644 --- a/.github/workflows/compile-python-requirements.yml +++ b/.github/workflows/compile-python-requirements.yml @@ -23,9 +23,10 @@ jobs: with: ref: "${{ inputs.branch }}" - - name: Set up Python environment - uses: actions/setup-python@v6 + - name: Install uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 with: + enable-cache: true python-version: "3.12" - name: Run make compile-requirements @@ -48,7 +49,13 @@ jobs: with: branch: "${{ github.triggering_actor }}/compile-python-deps" branch-suffix: short-commit-hash - add-paths: requirements + add-paths: | + requirements + scripts/**/pyproject.toml + scripts/**/uv.lock + scripts/**/requirements* + pyproject.toml + uv.lock commit-message: | feat: Recompile Python dependencies diff --git a/.github/workflows/js-tests.yml b/.github/workflows/js-tests.yml index 123934055bf9..f5bdac8969ef 100644 --- a/.github/workflows/js-tests.yml +++ b/.github/workflows/js-tests.yml @@ -43,27 +43,14 @@ jobs: - name: Install Required System Packages run: sudo apt-get update && sudo apt-get install libxmlsec1-dev ubuntu-restricted-extras xvfb - - name: Setup Python - uses: actions/setup-python@v6 + - name: Install uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 with: + enable-cache: true python-version: ${{ matrix.python-version }} - - name: Get pip cache dir - id: pip-cache-dir - run: | - echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT - - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v6 - with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/base.txt') }} - restore-keys: ${{ runner.os }}-pip- - - name: Install Required Python Dependencies - run: | - make base-requirements + run: make base-requirements - name: Install npm run: npm ci diff --git a/.github/workflows/lint-imports.yml b/.github/workflows/lint-imports.yml index eba3bf2bb0cc..7df07865cc7c 100644 --- a/.github/workflows/lint-imports.yml +++ b/.github/workflows/lint-imports.yml @@ -16,28 +16,14 @@ jobs: - name: Check out branch uses: actions/checkout@v7 - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: "3.12" - - name: Install system requirements run: sudo apt update && sudo apt install -y libxmlsec1-dev - - name: Install pip - run: make pre-requirements - - - name: Get pip cache dir - id: pip-cache-dir - run: echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT - - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v6 + - name: Install uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }} - restore-keys: ${{ runner.os }}-pip- + enable-cache: true + python-version: "3.12" - name: Install python dependencies run: make dev-requirements diff --git a/.github/workflows/migrations-check.yml b/.github/workflows/migrations-check.yml index 0993fbdd0a4f..f2a33010e34a 100644 --- a/.github/workflows/migrations-check.yml +++ b/.github/workflows/migrations-check.yml @@ -17,9 +17,10 @@ jobs: os: [ubuntu-24.04] python-version: - "3.12" - # 'pinned' is used to install the latest patch version of Django - # within the global constraint i.e. Django==4.2.8 in current case - # because we have global constraint of Django<4.2 + # 'pinned' installs whatever Django version `testing` resolves to under the + # global `Django<6.0` constraint (currently 5.2.x). To test another supported + # version instead, use a value matching a real `djangoNN` dependency-group in + # pyproject.toml (e.g. "42" for the `django42` group), synced below. django-version: ["pinned"] mongo-version: - "7" @@ -73,40 +74,28 @@ jobs: - name: Checkout repo uses: actions/checkout@v7 - - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 - with: - python-version: ${{ matrix.python-version }} - - name: Install system Packages run: | sudo apt-get update make ubuntu-requirements - - name: Get pip cache dir - id: pip-cache-dir - run: | - echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT - - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v6 + - name: Install uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }} - restore-keys: ${{ runner.os }}-pip- + enable-cache: true + python-version: ${{ matrix.python-version }} - name: Install Python dependencies run: | - make dev-requirements - if [[ "${{ matrix.django-version }}" != "pinned" ]]; then - pip install "django~=${{ matrix.django-version }}.0" - pip check # fail if this test-reqs/Django combination is broken + if [[ "${{ matrix.django-version }}" == "pinned" ]]; then + make dev-requirements + else + uv sync --no-default-groups --group testing --group "django${{ matrix.django-version }}" --frozen fi - name: list installed package versions run: | - sudo pip freeze + uv tree - name: Run Tests env: @@ -115,9 +104,9 @@ jobs: STUDIO_CFG: lms/envs/minimal.yml run: | echo "Running the LMS migrations." - ./manage.py lms migrate + uv run ./manage.py lms migrate echo "Running the CMS migrations." - ./manage.py cms migrate + uv run ./manage.py cms migrate # This job aggregates test results. It's the required check for branch protection. # https://github.com/marketplace/actions/alls-green#why diff --git a/.github/workflows/pylint-checks.yml b/.github/workflows/pylint-checks.yml index 6adf688c17f7..3adc5e969569 100644 --- a/.github/workflows/pylint-checks.yml +++ b/.github/workflows/pylint-checks.yml @@ -37,36 +37,22 @@ jobs: - name: Install required system packages run: sudo apt-get update && sudo apt-get install libxmlsec1-dev - - name: Set up Python - uses: actions/setup-python@v6 + - name: Install uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 with: - python-version: 3.12 - - - name: Get pip cache dir - id: pip-cache-dir - run: | - echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT - - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v6 - with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }} - restore-keys: ${{ runner.os }}-pip- + enable-cache: true + python-version: "3.12" - name: Install required Python dependencies run: | # dev-requirements is needed because the linter will otherwise # trip over some dev-only things like django-debug-toolbar - # (import debug_toolbar) that aren't in testing.txt. + # (import debug_toolbar) that aren't in the testing group. make dev-requirements - # After all requirements are installed, check that they're consistent with each other - pip check - name: Run quality tests run: | - pylint ${{ matrix.path }} + uv run pylint ${{ matrix.path }} # This job aggregates test results. It's the required check for branch protection. # https://github.com/marketplace/actions/alls-green#why diff --git a/.github/workflows/quality-checks.yml b/.github/workflows/quality-checks.yml index a412e6015bea..111b10e18f9d 100644 --- a/.github/workflows/quality-checks.yml +++ b/.github/workflows/quality-checks.yml @@ -30,11 +30,6 @@ jobs: - name: Install Required System Packages run: sudo apt-get update && sudo apt-get install libxmlsec1-dev - - name: Setup Python - uses: actions/setup-python@v6 - with: - python-version: ${{ matrix.python-version }} - - name: Setup Node uses: actions/setup-node@v6 with: @@ -43,42 +38,24 @@ jobs: - name: Setup npm run: npm i -g npm@8.5.x - - name: Get pip cache dir - id: pip-cache-dir - run: | - echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT - - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v6 + - name: Install uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/testing.txt') }} - restore-keys: ${{ runner.os }}-pip- + enable-cache: true + python-version: ${{ matrix.python-version }} - name: Install Required Python Dependencies - env: - PIP_SRC: ${{ runner.temp }} run: | make test-requirements - name: Install npm - env: - PIP_SRC: ${{ runner.temp }} run: npm ci - - name: Install python packages - env: - PIP_SRC: ${{ runner.temp }} - run: | - pip install -e . - - name: Run Quality Tests env: - PIP_SRC: ${{ runner.temp }} TARGET_BRANCH: ${{ github.base_ref }} run: | - ruff check --output-format=github . + uv run ruff check --output-format=github . make xsslint make pii_check make check_keywords diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml index 2a971adae360..44461291b189 100644 --- a/.github/workflows/semgrep.yml +++ b/.github/workflows/semgrep.yml @@ -27,14 +27,14 @@ jobs: with: fetch-depth: 1 - - uses: actions/setup-python@v6 + - name: Install uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 with: + enable-cache: true python-version: "${{ matrix.python-version }}" - name: Install semgrep - run: | - make pre-requirements - pip-sync requirements/edx/semgrep.txt + run: uv sync --project scripts/semgrep --frozen - name: Run semgrep env: @@ -43,5 +43,5 @@ jobs: # https://github.com/returntocorp/semgrep/issues/8608 COLUMNS: 80 run: | - semgrep scan --config test_root/semgrep/ --error --quiet \ + uv run --project scripts/semgrep semgrep scan --config test_root/semgrep/ --error --quiet \ -- lms cms common openedx diff --git a/.github/workflows/static-assets-check.yml b/.github/workflows/static-assets-check.yml index a82b97d4e86f..f5bdad03ba92 100644 --- a/.github/workflows/static-assets-check.yml +++ b/.github/workflows/static-assets-check.yml @@ -38,11 +38,6 @@ jobs: - name: Checkout repo uses: actions/checkout@v7 - - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 - with: - python-version: ${{ matrix.python-version }} - - name: Install system Packages run: | sudo apt-get update @@ -56,29 +51,14 @@ jobs: - name: Setup npm run: npm i -g npm@${{ matrix.npm-version }} - - name: Get pip cache dir - id: pip-cache-dir - run: | - echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT - - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v6 + - name: Install uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }} - restore-keys: ${{ runner.os }}-pip- + enable-cache: true + python-version: ${{ matrix.python-version }} - - name: Install Limited Python Deps for Build - run: | - # Install pip-tools.txt first to pin setuptools<82 before installing - # assets.txt. setuptools 82+ removed pkg_resources, which pyfilesystem2 - # (fs) still uses for namespace package declarations. The constraints.txt - # pin covers full installs, but this step only installs assets.txt so we - # pre-install pip-tools.txt to ensure setuptools 81.x is in place. - # See: https://github.com/PyFilesystem/pyfilesystem2/issues/577 - pip install -r requirements/pip-tools.txt - pip install -r requirements/edx/assets.txt + - name: Install Python Deps for Build and Collection + run: uv sync --no-default-groups --group bundled --group assets --frozen - name: Add node_modules bin to $Path run: echo $GITHUB_WORKSPACE/node_modules/.bin >> $GITHUB_PATH @@ -97,14 +77,10 @@ jobs: npm clean-install npm run build - - name: Install Full Python Deps for Collection - run: | - pip install -r requirements/edx/base.txt -e . - - name: Check Assets Collection env: LMS_CFG: lms/envs/minimal.yml CMS_CFG: lms/envs/minimal.yml run: | - DJANGO_SETTINGS_MODULE=lms.envs.production ./manage.py lms collectstatic --noinput - DJANGO_SETTINGS_MODULE=cms.envs.production ./manage.py cms collectstatic --noinput + DJANGO_SETTINGS_MODULE=lms.envs.production uv run ./manage.py lms collectstatic --noinput + DJANGO_SETTINGS_MODULE=cms.envs.production uv run ./manage.py cms collectstatic --noinput diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index e458c2bf103c..ef667d755673 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -86,32 +86,33 @@ jobs: with: mongodb-version: ${{ matrix.mongo-version }} - - name: Setup Python - uses: actions/setup-python@v6 + - name: Install uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 with: + enable-cache: true python-version: ${{ matrix.python-version }} - name: install requirements run: | - make test-requirements - if [[ "${{ matrix.django-version }}" != "pinned" ]]; then - pip install "django~=${{ matrix.django-version }}.0" - pip check # fail if this test-reqs/Django combination is broken + if [[ "${{ matrix.django-version }}" == "pinned" ]]; then + make test-requirements + else + uv sync --no-default-groups --group testing --group "django${{ matrix.django-version }}" --frozen fi - name: list installed package versions run: | - pip freeze + uv tree - name: set settings path shell: bash run: | - echo "settings_path=$(python scripts/unit_test_shards_parser.py --shard-name=${{ matrix.shard_name }} --output settings )" >> $GITHUB_ENV + echo "settings_path=$(uv run python scripts/unit_test_shards_parser.py --shard-name=${{ matrix.shard_name }} --output settings )" >> $GITHUB_ENV - name: get unit tests for shard shell: bash run: | - echo "unit_test_paths=$(python scripts/unit_test_shards_parser.py --shard-name=${{ matrix.shard_name }} )" >> $GITHUB_ENV + echo "unit_test_paths=$(uv run python scripts/unit_test_shards_parser.py --shard-name=${{ matrix.shard_name }} )" >> $GITHUB_ENV if [[ "${{ github.ref }}" == "refs/heads/master" ]]; then echo "report_log_arg=--report-log=reports/pytest-report-${{ matrix.shard_name }}.jsonl" >> $GITHUB_ENV else @@ -125,7 +126,7 @@ jobs: # (~3.7k lines of noise). The .coverage data file is still written and # uploaded as an artifact for the combined `coverage` job below, which # is what reports to codecov. - python -Wd -m pytest -p no:randomly --ds=${{ env.settings_path }} ${{ env.unit_test_paths }} \ + uv run python -Wd -m pytest -p no:randomly --ds=${{ env.settings_path }} ${{ env.unit_test_paths }} \ --cov=. --cov-report="" ${{ env.report_log_arg }} - name: Upload pytest timing report @@ -167,36 +168,37 @@ jobs: runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v7 - - name: Setup Python - uses: actions/setup-python@v6 - with: - python-version: 3.12 - name: install system requirements run: | sudo apt-get update && sudo apt-get install libxmlsec1-dev + - name: Install uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + enable-cache: true + python-version: "3.12" + - name: install requirements - run: | - make test-requirements + run: make test-requirements - name: collect tests from all modules shell: bash run: | - echo "root_cms_unit_tests_count=$(pytest --disable-warnings --collect-only --ds=cms.envs.test cms/ -q | head -n -2 | wc -l)" >> $GITHUB_ENV - echo "root_lms_unit_tests_count=$(pytest --disable-warnings --collect-only --ds=lms.envs.test lms/ openedx/ common/djangoapps/ xmodule/ -q | head -n -2 | wc -l)" >> $GITHUB_ENV + echo "root_cms_unit_tests_count=$(uv run pytest --disable-warnings --collect-only --ds=cms.envs.test cms/ -q | head -n -2 | wc -l)" >> $GITHUB_ENV + echo "root_lms_unit_tests_count=$(uv run pytest --disable-warnings --collect-only --ds=lms.envs.test lms/ openedx/ common/djangoapps/ xmodule/ -q | head -n -2 | wc -l)" >> $GITHUB_ENV - name: get GHA unit test paths shell: bash run: | - echo "cms_unit_test_paths=$(python scripts/gha_unit_tests_collector.py --cms-only)" >> $GITHUB_ENV - echo "lms_unit_test_paths=$(python scripts/gha_unit_tests_collector.py --lms-only)" >> $GITHUB_ENV + echo "cms_unit_test_paths=$(uv run python scripts/gha_unit_tests_collector.py --cms-only)" >> $GITHUB_ENV + echo "lms_unit_test_paths=$(uv run python scripts/gha_unit_tests_collector.py --lms-only)" >> $GITHUB_ENV - name: collect tests from GHA unit test shards shell: bash run: | - echo "cms_unit_tests_count=$(pytest --disable-warnings --collect-only --ds=cms.envs.test ${{ env.cms_unit_test_paths }} -q | head -n -2 | wc -l)" >> $GITHUB_ENV - echo "lms_unit_tests_count=$(pytest --disable-warnings --collect-only --ds=lms.envs.test ${{ env.lms_unit_test_paths }} -q | head -n -2 | wc -l)" >> $GITHUB_ENV + echo "cms_unit_tests_count=$(uv run pytest --disable-warnings --collect-only --ds=cms.envs.test ${{ env.cms_unit_test_paths }} -q | head -n -2 | wc -l)" >> $GITHUB_ENV + echo "lms_unit_tests_count=$(uv run pytest --disable-warnings --collect-only --ds=lms.envs.test ${{ env.lms_unit_test_paths }} -q | head -n -2 | wc -l)" >> $GITHUB_ENV - name: add unit tests count shell: bash @@ -288,7 +290,7 @@ jobs: coverage: if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/openedx-platform' && (startsWith(github.base_ref, 'release') == false)) runs-on: ubuntu-24.04 - needs: [run-tests] + needs: [merge-artifacts] strategy: matrix: python-version: @@ -297,25 +299,24 @@ jobs: - name: Checkout repo uses: actions/checkout@v7 - - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 - with: - python-version: ${{ matrix.python-version }} - - name: Download all artifacts uses: actions/download-artifact@v8 with: - pattern: coverage-* - merge-multiple: true + name: coverage path: reports + - name: Install uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + enable-cache: true + python-version: ${{ matrix.python-version }} + - name: Install Python dependencies - run: | - pip install -r requirements/edx/coverage.txt + run: uv sync --no-default-groups --only-group coverage --no-install-project --frozen - name: Run coverage run: | - coverage combine reports/* - coverage report - coverage xml + uv run coverage combine reports/* + uv run coverage report + uv run coverage xml - uses: codecov/codecov-action@v7 diff --git a/.github/workflows/units-test-scripts-structures-pruning.yml b/.github/workflows/units-test-scripts-structures-pruning.yml index 57efd31db9e4..39659447688f 100644 --- a/.github/workflows/units-test-scripts-structures-pruning.yml +++ b/.github/workflows/units-test-scripts-structures-pruning.yml @@ -20,16 +20,11 @@ jobs: - name: Checkout code uses: actions/checkout@v7 - - name: Set up Python - uses: actions/setup-python@v6 + - name: Install uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 with: + enable-cache: true python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r scripts/structures_pruning/requirements/testing.txt - - name: Run pytest - run: | - pytest scripts/structures_pruning + run: uv run --project scripts/structures_pruning --group test pytest scripts/structures_pruning diff --git a/.github/workflows/units-test-scripts-user-retirement.yml b/.github/workflows/units-test-scripts-user-retirement.yml index bb5319321fe5..8bccf2018cb7 100644 --- a/.github/workflows/units-test-scripts-user-retirement.yml +++ b/.github/workflows/units-test-scripts-user-retirement.yml @@ -20,16 +20,11 @@ jobs: - name: Checkout code uses: actions/checkout@v7 - - name: Set up Python - uses: actions/setup-python@v6 + - name: Install uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 with: + enable-cache: true python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r scripts/user_retirement/requirements/testing.txt - - name: Run pytest - run: | - pytest scripts/user_retirement + run: uv run --project scripts/user_retirement --group test pytest scripts/user_retirement diff --git a/.github/workflows/upgrade-one-python-dependency.yml b/.github/workflows/upgrade-one-python-dependency.yml index d0d55808a82f..3c652d4a0e55 100644 --- a/.github/workflows/upgrade-one-python-dependency.yml +++ b/.github/workflows/upgrade-one-python-dependency.yml @@ -13,7 +13,7 @@ on: required: true type: string version: - description: "Version number to upgrade to in constraints.txt (only needed if pinned)" + description: "Version number to upgrade to, if pinned in [tool.edx_lint].uv_constraints (only needed if pinned)" default: "" type: string change_desc: @@ -36,9 +36,10 @@ jobs: with: ref: "${{ inputs.branch }}" - - name: Set up Python environment - uses: actions/setup-python@v6 + - name: Install uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 with: + enable-cache: true python-version: "3.12" - name: Update any pinned dependencies @@ -46,7 +47,8 @@ jobs: NEW_VERSION: "${{ inputs.version }}" PACKAGE: "${{ inputs.package }}" run: | - sed 's/^\('$PACKAGE'[^#]*\)==[^ #]\+/\1=='$NEW_VERSION'/' -i requirements/constraints.txt + test -z "$NEW_VERSION" && exit 0 + uv run --no-project --with tomlkit python3 scripts/update_uv_constraint_version.py - name: Run make upgrade-package env: @@ -88,7 +90,11 @@ jobs: branch-suffix: short-commit-hash add-paths: | requirements + scripts/**/pyproject.toml + scripts/**/uv.lock scripts/**/requirements* + pyproject.toml + uv.lock commit-message: | feat: Upgrade Python dependency ${{ inputs.package }} diff --git a/.gitignore b/.gitignore index 9e69f1e92306..346c144d81f2 100644 --- a/.gitignore +++ b/.gitignore @@ -6,9 +6,6 @@ # Files that should be git-ignored, but are hand-edited or otherwise valued, # and so should not be destroyed by "make clean". # start-noclean -requirements/private.txt -requirements/edx/private.in -requirements/edx/private.txt lms/envs/private.py cms/envs/private.py .venv/ diff --git a/.readthedocs.yaml b/.readthedocs.yaml index fef9e3c2a9d2..3966059ec16a 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -5,15 +5,12 @@ build: tools: python: "3.12" -sphinx: - configuration: docs/conf.py - python: install: - # Need to install this to set the correct version of steuptools for now - # because it is needed by fs - # See https://github.com/openedx/openedx-platform/issues/38068 for details. - - requirements: "requirements/pip-tools.txt" - - requirements: "requirements/edx/doc.txt" - - method: pip - path: . + - method: uv + command: sync + groups: + - docs + +sphinx: + configuration: docs/conf.py diff --git a/Makefile b/Makefile index 98931cb608d2..c58afb8a106e 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ compile-requirements detect_changed_source_translations dev-requirements \ docs extract_translations \ guides help lint-imports local-requirements migrate migrate-lms migrate-cms \ - pre-requirements pull pull_xblock_translations pull_translations push_translations \ + pull pull_xblock_translations pull_translations push_translations \ requirements shell swagger \ technical-docs test-requirements ubuntu-requirements upgrade-package upgrade @@ -30,20 +30,20 @@ docs: swagger guides technical-docs ## build the documentation for this reposito $(MAKE) -C docs html swagger: ## generate the swagger.yaml file - DJANGO_SETTINGS_MODULE=docs.docs_settings python manage.py lms generate_swagger --generator-class=edx_api_doc_tools.ApiSchemaGenerator -o $(SWAGGER) + DJANGO_SETTINGS_MODULE=docs.docs_settings uv run python manage.py lms generate_swagger --generator-class=edx_api_doc_tools.ApiSchemaGenerator -o $(SWAGGER) extract_translations: ## extract localizable strings from sources - i18n_tool extract --no-segment -v + uv run i18n_tool extract --no-segment -v cd conf/locale/en/LC_MESSAGES && msgcat djangojs.po underscore.po -o djangojs.po pull_plugin_translations: ## Pull translations for edx_django_utils.plugins for both lms and cms - python manage.py lms pull_plugin_translations --verbose $(ATLAS_OPTIONS) - python manage.py lms compile_plugin_translations + uv run python manage.py lms pull_plugin_translations --verbose $(ATLAS_OPTIONS) + uv run python manage.py lms compile_plugin_translations pull_xblock_translations: ## pull xblock translations via atlas - python manage.py lms pull_xblock_translations --verbose $(ATLAS_OPTIONS) - python manage.py lms compile_xblock_translations - python manage.py cms compile_xblock_translations + uv run python manage.py lms pull_xblock_translations --verbose $(ATLAS_OPTIONS) + uv run python manage.py lms compile_xblock_translations + uv run python manage.py cms compile_xblock_translations clean_translations: ## Remove existing translations to prepare for a fresh pull # Removes core edx-platform translations but keeps config files and Esperanto (eo) test translations @@ -54,107 +54,129 @@ clean_translations: ## Remove existing translations to prepare for a fresh pull pull_translations: clean_translations ## pull translations via atlas make pull_xblock_translations make pull_plugin_translations - atlas pull $(ATLAS_OPTIONS) \ + uv run atlas pull $(ATLAS_OPTIONS) \ translations/edx-platform/conf/locale:conf/locale \ $(ATLAS_EXTRA_SOURCES) - python manage.py lms compilemessages - python manage.py lms compilejsi18n - python manage.py cms compilejsi18n + uv run python manage.py lms compilemessages + uv run python manage.py lms compilejsi18n + uv run python manage.py cms compilejsi18n detect_changed_source_translations: ## check if translation files are up-to-date - i18n_tool changed + uv run i18n_tool changed -pre-requirements: ## install Python requirements for running pip-tools - pip install -r requirements/pip-tools.txt +local-requirements: ## no-op; kept for backwards compatibility -- uv sync handles this now + @echo "WARNING: 'make local-requirements' is a no-op post uv migration. Please update your code." -local-requirements: -# edx-platform installs some Python projects from within the edx-platform repo itself. - pip install -e . +dev-requirements: ## install development environment requirements + uv sync --group development --group ci --frozen -dev-requirements: pre-requirements - @# The "$(wildcard..)" is to include private.txt if it exists, and make no mention - @# of it if it does not. Shell wildcarding can't do that with default options. - pip-sync requirements/edx/development.txt $(wildcard requirements/edx/private.txt) - make local-requirements +base-requirements: ## install only production/runtime dependencies + uv sync --no-default-groups --group bundled --frozen -base-requirements: pre-requirements - pip-sync requirements/edx/base.txt - make local-requirements - -test-requirements: pre-requirements - pip-sync --pip-args="--exists-action=w" requirements/edx/testing.txt - make local-requirements +test-requirements: ## install production dependencies plus the testing group (used by CI and tox) + uv sync --no-default-groups --group testing --frozen requirements: dev-requirements ## install development environment requirements -# Order is very important in this list: files must appear after everything they include! -REQ_FILES = \ - requirements/edx/coverage \ - requirements/edx-sandbox/base \ - requirements/edx/base \ - requirements/edx/doc \ - requirements/edx/testing \ - requirements/edx/assets \ - requirements/edx/development \ - requirements/edx/semgrep \ - scripts/xblock/requirements \ - scripts/user_retirement/requirements/base \ - scripts/user_retirement/requirements/testing \ - scripts/structures_pruning/requirements/base \ - scripts/structures_pruning/requirements/testing - -define COMMON_CONSTRAINTS_TEMP_COMMENT -# This is a temporary solution to override the real common_constraints.txt\n# In edx-lint, until the pyjwt constraint in edx-lint has been removed.\n# See BOM-2721 for more details.\n# Below is the copied and edited version of common_constraints\n -endef - -COMMON_CONSTRAINTS_TXT=requirements/common_constraints.txt -.PHONY: $(COMMON_CONSTRAINTS_TXT) -$(COMMON_CONSTRAINTS_TXT): - curl -L https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt > "$(@)" - printf "$(COMMON_CONSTRAINTS_TEMP_COMMENT)" | cat - $(@) > temp && mv temp $(@) - -compile-requirements: export CUSTOM_COMPILE_COMMAND=make upgrade -compile-requirements: pre-requirements ## Re-compile *.in requirements to *.txt - @# Bootstrapping: Rebuild pip and pip-tools first, and then install them - @# so that if there are any failures we'll know now, rather than the next - @# time someone tries to use the outputs. - sed 's/Django<5.0//g' requirements/common_constraints.txt > requirements/common_constraints.tmp - mv requirements/common_constraints.tmp requirements/common_constraints.txt - sed 's/pip<25.3//g' requirements/common_constraints.txt > requirements/common_constraints.tmp - mv requirements/common_constraints.tmp requirements/common_constraints.txt - - pip-compile -v --allow-unsafe ${COMPILE_OPTS} -o requirements/pip-tools.txt requirements/pip-tools.in - pip install -r requirements/pip-tools.txt - - @ export REBUILD='--rebuild'; \ - for f in $(REQ_FILES); do \ +compile-requirements: ## Regenerate uv.lock for the root project and all uv sub-projects + uv run --no-project --isolated --with edx-lint edx_lint write_uv_constraints pyproject.toml + uv lock ${UV_LOCK_OPTS} + + @# Lock every uv-managed sub-project (each has its own pyproject.toml + uv.lock, + @# independent of the root project's dependency graph) before exporting any + @# compat file below, so a failure in one halts the whole target before + @# anything downstream of it is (re)generated. + @for d in requirements/edx-sandbox scripts/xblock scripts/semgrep scripts/user_retirement scripts/structures_pruning; do \ echo ; \ - echo "== $$f ===============================" ; \ - echo "pip-compile -v $$REBUILD ${COMPILE_OPTS} -o $$f.txt $$f.in"; \ - pip-compile -v $$REBUILD ${COMPILE_OPTS} -o $$f.txt $$f.in || exit 1; \ - export REBUILD=''; \ + echo "== $$d ===============================" ; \ + uv run --no-project --isolated --with edx-lint edx_lint write_uv_constraints $$d/pyproject.toml && \ + (cd $$d && uv lock ${UV_LOCK_OPTS}) \ + || exit 1; \ + done + + @# --- Everything below is DEPR-tracked compatibility-export scaffolding for + @# external tooling (e.g. Tutor's Dockerfile) that still does + @# `pip install -r requirements/edx/.txt` directly instead of using uv. + @# These are GENERATED FILES -- see the header comment in each for what + @# regenerates them. Remove this whole section (and the scripts/*/requirements/ + @# *.txt targets it writes) once external consumers have migrated to `uv sync`: + @# https://github.com/openedx/public-engineering/issues/552 + @mkdir -p requirements/edx + @{ \ + echo "# GENERATED FILE, DO NOT EDIT DIRECTLY."; \ + echo "# Compatibility export of [project.dependencies] plus the 'bundled' group"; \ + echo "# (optional third-party add-ons installed by default) for tools that still"; \ + echo "# 'pip install -r requirements/edx/base.txt' directly instead of using uv."; \ + echo "# Source of truth: [project.dependencies] / [dependency-groups].bundled in pyproject.toml / uv.lock."; \ + uv export --frozen --no-hashes --no-default-groups --group bundled --no-emit-project; \ + } > requirements/edx/base.txt + @{ \ + echo "# GENERATED FILE, DO NOT EDIT DIRECTLY."; \ + echo "# Compatibility export of the 'assets' dependency-group for tools that still"; \ + echo "# 'pip install -r requirements/edx/assets.txt' directly instead of using uv."; \ + echo "# Source of truth: [dependency-groups].assets in pyproject.toml / uv.lock."; \ + uv export --frozen --no-hashes --only-group assets --no-emit-project; \ + } > requirements/edx/assets.txt + @{ \ + echo "# GENERATED FILE, DO NOT EDIT DIRECTLY."; \ + echo "# Compatibility export of the 'development' and 'ci' dependency-groups for"; \ + echo "# tools that still 'pip install -r requirements/edx/development.txt'"; \ + echo "# directly instead of using uv."; \ + echo "# Source of truth: [dependency-groups].development / .ci in pyproject.toml / uv.lock."; \ + uv export --frozen --no-hashes --group development --group ci --no-emit-project; \ + } > requirements/edx/development.txt + + @# requirements/edx-sandbox, scripts/xblock: single compat export, no dependency-groups. + @{ \ + echo "# GENERATED FILE, DO NOT EDIT DIRECTLY."; \ + echo "# Compatibility export for anyone still 'pip install -r requirements/edx-sandbox/base.txt'"; \ + echo "# directly instead of using uv. Source of truth: requirements/edx-sandbox/pyproject.toml / uv.lock."; \ + (cd requirements/edx-sandbox && uv export --frozen --no-hashes --no-emit-project); \ + } > requirements/edx-sandbox/base.txt + @{ \ + echo "# GENERATED FILE, DO NOT EDIT DIRECTLY."; \ + echo "# Compatibility export for anyone still 'pip install -r scripts/xblock/requirements.txt'"; \ + echo "# directly instead of using uv. Source of truth: scripts/xblock/pyproject.toml / uv.lock."; \ + (cd scripts/xblock && uv export --frozen --no-hashes --no-emit-project); \ + } > scripts/xblock/requirements.txt + + @# scripts/user_retirement and scripts/structures_pruning: base + testing (test group) compat exports. + @for d in scripts/user_retirement scripts/structures_pruning; do \ + { \ + echo "# GENERATED FILE, DO NOT EDIT DIRECTLY."; \ + echo "# Compatibility export for anyone still 'pip install -r $$d/requirements/base.txt'"; \ + echo "# directly instead of using uv. Source of truth: $$d/pyproject.toml / uv.lock."; \ + (cd $$d && uv export --frozen --no-hashes --no-emit-project); \ + } > $$d/requirements/base.txt && \ + { \ + echo "# GENERATED FILE, DO NOT EDIT DIRECTLY."; \ + echo "# Compatibility export for anyone still 'pip install -r $$d/requirements/testing.txt'"; \ + echo "# directly instead of using uv. Source of truth: $$d/pyproject.toml (test group) / uv.lock."; \ + (cd $$d && uv export --frozen --no-hashes --group test --no-emit-project); \ + } > $$d/requirements/testing.txt \ + || exit 1; \ done -upgrade: $(COMMON_CONSTRAINTS_TXT) ## update the pip requirements files to use the latest releases satisfying our constraints - $(MAKE) compile-requirements COMPILE_OPTS="--upgrade" +upgrade: ## update all dependencies (uv.lock for the root project and all uv sub-projects) to the latest releases satisfying our constraints + $(MAKE) compile-requirements UV_LOCK_OPTS="--upgrade" upgrade-package: ## update just one package to the latest usable release @test -n "$(package)" || { echo "\nUsage: make upgrade-package package=...\n"; exit 1; } - $(MAKE) compile-requirements COMPILE_OPTS="--upgrade-package $(package)" + $(MAKE) compile-requirements UV_LOCK_OPTS="--upgrade-package $(package)" check-types: ## run static type-checking tests - mypy + uv run mypy lint-imports: - lint-imports + uv run lint-imports migrate-lms: - python manage.py lms showmigrations --database default --traceback --pythonpath=. - python manage.py lms migrate --database default --traceback --pythonpath=. + uv run python manage.py lms showmigrations --database default --traceback --pythonpath=. + uv run python manage.py lms migrate --database default --traceback --pythonpath=. migrate-cms: - python manage.py cms showmigrations --database default --traceback --pythonpath=. - python manage.py cms migrate --database default --noinput --traceback --pythonpath=. + uv run python manage.py cms showmigrations --database default --traceback --pythonpath=. + uv run python manage.py cms migrate --database default --noinput --traceback --pythonpath=. migrate: migrate-lms migrate-cms @@ -166,33 +188,33 @@ ubuntu-requirements: ## Install ubuntu 22.04 system packages needed for `pip ins sudo apt install libmysqlclient-dev libxmlsec1-dev xsslint: ## check xss for quality issuest - python scripts/xsslint/xss_linter.py \ + uv run python scripts/xsslint/xss_linter.py \ --rule-totals \ --config=scripts.xsslint_config \ --thresholds=scripts/xsslint_thresholds.json ruff: ## check python files with ruff - ruff check . + uv run ruff check . ## Re-enable --lint flag when this issue https://github.com/openedx/edx-platform/issues/35775 is resolved pii_check: ## check django models for pii annotations DJANGO_SETTINGS_MODULE=cms.envs.test \ - code_annotations django_find_annotations \ + uv run code_annotations django_find_annotations \ --config_file .pii_annotations.yml \ --coverage \ --lint DJANGO_SETTINGS_MODULE=lms.envs.test \ - code_annotations django_find_annotations \ + uv run code_annotations django_find_annotations \ --config_file .pii_annotations.yml \ --coverage \ --lint check_keywords: ## check django models for reserve keywords DJANGO_SETTINGS_MODULE=cms.envs.test \ - python manage.py cms check_reserved_keywords \ + uv run python manage.py cms check_reserved_keywords \ --override_file db_keyword_overrides.yml DJANGO_SETTINGS_MODULE=lms.envs.test \ - python manage.py lms check_reserved_keywords \ + uv run python manage.py lms check_reserved_keywords \ --override_file db_keyword_overrides.yml diff --git a/README.rst b/README.rst index 3dd3930626f0..fa344baa2e63 100644 --- a/README.rst +++ b/README.rst @@ -99,12 +99,12 @@ Language Packages: * Backend build: - - ``pip install -r requirements/edx/assets.txt`` + - ``uv sync --no-default-groups --only-group assets`` * Backend application: - - ``pip install -r requirements/edx/base.txt`` (production) - - ``pip install -r requirements/edx/development.txt`` (development) + - ``uv sync --no-default-groups --group bundled`` (production) + - ``uv sync --group development --group ci`` (development, i.e. ``make dev-requirements``) Some Python packages have system dependencies. For example, installing these packages on Debian or Ubuntu will require first running ``sudo apt install python3-dev default-libmysqlclient-dev build-essential pkg-config`` to satisfy the requirements of the ``mysqlclient`` Python package. diff --git a/docs/concepts/testing/testing.rst b/docs/concepts/testing/testing.rst index 5e4248523c65..a4b99d74f905 100644 --- a/docs/concepts/testing/testing.rst +++ b/docs/concepts/testing/testing.rst @@ -99,7 +99,8 @@ Running Python Unit tests ************************* The following commands need to be run within a Python environment in -which requirements/edx/testing.txt has been installed. If you are using a +which the ``testing`` dependency group (``uv sync --no-default-groups --group testing``) has +been installed. If you are using a Docker-based Open edX distribution, then you probably will want to run these commands within the LMS and/or CMS Docker containers. diff --git a/docs/references/static-assets.rst b/docs/references/static-assets.rst index 9a3e3c25b663..55286e5501da 100644 --- a/docs/references/static-assets.rst +++ b/docs/references/static-assets.rst @@ -72,7 +72,7 @@ Building frontend assets requires an active Node and Python environment with dependencies installed:: npm clean-install - pip install -r requirements/edx/assets.txt + uv sync --no-default-groups --group assets Once your environment variables are set and build dependencies are installed, @@ -139,7 +139,7 @@ skip this section. First, ensure you have a Python enironment with all edx-platform dependencies installed:: - pip install -r requirements/edx/base.txt -e . + uv sync --no-default-groups --group bundled Next, download localized versions of edx-platform assets. Under the hood, this command uses the `Open edX Atlas`_ tool, which manages aggregated translations diff --git a/package.json b/package.json index 986878113351..0ab1acaa1ad3 100644 --- a/package.json +++ b/package.json @@ -8,8 +8,8 @@ "build-dev": "npm run webpack-dev && npm run compile-sass-dev", "webpack": "NODE_ENV=${NODE_ENV:-production} webpack --config=${WEBPACK_CONFIG_PATH:-webpack.prod.config.js}", "webpack-dev": "NODE_ENV=development webpack --config=webpack.dev.config.js", - "compile-sass": "scripts/compile_sass.py --env=${NODE_ENV:-production}", - "compile-sass-dev": "scripts/compile_sass.py --env=development", + "compile-sass": "uv run --active python scripts/compile_sass.py --env=${NODE_ENV:-production}", + "compile-sass-dev": "uv run --active python scripts/compile_sass.py --env=development", "watch": "{ npm run watch-webpack& npm run watch-sass& } && sleep infinity", "watch-webpack": "npm run webpack-dev -- --watch", "watch-sass": "scripts/watch_sass.sh", diff --git a/pyproject.toml b/pyproject.toml index 44488938ec95..05771907bb0b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,413 @@ build-backend = "setuptools.build_meta" name = "openedx-platform" version = "0.13" requires-python = ">=3.12" -dependencies = ["setuptools"] +dependencies = [ + "acid-xblock", # This XBlock is used for unit tests as well as manual tests + "analytics-python", # Used for Segment analytics + "attrs", # Reduces boilerplate code involving class attributes + "Babel", # Internationalization utilities, used for date formatting in a few places + "boto3", # Amazon Web Services SDK for Python + "botocore", # via boto3, s3transfer + "bridgekeeper", # Used for determining permissions for courseware. Use of django-rules is preferred. + "celery", # Asynchronous task execution library + "chem", # A helper library for chemistry calculations + "codejail-includes", # CodeJail manages execution of untrusted code in secure sandboxes. + "cryptography", # Implementations of assorted cryptography algorithms + "defusedxml", + "Django", # Web application framework + "django-autocomplete-light", # Enhances Django admin with single-select autocomplete dropdowns for a better user experience. + "django-celery-results", # Only used for the CacheBackend for celery results + "django-config-models", # Configuration models for Django allowing config management with auditing + "django-cors-headers", # Used to allow to configure CORS headers for cross-domain requests + "django-countries", # Country data for Django forms and model fields + "django-crum", # Middleware that stores the current request and user in thread local storage + "django-filter", # Allows users to filter Django querysets dynamically + "django-ipware", # Get the client's real IP address + "django-method-override", # Allow use of HTTP methods normally not supported by browsers via the X-HTTP-Method-Override header + "django-model-utils", + "django-mptt", + "django-mysql", + "django-oauth-toolkit", # Provides oAuth2 capabilities for Django + "django-pipeline", + "django-ratelimit", + "django-sekizai", + "django-simple-history", + "django-statici18n", + "django-storages", + "django-user-tasks", + "django-waffle", + "django-webpack-loader", # Used to wire webpack bundles into the django asset pipeline + "djangorestframework", + "drf-spectacular", + "edx-ace", + "edx-api-doc-tools", + "edx-auth-backends", # Allow Studio to use LMS SSO + "edx-bulk-grades", # LMS REST API for managing bulk grading operations + "edx-ccx-keys", + "edx-celeryutils", + "edx-completion", + "edx-django-release-util", # Release utils for the edx release pipeline + "edx-django-sites-extensions", + "edx-codejail>=4.0.0", # Codejail 4 brings important safety improvements (no unsafe mode by default) + "edx-django-utils>=5.14.1", # edx-django-utils 5.14.1 adds FrontendMonitoringMiddleware; Utilities for cache, monitoring, and plugins + "edx-drf-extensions", + "edx-enterprise", + "edx-event-bus-kafka>=5.6.0", # edx-event-bus-kafka 5.6.0 adds support for putting client ids on event producers/consumers; Kafka implementation of event bus + "edx-event-bus-redis", + "edx-milestones", + "edx-opaque-keys>=2.12.0", + "edx-organizations", + "edx-proctoring>=2.0.1", + "edx-rest-api-client", + "edx-search", + "edx-submissions", + "edx-toggles", # Feature toggles management + "edx-when", + "edxval", + "event-tracking", + "enmerkar-underscore", # Implements a underscore extractor for django-babel. + "fs", + "fs-s3fs", + "geoip2", # Python API for the GeoIP web services and databases + "glob2", # Enhanced glob module, used in openedx.core.lib.rooted_paths + "gunicorn", + "help-tokens", + "html5lib", # HTML parser, used for capa problems + "icalendar", # .ics generator, used by calendar_sync + "ipaddress", # Ip network support for Embargo feature + "importlib_metadata", # Used to access entry_points in i18n_api plugin + "jsonfield", # Django model field for validated JSON; used in several apps + "laboratory", # Library for testing that code refactors/infrastructure changes produce identical results + "lxml[html_clean]", # XML parser + "lti-consumer-xblock>=11.0.0", + "mako", # Primary template language used for server-side page rendering + "Markdown", # Convert text markup to HTML; used in capa problems, forums, and course wikis + "meilisearch", # Library to access Meilisearch search engine (will replace ElasticSearch) + "mongoengine", # Object-document mapper for MongoDB, used in the LMS dashboard + "mysqlclient", # Driver for the default production relational database + "nh3", # Python bindings to the ammonia (whitelist-based HTML sanitizing library); used for capa and LTI + "nodeenv", # Utility for managing Node.js environments; we use this for deployments and testing + "oauthlib", # OAuth specification support for authenticating via LTI or other Open edX services + "olxcleaner", + "openedx-atlas", # CLI tool to manage translations + "openedx-calc", # Library supporting mathematical calculations for Open edX + "openedx-core", + "openedx-django-require", + "openedx-events", # Open edX Events from Hooks Extension Framework (OEP-50) + "openedx-filters", # Open edX Filters from Hooks Extension Framework (OEP-50) + "openedx-forum", # Open edX forum v2 application + "openedx-django-wiki", + "path", + "piexif", # Exif image metadata manipulation, used in the profile_images app + "Pillow", # Image manipulation library; used for course assets, profile images, invoice PDFs, etc. + "psutil", # Library for retrieving information on running processes and system utilization + "pycountry", + "pycryptodomex", + "PyJWT>=1.6.3", # PyJWT 1.6.3 contains PyJWTError, which is required by Apple auth in social-auth-core + "pylti1p3", # Required by content_libraries core library to support LTI 1.3 launches + "pymemcache", # Python interface to the memcached memory cache daemon + "pymongo", # MongoDB driver + "pynliner", # Inlines CSS styles into HTML for email notifications + "python-dateutil", + "python3-openid ; python_version>='3'", + "python3-saml", + "pyuca", # For more accurate sorting of translated country names in django-countries + "pysrt", # Support for SubRip subtitle files, used in the video XModule + "pytz", # Time zone information database + "PyYAML", # Used to parse XModule resource templates + "redis", # celery task broker + "requests-oauthlib", # Simplifies use of OAuth via the requests library, used for CCX and LTI + "random2", + "rules", # Django extension for rules-based authorization checks + "social-auth-core", + "simplejson", + "Shapely", # Geometry library, used for image click regions in capa + "six", # Utilities for supporting Python 2 & 3 in the same codebase + "slumber", # The following dependency is unsupported and used by the DeprecatedRestApiClient + "social-auth-app-django", + "sorl-thumbnail", + "sortedcontainers", # Provides SortedKeyList, used for lists of XBlock assets + "stevedore", # Support for runtime plugins, used for XBlocks and edx-platform Django app plugins + "unicodecsv", # Easier support for CSV files with unicode text + "webob", + "web-fragments", # Provides the ability to render fragments of web pages + "wrapt", # Better functools.wrapped. TODO: functools has since improved, maybe we can switch? + "XBlock[django]", # Courseware component architecture + "xss-utils", # https://github.com/openedx/edx-platform/pull/20633 Fix XSS via Translations + "unicodeit", # Converts mathjax equation to plain text by using unicode symbols + "psycopg2-binary", + "openedx-authz", # Authorization Framework for the Open edX Ecosystem +] + +[project.optional-dependencies] +openstack = [ + "django-storage-swift==1.2.19", +] +[dependency-groups] +coverage = [ + "coverage", # Code coverage testing for Python + "diff-cover", # Automatically find diff lines that need test coverage +] +bundled = [ + # Dependencies that are normally bundled with the platform, but are not core to + # the platform's functionality. They are still installed by default (this group is + # included from `testing`, and explicitly synced in `base-requirements`), but they + # should not be required for the platform to run nor for the test suite to pass. + "edx-i18n-tools>=0.4.6", # i18n_tool is needed at build time for pulling translations; Commands for developers and translators to extract, compile and validate translations + "django-ses", # Django email backend for Amazon’s Simple Email Service + "mailsnake", # MailChimp API; used for two management commands in the "mailing" djangoapp + "optimizely-sdk", # Optimizely provides A/B testing and other features, used by edx.org + "crowdsourcehinter-xblock", + "done-xblock", # a very simple XBlock that allows learners to mark an activity as Done + "recommender-xblock", # https://github.com/edx/RecommenderXBlock + "staff-graded-xblock", # https://github.com/openedx/staff_graded-xblock Allows off-site bulk scoring. + "edx-sga", # The more well known "staff graded assignment" XBlock, from MIT. + "ora2>=4.5.0", # Open Response Assessment XBlock + "xblock-poll", # Xblock for polling users + "xblock-drag-and-drop-v2", # Drag and Drop XBlock + "xblock-google-drive", # XBlock for google docs and calendar + "xblocks-contrib", # Package having multiple core XBlocks, https://github.com/openedx/xblocks-contrib?tab=readme-ov-file#xblocks-being-moved-here + "enterprise-integrated-channels", # Integrated Channels to transmit content metadata and learner data. +] +testing = [ + {include-group = "coverage"}, + {include-group = "bundled"}, + "beautifulsoup4", # Library for extracting data from HTML and XML files + "code-annotations", # Perform code annotation checking, such as for PII annotations + "cssselect", # Used to extract HTML fragments via CSS selectors in 2 test cases and pyquery + "ddt", # Run a test case multiple times with different input; used in many, many of our tests + "edx-lint", # pylint extensions for Open edX repositories + "factory-boy", # Library for creating test fixtures, used in many tests + "freezegun", # Pinning the freezegun version because 0.3.13 is causing failures which have also been reported on the git repo by public.; Allows tests to mock the output of assorted datetime module functions + "httpretty", # Library for mocking HTTP requests, used in many tests + "import-linter", # Tool for making assertions about which modules can import which others + "mock", # Deprecated alias to standard library `unittest.mock` + "ruff", # Fast Python linter and formatter + "polib", # Library for manipulating gettext translation files, used to test paver i18n commands + "pyquery", # jQuery-like API for retrieving fragments of HTML and XML files in tests + "pytest", # Testing framework + "pytest-attrib", # Select tests based on attributes + "pytest-cov", # pytest plugin for measuring code coverage + "pytest-django", # Django support for pytest + "pytest-json-report", # Output json formatted warnings after running pytest + "pytest-metadata", # To prevent 'make upgrade' failure, dependency of pytest-json-report + "pytest-randomly", # pytest plugin to randomly order tests + "pytest-reportlog", # Per-test timing data including setup/teardown (used for shard rebalancing) + "pytest-xdist[psutil]", # Parallel execution of tests on multiple CPU cores or hosts + "singledispatch", # Backport of functools.singledispatch from Python 3.4+, used in tests of XBlock rendering + "testfixtures", # Provides a LogCapture utility used by several tests + "unidiff", # Required by coverage_pytest_plugin + "pylint-pytest", # A Pylint plugin to suppress pytest-related false positives. + "pact-python", # Library for contract testing + "py", # Needed for pytest configurations, was previously been fetched through tox +] +docs = [ + # matches requirements/edx/doc.in's old `-r base.txt` -- docs/conf.py imports + # cms.envs.common, whose add_optional_apps() needs the bundled XBlocks/apps + # importable to succeed (e.g. openassessment), not just doc-building tools. + {include-group = "bundled"}, + "code-annotations", # provides annotations for certain documentation + "sphinx-book-theme", # Common theme for all Open edX projects + "gitpython", # fetch git repo information + "Sphinx", # Documentation builder + "sphinx-design", # provides various responsive web-components + "sphinxcontrib-openapi", # Be able to render openapi schema in a sphinx project + "sphinxext-rediraffe", # Quickly and easily redirect when we move pages around. + "sphinx-reredirects", # Redirect from a sphinx project out to other places on the web including other sphinx projects + "sphinx-autoapi", +] +assets = [ + "click", + "libsass", + "nodeenv", +] +development = [ + {include-group = "testing"}, + {include-group = "docs"}, + {include-group = "assets"}, + "django-debug-toolbar", # A set of panels that display debug information about the current request/response + "django-stubs[compatible-mypy]", # Typing stubs for Django, so it works with mypy + "djangorestframework-stubs", # Typing stubs for DRF + "mypy", # static type checking + "pywatchman", # More efficient checking for runserver reload trigger events + "types-requests", # Typing stubs for requests + "vulture", # Detects possible dead/unused code, used in scripts/find-dead-code.sh + "watchdog", # Used by `npm run watch` to auto-recompile when assets are changed +] +# Alternate-Django-version test matrix groups. Each is a real, independently-resolved +# environment (not a runtime `uv pip install` patch on top of `testing`) — see +# [tool.uv].conflicts below, which is required for these to coexist in one uv.lock at +# all. Currently unexercised: the CI matrices that would select one of these +# (migrations-check.yml, unit-tests.yml) only ever run with django-version: "pinned", +# which resolves independently via `testing`'s own (unconstrained-except-<6.0) Django +# requirement. If re-enabling that matrix dimension, add the matching fork to +# [tool.uv].conflicts for whichever combination gets exercised together. +django42 = [ + {include-group = "testing"}, + "django>=4.2,<4.3", +] +django52 = [ + {include-group = "testing"}, + "django>=5.2,<5.3", +] +ci = [ + "tox", + "tox-uv", +] + +[tool.uv] +default-groups = [] # explicit, not omitted -- uv's own default-groups fallback is ["dev"] +conflicts = [ + [ + { group = "django42" }, + { group = "django52" }, + ], +] + +# DO NOT EDIT constraint-dependencies DIRECTLY. +# This list is managed by `edx_lint write_uv_constraints` and will be +# overwritten the next time `make upgrade` is run. +# - GLOBAL constraints: edit edx_lint/files/common_constraints.txt +# - REPO-SPECIFIC constraints: edit [tool.edx_lint].uv_constraints in this file +constraint-dependencies = [ + "Django<6.0", + "elasticsearch==7.9.1", + "social-auth-app-django<=5.4.1", + "social-auth-core<5.0.0", + "pip<26.2.1", + "celery>=5.2.2,!=5.6.1,<6.0.0", + "django-oauth-toolkit==1.7.1", + "django-stubs<6", + "libsass==0.10.0", + "numpy<2.0.0", + "openedx-core<2", + "path<16.12.0", + "pymongo<4.4.1", + "lxml==5.3.2", + "xmlsec==1.3.14", + "django-debug-toolbar<6.0.0", + "sphinx-autoapi<3.6.1", + "setuptools<82", + "astroid==4.0.4", + "edx-enterprise==8.9.2", +] +[tool.edx_lint] +uv_constraints = [ + # Date: 2025-10-07 + # Stay on LTS version, remove once this is added to common constraint + "Django<6.0", + # Date: 2026-01-13 + # We would normally pin celery to <6.0.0 to avoid auto-updating across a major + # version boundary without more thorough testing. The reason it's currently also + # pinned to !=5.6.1 is because of a celery bug related to the eta and countdown + # parameters. This bug caused operational issues in MIT's deployment. + # Issue for unpinning: https://github.com/openedx/edx-platform/issues/35280 + "celery>=5.2.2,!=5.6.1,<6.0.0", + # Date: 2020-02-10 + # django-oauth-toolkit version >=2.0.0 has breaking changes. More details + # mentioned on this issue https://github.com/openedx/edx-platform/issues/32884 + # Issue for unpinning: https://github.com/openedx/edx-platform/issues/35277 + "django-oauth-toolkit==1.7.1", + # Date: 2024-07-19 + # Generally speaking, the major version of django-stubs must either match the major version + # of django, or exceed it by 1. So, we will need to perpetually constrain django-stubs and + # update it as we perform django upgrades. For more details, see: + # https://github.com/typeddjango/django-stubs?tab=readme-ov-file#version-compatibility + # including the note on "Partial Support". + # Issue: https://github.com/openedx/edx-platform/issues/35275 + "django-stubs<6", + # Date: 2023-07-26 + # Our legacy Sass code is incompatible with anything except this ancient libsass version. + # Here is a ticket to upgrade, but it's of debatable importance given that we are rapidly moving + # away from legacy LMS/CMS frontends: + # https://github.com/openedx/edx-platform/issues/31616 + "libsass==0.10.0", + # Date: 2024-07-16 + # We need to upgrade the version of elasticsearch to at least 7.15 before we can upgrade to Numpy 2.0.0 + # Otherwise we see a failure while running the following command: + # export DJANGO_SETTINGS_MODULE=cms.envs.test; python manage.py cms check_reserved_keywords --override_file db_keyword_overrides.yml --report_path reports/reserved_keywords --report_file cms_reserved_keyword_report.csv + # Issue for unpinning: https://github.com/openedx/edx-platform/issues/35126 + "numpy<2.0.0", + # Date: 2023-09-18 + # Library is still in active development. Major versions require review + # from openedx-maintainers. Minor and patch versions can roll out automatically. + # Issue for unpinning: https://github.com/openedx/edx-platform/issues/35269 + "openedx-core<2", + # Date: 2024-04-26 + # path==16.12.0 breaks the unit test collections check + # needs to be investigated and fixed separately + # Issue for unpinning: https://github.com/openedx/edx-platform/issues/35267 + "path<16.12.0", + # Date: 2020-04-08 + # Adding pin to avoid any major upgrade + # Issue for unpinning: https://github.com/openedx/edx-platform/issues/35265 + "pymongo<4.4.1", + # Date: 2024-08-06 + # social-auth-app-django 5.4.2 introduces a new migration that will not play nicely with large installations. This will touch + # user tables, which are quite large, especially on instances like edx.org. + # We are pinning this until after all the smaller migrations get handled and then we can migrate this all at once. + # Issue for unpinning: https://github.com/openedx/edx-platform/issues/37639 + "social-auth-app-django<=5.4.1", + # Date 2026-07-02 + # social-auth-core 5.x changes the OAuth pipeline's post-login redirect + # behavior, breaking common/djangoapps/third_party_auth's integration + # tests (AzureAD/Google/LinkedIn/Twitter full-pipeline specs). Keep on + # the 4.x line until that's investigated and fixed separately -- see the + # social-auth-app-django pin above for a related, already-deferred + # migration in this dependency family. + # Issue for unpinning: https://github.com/openedx/edx-platform/issues/38841 + "social-auth-core<5.0.0", + # Date 2025-01-08 + # elasticsearch==7.13.x is downgrading urllib3 from 2.2.3 to 1.26.20 + # https://github.com/elastic/elasticsearch-py/blob/v7.13.4/setup.py#L42 + # We are pinning this until we can upgrade to a version of elasticsearch that uses a more recent version of urllib3. + # Issue for unpinning: https://github.com/openedx/edx-platform/issues/35126 + "elasticsearch==7.9.1", + # Date 2025-05-09 + # lxml and xmlsec need to be constrained because the latest version builds against a newer + # version of libxml2 than what we're running with. This leads to a version mismatch error + # at runtime. You can re-produce it by running any test. + # If lxml is pinned in the future and you see this error, it may be that the system libxml2 + # is now shipping the correct version and we can un-pin this. + # Issue: https://github.com/openedx/edx-platform/issues/36695 + "lxml==5.3.2", + "xmlsec==1.3.14", + # Date 2025-08-12 + # The newest version of the debug toolbar has a bug in it + # https://github.com/django-commons/django-debug-toolbar/issues/2172 + # Pin this back to the previous version until that bug is fixed. + "django-debug-toolbar<6.0.0", + # Date 2026-01-13 + # Sphinx-autoapi changed the version of astroid it needs + # but the newer version is not compatible with the current pylint version + # which wants a newer version of astroid. This can be removed once we're + # building requirements with Python 3.12 + # https://github.com/openedx/edx-platform/issues/37880 + "sphinx-autoapi<3.6.1", + # Date 2026-03-02 + # setuptools 82.0.0 removed pkg_resources from its distribution, but fs (pyfilesystem2) + # still uses pkg_resources for namespace package declarations. This constraint can be + # removed once pyfilesystem2 drops its pkg_resources usage. + # https://github.com/PyFilesystem/pyfilesystem2/issues/577 + # Issue for unpinning: https://github.com/openedx/openedx-platform/issues/38068 + "setuptools<82", + # Date 2026-03-02 + # The latest version of pylint pins back astroid to an older version. + # This holdback is not caught in the docs requirements file and since both the docs + # and testing file are required in the development.in file, we fail to compile + # development.txt because of conflicting dependencies. + # + # Holding astroid back until pylint releases a new version that works with the latest + # version of astroid. + # https://github.com/openedx/openedx-platform/issues/38066 + "astroid==4.0.4", + # Date: 2019-08-16 (bumped 2026-08-22 to match master's own already-reviewed + # bump in requirements/constraints.txt, PR #39012) + # The team that owns this package will manually bump this package rather than + # having it pulled in automatically. This is to allow them to better control its + # deployment and to do it in a process that works better for them. + "edx-enterprise==8.9.2", +] [tool.setuptools] packages = ["cms", "common", "lms", "openedx", "xmodule"] @@ -206,6 +612,60 @@ norecursedirs = ". .* *.egg build conf dist node_modules test_root cms/envs lms/ python_classes = [] python_files = ["tests.py", "test_*.py", "tests_*.py", "*_tests.py", "__init__.py"] +[tool.coverage.run] +branch = true +data_file = "reports/.coverage" +source = [ + "cms", + "common/djangoapps", + "lms", + "openedx", + "pavelib", + "scripts", + "xmodule", +] +omit = [ + "cms/envs/*", + "cms/manage.py", + "cms/djangoapps/contentstore/views/dev.py", + "cms/djangoapps/*/migrations/*", + "cms/djangoapps/*/features/*", + "cms/lib/*/migrations/*", + "lms/debug/*", + "lms/envs/*", + "lms/djangoapps/*/migrations/*", + "lms/djangoapps/*/features/*", + "common/djangoapps/*/migrations/*", + "openedx/core/djangoapps/*/migrations/*", + "openedx/core/djangoapps/debug/*", + "openedx/envs/*", + "openedx/features/*/migrations/*", +] +concurrency = ["multiprocessing"] +parallel = true +relative_files = true + +[tool.coverage.report] +ignore_errors = true +exclude_lines = [ + "pragma: no cover", + "raise NotImplementedError", +] + +[tool.coverage.html] +title = "edx-platform Python Test Coverage Report" +directory = "reports/cover" + +[tool.coverage.xml] +output = "reports/coverage.xml" + +[tool.coverage.paths] +jenkins_source = [ + "/home/jenkins/workspace/$JOB_NAME", + "/home/jenkins/workspace/$SUBSET_JOB", + "/home/jenkins/edx-platform", +] + [tool.ruff] line-length = 120 exclude = [ diff --git a/requirements/README.rst b/requirements/README.rst index c602cc8905b1..1ca5a31bb91f 100644 --- a/requirements/README.rst +++ b/requirements/README.rst @@ -1,20 +1,45 @@ Requirements/dependencies ######################### -These directories specify the Python (and system) dependencies for the LMS and Studio. +The main application's Python dependencies are declared in the root +``pyproject.toml`` (``[project.dependencies]`` for runtime deps, +``[dependency-groups]`` for testing/development/doc/assets tooling, +and ``[tool.edx_lint].uv_constraints`` for repo-specific version pins) and +locked in the root ``uv.lock``, managed with `uv`_. + +This ``requirements/`` directory now holds ``edx-sandbox``, its own +standalone ``uv``-managed project (``pyproject.toml`` + ``uv.lock``) for +Codejail's isolated sandbox environment, plus ``requirements/edx/*.txt`` -- +generated compatibility exports of the main app's dependencies (regenerated +by ``make compile-requirements``, not hand-edited) for external tools like +Tutor/Devstack that still do ``pip install -r requirements/edx/base.txt`` +directly instead of using ``uv sync``. + +The four standalone script directories at the repo root (``scripts/xblock``, +``scripts/user_retirement``, ``scripts/structures_pruning``, ``scripts/semgrep``) +each have their own ``pyproject.toml`` + ``uv.lock`` too, independent of both +the main app and each other. + +All of these are manipulated using the Makefile targets below in a Linux +environment (to match our build and deploy systems); for developers on Mac, +this can be achieved by using the GitHub workflows or by running Make targets +from inside devstack's lms-shell or another Linux environment. + +.. _uv: https://docs.astral.sh/uv/ -- ``edx`` contains the normal Python requirements files -- ``edx-sandbox`` contains the requirements files for Codejail -- ``constraints.txt`` is shared between the two - -(In a normal `OEP-18`_-compliant repository, the ``*.in`` and ``*.txt`` files would be -directly in the requirements directory.) - -.. _OEP-18: https://github.com/openedx/open-edx-proposals/blob/master/oeps/oep-0018-bp-python-dependencies.rst +If you don't have write permissions to openedx/edx-platform, you'll need to run these workflows on a fork. -While the ``*.in`` files are intended to be updated manually, the ``*.txt`` files should only be manipulated using Makefile targets in a Linux environment (to match our build and deploy systems). For developers on Mac, this can be achieved by using the GitHub workflows or by running Make targets from inside devstack's lms-shell or another Linux environment. +Keeping extra personal packages installed +========================================= -If you don't have write permissions to openedx/edx-platform, you'll need to run these workflows on a fork. +The old ``requirements/edx/private.in``/``private.txt`` mechanism (an +uncommitted, git-ignored pair of files letting you keep extra personal +packages installed alongside the official requirements, surviving a +``pip-sync``) has no direct successor file, but the same need is covered by +``uv sync``'s ``--inexact`` flag: ``uv pip install `` your extra +tool once, then pass ``--inexact`` on subsequent ``uv sync`` calls (e.g. +``uv sync --inexact``) to keep it from being removed as an "extraneous" +package. Workflows and Makefile targets ****************************** @@ -22,7 +47,9 @@ Workflows and Makefile targets Add a dependency ================ -To add a Python dependency, specify it in the appropriate ``requirements/edx/*.in`` file, push that up to a branch, and then use the `compile-python-requirements.yml workflow `_ to run ``make compile-requirements`` against your branch. This will ensure the lockfiles are updated with any transitive dependencies and will ping you on a PR for updating your branch. +To add a Python dependency, add it to ``[project.dependencies]`` (or the +appropriate ``[dependency-groups]`` entry) in ``pyproject.toml``, push that up +to a branch, and then use the `compile-python-requirements.yml workflow `_ to run ``make compile-requirements`` against your branch. This will ensure ``uv.lock`` is updated with any transitive dependencies and will ping you on a PR for updating your branch. Upgrade just one dependency =========================== @@ -31,17 +58,17 @@ Want to upgrade just *one* dependency without pulling in other upgrades? You can Or, if you need to do it locally, you can use the ``upgrade-package`` make target directly. For example, you could run ``make upgrade-package package=ecommerce``. -If your dependency is pinned in constraints.txt, you'll need to enter an explicit version number in the appropriate field when running the workflow; this will include an update to the constraint file in the resulting PR. +If your dependency is pinned in ``[tool.edx_lint].uv_constraints`` (in ``pyproject.toml``), you'll need to enter an explicit version number in the appropriate field when running the workflow; this will include an update to that constraint in the resulting PR. Downgrade a dependency ====================== If you instead need to surgically *downgrade* a dependency: -1. Add an exact-match or max-version constraint to ``constraints.txt`` with a comment explaining why (and ideally a ticket or issue link). Here's what it might look like:: +1. Add an exact-match or max-version constraint to ``[tool.edx_lint].uv_constraints`` in ``pyproject.toml`` with a comment explaining why (and ideally a ticket or issue link). Here's what it might look like:: # frobulator 2.x has breaking API changes; see https://github.com/openedx/edx-platform/issue/1234567 for fixing it - frobulator<2.0.0 + "frobulator<2.0.0", 2. After pushing that up to a branch, use the `compile-python-requirements.yml workflow `_ to run ``make compile-requirements`` against your branch. @@ -78,7 +105,7 @@ Luckily, we have simple runbooks for upgrading or downgrading a single package, Is there an unpinned git dependency? ==================================== -If the diff relates to a dependency that is installed from git rather than from PyPI (such as being a transitive dependency of anything in github.in), check whether any of the dependencies in github.in has failed to pin a specific commit. We want to have as few of these dependencies as possible, as they're a maintenance and performance problem, and there are important instructions at the top of that file for how to manage them. +If the diff relates to a dependency that is installed from git rather than from PyPI, check ``[project.dependencies]`` in ``pyproject.toml`` for a direct reference (``name @ git+https://...@TAG-OR-SHA``) that has failed to pin a specific commit. We want to have as few of these dependencies as possible, as they're a maintenance and performance problem. Help, I didn't change any dependencies, and this is still failing! ================================================================== diff --git a/requirements/common_constraints.txt b/requirements/common_constraints.txt deleted file mode 100644 index 45750edf0b20..000000000000 --- a/requirements/common_constraints.txt +++ /dev/null @@ -1,30 +0,0 @@ -# This is a temporary solution to override the real common_constraints.txt -# In edx-lint, until the pyjwt constraint in edx-lint has been removed. -# See BOM-2721 for more details. -# Below is the copied and edited version of common_constraints -# A central location for most common version constraints -# (across edx repos) for pip-installation. -# -# Similar to other constraint files this file doesn't install any packages. -# It specifies version constraints that will be applied if a package is needed. -# When pinning something here, please provide an explanation of why it is a good -# idea to pin this package across all edx repos, Ideally, link to other information -# that will help people in the future to remove the pin when possible. -# Writing an issue against the offending project and linking to it here is good. -# -# Note: Changes to this file will automatically be used by other repos, referencing -# this file from Github directly. It does not require packaging in edx-lint. - -# using LTS django version -Django<6.0 - -# elasticsearch>=7.14.0 includes breaking changes in it which caused issues in discovery upgrade process. -# elastic search changelog: https://www.elastic.co/guide/en/enterprise-search/master/release-notes-7.14.0.html -# See https://github.com/openedx/edx-platform/issues/35126 for more info -elasticsearch<7.14.0 - -# 2026-07-06: Constrain social-auth-* packages to the latest version compatible with -# edx-drf-extensions<=10.6.0. (Newer versions require a POST instead of a GET.) -# Tracking issue: https://github.com/openedx/edx-drf-extensions/issues/561 -social-auth-app-django<6.0.0 -social-auth-core<5.0.0 diff --git a/requirements/constraints.txt b/requirements/constraints.txt deleted file mode 100644 index 2e89b7ae1cc2..000000000000 --- a/requirements/constraints.txt +++ /dev/null @@ -1,139 +0,0 @@ -# Version constraints for pip-installation. -# -# This file doesn't install any packages. It specifies version constraints -# that will be applied if a package is needed. -# -# When pinning something here, please provide an explanation of why. Ideally, -# link to other information that will help people in the future to remove the -# pin when possible. Writing an issue against the offending project and -# linking to it here is good. -# For further details on how to properly write constraints here please consult -# https://openedx.atlassian.net/wiki/spaces/COMM/pages/4400250883/Adding+pinned+dependencies+in+constraint+file - -# This file contains all common constraints for edx-repos --c common_constraints.txt - -# Date: 2025-10-07 -# Stay on LTS version, remove once this is added to common constraint -Django<6.0 - -# Date: 2026-01-13 -# We would normally pin celery to <6.0.0 to avoid auto-updating across a major -# version boundary without more thorough testing. The reason it's currently also -# pinned to !=5.6.1 is because of a celery bug related to the eta and countdown -# parameters. This bug caused operational issues in MIT's deployment. -# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35280 -celery>=5.2.2,!=5.6.1,<6.0.0 - -# Date: 2020-02-10 -# django-oauth-toolkit version >=2.0.0 has breaking changes. More details -# mentioned on this issue https://github.com/openedx/edx-platform/issues/32884 -# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35277 -django-oauth-toolkit==1.7.1 - -# Date: 2024-07-19 -# Generally speaking, the major version of django-stubs must either match the major version -# of django, or exceed it by 1. So, we will need to perpetually constrain django-stubs and -# update it as we perform django upgrades. For more details, see: -# https://github.com/typeddjango/django-stubs?tab=readme-ov-file#version-compatibility -# including the note on "Partial Support". -# Issue: https://github.com/openedx/edx-platform/issues/35275 -django-stubs<6 - -# Date: 2019-08-16 -# The team that owns this package will manually bump this package rather than having it pulled in automatically. -# This is to allow them to better control its deployment and to do it in a process that works better -# for them. -edx-enterprise==8.9.2 - -# Date: 2023-07-26 -# Our legacy Sass code is incompatible with anything except this ancient libsass version. -# Here is a ticket to upgrade, but it's of debatable importance given that we are rapidly moving -# away from legacy LMS/CMS frontends: -# https://github.com/openedx/edx-platform/issues/31616 -libsass==0.10.0 - -# Date: 2024-07-16 -# We need to upgrade the version of elasticsearch to at least 7.15 before we can upgrade to Numpy 2.0.0 -# Otherwise we see a failure while running the following command: -# export DJANGO_SETTINGS_MODULE=cms.envs.test; python manage.py cms check_reserved_keywords --override_file db_keyword_overrides.yml --report_path reports/reserved_keywords --report_file cms_reserved_keyword_report.csv -# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35126 -numpy<2.0.0 - -# Date: 2023-09-18 -# Library is still in active development. Major versions require review -# from openedx-maintainers. Minor and patch versions can roll out automatically. -# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35269 -openedx-core<2 - -# Date: 2024-04-26 -# path==16.12.0 breaks the unit test collections check -# needs to be investigated and fixed separately -# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35267 -path<16.12.0 - -# Date: 2020-04-08 -# Adding pin to avoid any major upgrade -# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35265 -pymongo<4.4.1 - -# Date: 2024-08-06 -# social-auth-app-django 5.4.2 introduces a new migration that will not play nicely with large installations. This will touch -# user tables, which are quite large, especially on instances like edx.org. -# We are pinning this until after all the smaller migrations get handled and then we can migrate this all at once. -# Issue for unpinning: https://github.com/openedx/edx-platform/issues/37639 -social-auth-app-django<=5.4.1 - -# # Date: 2024-10-14 -# # The edx-enterprise is currently using edx-rest-api-client==5.7.1, which needs to be updated first. -# edx-rest-api-client==5.7.1 - -# Date 2025-01-08 -# elasticsearch==7.13.x is downgrading urllib3 from 2.2.3 to 1.26.20 -# https://github.com/elastic/elasticsearch-py/blob/v7.13.4/setup.py#L42 -# We are pinning this until we can upgrade to a version of elasticsearch that uses a more recent version of urllib3. -# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35126 -elasticsearch==7.9.1 - -# Date 2025-05-09 -# lxml and xmlsec need to be constrained because the latest version builds against a newer -# version of libxml2 than what we're running with. This leads to a version mismatch error -# at runtime. You can re-produce it by running any test. -# If lxml is pinned in the future and you see this error, it may be that the system libxml2 -# is now shipping the correct version and we can un-pin this. -# Issue: https://github.com/openedx/edx-platform/issues/36695 -lxml==5.3.2 -xmlsec==1.3.14 - -# Date 2025-08-12 -# The newest version of the debug toolbar has a bug in it -# https://github.com/django-commons/django-debug-toolbar/issues/2172 -# Pin this back to the previous version until that bug is fixed. -django-debug-toolbar<6.0.0 - -# Date 2026-01-13 -# Sphinx-autoapi changed the version of astroid it needs -# but the newer version is not compatible with the current pylint version -# which wants a newer version of astroid. This can be removed once we're -# building requirements with Python 3.12 -# https://github.com/openedx/edx-platform/issues/37880 -sphinx-autoapi<3.6.1 - -# Date 2026-03-02 -# setuptools 82.0.0 removed pkg_resources from its distribution, but fs (pyfilesystem2) -# still uses pkg_resources for namespace package declarations. This constraint can be -# removed once pyfilesystem2 drops its pkg_resources usage. -# https://github.com/PyFilesystem/pyfilesystem2/issues/577 -# Issue for unpinning: https://github.com/openedx/openedx-platform/issues/38068 -setuptools<82 - -# Date 2026-03-02 -# The latest version of pylint pins back astroid to an older version. -# This holdback is not caught in the docs requirements file and since both the docs -# and testing file are required in the development.in file, we fail to compile -# development.txt because of conflicting dependencies. -# -# Holding astroid back until pylint releases a new version that works with the latest -# version of astroid. -# https://github.com/openedx/openedx-platform/issues/38066 -astroid==4.0.4 diff --git a/requirements/edx-sandbox/README.rst b/requirements/edx-sandbox/README.rst index 96c90ee911f4..9acbe9ebb823 100644 --- a/requirements/edx-sandbox/README.rst +++ b/requirements/edx-sandbox/README.rst @@ -10,22 +10,26 @@ within ``