diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9ff8497..f0598b2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -149,13 +149,27 @@ jobs: name: remove-old-wheels steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + # Dogfood the remove-wheels action on PRs here. If we keep zero uploads, + # it marks every version of the package for removal. + - name: Remove test package uploads via the action + uses: ./remove-wheels + with: + n_latest_uploads_to_keep: 0 + anaconda_nightly_upload_organization: scientific-python-nightly-wheels + anaconda_nightly_token: ${{ secrets.ANACONDA_TOKEN }} + package_name: test-package + + # The action removes per version, so delete the (now empty) package + # registration as well - name: Set up pixi uses: prefix-dev/setup-pixi@d3f436a425481402e6a95a1d1fc10331c708cd9e # v0.10.2 with: - run-install: false + locked: true - - name: Remove test package upload - shell: pixi exec --spec anaconda-client -- bash -e {0} + - name: Remove the empty test package + shell: pixi run --manifest-path pixi.toml bash -e {0} run: | anaconda --token ${{ secrets.ANACONDA_TOKEN }} remove \ --force \ diff --git a/.github/workflows/remove-wheels.yml b/.github/workflows/remove-wheels.yml index a3b3724..79992f0 100644 --- a/.github/workflows/remove-wheels.yml +++ b/.github/workflows/remove-wheels.yml @@ -3,25 +3,19 @@ name: Remove old wheels on: # Run daily at 1:23 UTC schedule: - - cron: '23 1 * * *' + - cron: '23 1 * * *' workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true -# Needed for micromamba pickup -defaults: - run: - shell: bash -l {0} - env: N_LATEST_UPLOADS: 5 ANACONDA_USER: "scientific-python-nightly-wheels" jobs: - remove: - + get-pkgs: runs-on: ubuntu-latest if: github.repository_owner == 'scientific-python' # Set required workflow secrets in the environment for additional security @@ -29,86 +23,50 @@ jobs: environment: name: remove-old-wheels + outputs: + pkgnames: ${{ steps.set-outputs.outputs.pkgnames }} + steps: - - name: Install micromamba and anaconda-client - uses: mamba-org/setup-micromamba@f457c30a868e4760d3a6fcea5f25dc655b8edf39 # v3.2.1 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Set up pixi + uses: prefix-dev/setup-pixi@d3f436a425481402e6a95a1d1fc10331c708cd9e # v0.10.2 with: - environment-name: remove-wheels - create-args: >- - anaconda-client==1.14.0 - curl - jq - - - name: Show environment - run: env - - - name: Show CLI API info - run: | - anaconda show --help - echo "" - anaconda remove --help + locked: true - - name: Query package index for packages + - name: Generate list of package names + id: set-outputs + shell: pixi run --manifest-path pixi.toml bash -e {0} run: | - curl https://raw.githubusercontent.com/scientific-python/upload-nightly-action/main/packages-ignore-from-cleanup.txt --output packages-ignore-from-cleanup.txt - anaconda show "${ANACONDA_USER}" &> >(grep "${ANACONDA_USER}/") | \ + pkgs_json=$(anaconda show "${ANACONDA_USER}" 2>&1 | grep "${ANACONDA_USER}/" | \ awk '{print $1}' | \ sed 's|.*/||g' | \ - grep -vf packages-ignore-from-cleanup.txt > package-names.txt - - - name: Remove old uploads to save space - run: | - # Remove all _but_ the last ${N_LATEST_UPLOADS} package versions and - # remove all package versions older than 30 days. - - if [ -s package-names.txt ]; then - threshold_date="$(date +%F -d '30 days ago')" - - # Remember can't quote subshell as need to split on (space seperated) token - for package_name in $(cat package-names.txt); do - - echo -e "\n# package: ${package_name}" + grep -vf packages-ignore-from-cleanup.txt | \ + python -c 'import sys, json; print(json.dumps([line.strip() for line in sys.stdin if line.strip()]))') + echo "pkgnames=${pkgs_json}" >> "$GITHUB_OUTPUT" - curl --silent https://api.anaconda.org/package/"${ANACONDA_USER}/${package_name}" | \ - jq -r '.releases[].version' > package-versions.txt - head --lines "-${N_LATEST_UPLOADS}" package-versions.txt > remove-package-versions.txt - - for package_version in $(cat package-versions.txt); do - # c.f. https://github.com/Anaconda-Platform/anaconda-client/issues/682#issuecomment-1677283067 - upload_date=$(curl --silent https://api.anaconda.org/release/"${ANACONDA_USER}/${package_name}/${package_version}" | \ - jq -r '.distributions[].upload_time' | \ - sort | \ - tail --lines 1 | \ - awk '{print $1}') - - # check upload_date is YYYY-MM-DD formatted - # c.f. https://github.com/scientific-python/upload-nightly-action/issues/73 - if [[ "${upload_date}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]; then - if [[ "${upload_date}" < "${threshold_date}" ]]; then - echo "# ${ANACONDA_USER}/${package_name}/${package_version} last uploaded on ${upload_date}" - echo "${package_version}" >> remove-package-versions.txt - fi - else - echo "# ERROR: ${ANACONDA_USER}/${package_name}/${package_version} upload date ${upload_date} is not YYYY-MM-DD." - fi - - done + remove: + runs-on: ubuntu-latest + if: github.repository_owner == 'scientific-python' + # Set required workflow secrets in the environment for additional security + # https://github.com/scientific-python/upload-nightly-action/settings/environments + environment: + name: remove-old-wheels - if [ -s remove-package-versions.txt ]; then - # Guard against duplicate entries from packages over - # count and time thresholds - sort --output remove-package-versions.txt --unique remove-package-versions.txt + needs: [get-pkgs] - for package_version in $(cat remove-package-versions.txt); do - echo "# Removing ${ANACONDA_USER}/${package_name}/${package_version}" - anaconda --token ${{ secrets.ANACONDA_TOKEN }} remove \ - --force \ - "${ANACONDA_USER}/${package_name}/${package_version}" - done - fi + strategy: + matrix: + pkgname: ${{fromJSON(needs.get-pkgs.outputs.pkgnames)}} - done - fi + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Remove old wheels + uses: ./remove-wheels + with: + n_latest_uploads_to_keep: ${{ env.N_LATEST_UPLOADS }} + anaconda_nightly_upload_organization: ${{ env.ANACONDA_USER }} + anaconda_nightly_token: ${{ secrets.ANACONDA_TOKEN }} + package_name: ${{ matrix.pkgname }} report-failure: needs: [remove] diff --git a/README.md b/README.md index 3695f0e..2e6fb74 100644 --- a/README.md +++ b/README.md @@ -18,10 +18,47 @@ jobs: anaconda_nightly_upload_token: ${{secrets.UPLOAD_TOKEN}} ``` -Note that we recommend pinning the action against a specific SHA +> [!IMPORTANT] +> Note that we recommend pinning the action against a specific SHA (rather than a tag), to guard against the unlikely event of upstream being compromised. +## Removing old nightly builds + +Note that ``scientific-python-nightly-wheels``, specifically, already removes +old artifacts daily. The `remove-wheels` action is therefore intended for use with other channels. + +This repository also ships with an action to ease removals of older nightly wheels from a channel. +Please note that the default configuration below will remove all but the `n_latest_uploads_to_keep` +latest uploads from the channel. This is useful to avoid hosting outdated development +versions, as well as to clean up space. + +If you do not wish to have this automated cleanup, please [open an issue](https://github.com/scientific-python/upload-nightly-action/) +to be added to the list of packages exempt from it. The current ones are named in +[`packages-ignore-from-cleanup.txt`](packages-ignore-from-cleanup.txt). + +Please refer to the [artifact cleanup policy][] for more information. + +To use this functionality, add the following snippet to your workflow: + +```yml +jobs: + steps: + ... + - name: Remove old wheels + uses: scientific-python/upload-nightly-action/remove-wheels@main # pin to a SHA in practice + with: + n_latest_uploads_to_keep: 5 + anaconda_nightly_upload_organization: "your-organization" + anaconda_nightly_token: ${{secrets.ANACONDA_TOKEN}} + package_name: "your-package" +``` + +Please note that the `anaconda_nightly_token` secret must have the necessary permissions to +remove artifacts from the channel. A token scoped to a particular package will delete only +the artifacts for that package. If you need to delete artifacts for multiple packages, run +the action once per package (such as in a matrix). + ## Updating the action You can [use Dependabot to keep the GitHub Action up to date][], @@ -169,3 +206,4 @@ dependencies: [PyPI]: https://pypi.org/ [scientific-python nightly channel]: https://anaconda.org/scientific-python-nightly-wheels [SPEC4 — Using and Creating Nightly Wheels]: https://scientific-python.org/specs/spec-0004/ +[artifact cleanup policy]: #artifact-cleanup-policy-at-the-scientific-python-nightly-wheels-channel diff --git a/remove-wheels/action.yml b/remove-wheels/action.yml new file mode 100644 index 0000000..21bbf0b --- /dev/null +++ b/remove-wheels/action.yml @@ -0,0 +1,43 @@ +name: Remove Old Wheels +description: A GitHub Action to remove old wheels +author: "Scientific-Python" + +inputs: + n_latest_uploads_to_keep: + description: 'The number of previous wheel uploads to keep' + required: false + default: '5' + anaconda_nightly_upload_organization: + description: 'Anaconda Cloud organisation name to remove the wheels from' + required: false + default: scientific-python-nightly-wheels + anaconda_nightly_token: + description: 'Anaconda Cloud API token to authenticate with' + required: true + package_name: + description: 'Name of the package to remove old uploads for' + required: true + +runs: + using: "composite" + steps: + - name: Set up pixi + uses: prefix-dev/setup-pixi@d3f436a425481402e6a95a1d1fc10331c708cd9e # v0.10.2 + with: + locked: true + cache: true + cache-write: ${{ github.event_name == 'push' && github.ref_name == 'main' }} + # Avoid post cleanup errors if action run multiple times + post-cleanup: false + # Action consumers should load the lock file from the action repo + manifest-path: ${{ github.action_path }}/../pixi.toml + + - name: Remove old wheels + shell: bash + env: + INPUT_N_LATEST_UPLOADS: ${{ inputs.n_latest_uploads_to_keep }} + INPUT_ANACONDA_USER: ${{ inputs.anaconda_nightly_upload_organization }} + INPUT_ANACONDA_TOKEN: ${{ inputs.anaconda_nightly_token }} + INPUT_PACKAGE_NAME: ${{ inputs.package_name }} + run: | + pixi run --manifest-path ${{ github.action_path }}/../pixi.toml python ${{ github.action_path }}/remove_wheels.py diff --git a/remove-wheels/remove_wheels.py b/remove-wheels/remove_wheels.py new file mode 100644 index 0000000..109fc09 --- /dev/null +++ b/remove-wheels/remove_wheels.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python +""" +Remove old nightly wheel uploads from an Anaconda.org channel. + +For a single package, it keeps the newest INPUT_N_LATEST_UPLOADS versions and additionally removes any version whose most recent upload is older than 30 days. + +The inputs are read from the environment (see remove-wheels/action.yml): + +- INPUT_ANACONDA_USER - the Anaconda.org organisation/user to clean up +- INPUT_ANACONDA_TOKEN - a token with permission to remove artifacts +- INPUT_N_LATEST_UPLOADS - the number of newest versions to keep +- INPUT_PACKAGE_NAME - the package to prune +""" + +import json +import os +import subprocess +import sys +import urllib.request +from datetime import datetime, timedelta, timezone + +API = "https://api.anaconda.org" + + +def get_json(url): + with urllib.request.urlopen(url, timeout=30) as response: + return json.load(response) + + +def latest_upload_date(distributions): + """ + Return the date (YYYY-MM-DD) of the most recent distribution upload + or an empty string if there are no upload times present. c.f. + https://github.com/Anaconda-Platform/anaconda-client/issues/682#issuecomment-1677283067 + """ + upload_times = [ + "null" if d.get("upload_time") is None else str(d["upload_time"]) + for d in distributions + ] + if not upload_times: + return "" + latest = max(upload_times) + fields = latest.split() + return fields[0] if fields else "" + + +def main(): + anaconda_user = os.environ.get("INPUT_ANACONDA_USER", "") + anaconda_token = os.environ.get("INPUT_ANACONDA_TOKEN", "") + n_latest_uploads = os.environ.get("INPUT_N_LATEST_UPLOADS", "") + package_name = os.environ.get("INPUT_PACKAGE_NAME", "") + + if not anaconda_token: + print("ANACONDA_TOKEN is empty, exiting...") + sys.exit(1) + if not n_latest_uploads: + print("N_LATEST_UPLOADS is empty, exiting...") + sys.exit(1) + if not package_name: + print("PACKAGE_NAME is empty, exiting...") + sys.exit(1) + + n_latest = int(n_latest_uploads) + + # 1. Remove all but the last ${n_latest} package versions. + # 2. Remove all package versions older than 30 days. + + print(f"\n# package: {package_name}") + + threshold_date = datetime.now(timezone.utc).date() - timedelta(days=30) + + # The API lists releases oldest-first, so keep the newest ${n_latest} + # uploads by marking all but the last ${n_latest} versions for removal. + releases = get_json(f"{API}/package/{anaconda_user}/{package_name}").get( + "releases", [] + ) + versions = [release["version"] for release in releases] + if n_latest == 0: + remove_versions = set(versions) + elif n_latest < len(versions): + remove_versions = set(versions[:-n_latest]) + else: + remove_versions = set() + + for version in versions: + # c.f. https://github.com/Anaconda-Platform/anaconda-client/issues/682#issuecomment-1677283067 + distributions = get_json( + f"{API}/release/{anaconda_user}/{package_name}/{version}" + ).get("distributions", []) + upload_date = latest_upload_date(distributions) + + # check that the upload_date is YYYY-MM-DD formatted + # c.f. https://github.com/scientific-python/upload-nightly-action/issues/73 + try: + parsed_date = datetime.strptime(upload_date, "%Y-%m-%d").date() + except ValueError: + print( + f"# ERROR: {anaconda_user}/{package_name}/{version} upload date {upload_date} is not YYYY-MM-DD." + ) + else: + if parsed_date < threshold_date: + print( + f"# {anaconda_user}/{package_name}/{version} last uploaded on {upload_date}" + ) + remove_versions.add(version) + + for version in sorted(remove_versions): + print(f"# Removing {anaconda_user}/{package_name}/{version}") + subprocess.check_call( + [ + "anaconda", + "--token", + anaconda_token, + "remove", + "--force", + f"{anaconda_user}/{package_name}/{version}", + ] + ) + + print( + f"Finished removing old wheels except the last {n_latest_uploads} uploads " + f"for {anaconda_user}/{package_name}." + ) + + +if __name__ == "__main__": + main()