Skip to content
Merged

Updates #8773

Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .github/workflows/currency-build.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,13 @@ jobs:
echo "__EOF__"
} >> $GITHUB_OUTPUT

- name: Update wheel mapping in COS
env:
GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }}
run: |
source variable.sh
python3 ./gha-script/upload-scripts/update_pkg_wheel_name_mapping.py

- name: Create scanner-env.sh
run: |
mkdir package-cache
Expand Down
6 changes: 3 additions & 3 deletions gha-script/build_wheels.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import docker
import json

def trigger_build_wheel(wrapper_file, python_version, image_name, file_name, version, post_process_file):
def trigger_build_wheel(wrapper_file, python_version, image_name, file_name, version, post_process_file, check_wheel_version_file):
# Docker client setup
client = docker.DockerClient(base_url='unix://var/run/docker.sock')

Expand Down Expand Up @@ -41,7 +41,7 @@ def trigger_build_wheel(wrapper_file, python_version, image_name, file_name, ver
command = [
"bash",
"-c",
f"{setup}cd /home/tester/ && {run_script} {python_version} {file_name} {version} {post_process_file}"
f"{setup}cd /home/tester/ && {run_script} {python_version} {file_name} {version} {post_process_file} {check_wheel_version_file}"
]

# Run container
Expand Down Expand Up @@ -99,4 +99,4 @@ def trigger_build_wheel(wrapper_file, python_version, image_name, file_name, ver

if __name__=="__main__":
print("Inside python program")
trigger_build_wheel(sys.argv[1],sys.argv[2],sys.argv[3],sys.argv[4],sys.argv[5],sys.argv[6])
trigger_build_wheel(sys.argv[1],sys.argv[2],sys.argv[3],sys.argv[4],sys.argv[5],sys.argv[6],sys.argv[7])
5 changes: 4 additions & 1 deletion gha-script/build_wheels.sh
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,10 @@ WHEEL_SCRIPT=gha-script/create_wheel_wrapper.sh
# path to post_process_wheel script (suffix addition, license addition, metadata addition)
POST_PROCESS_SCRIPT_PATH=gha-script/post_process_wheel.py

python3 gha-script/build_wheels.py "$WHEEL_SCRIPT" "$PYTHON_VERSION" "$docker_image" "$PKG_DIR_PATH$BUILD_SCRIPT" "$VERSION" "$POST_PROCESS_SCRIPT_PATH" 2>&1 | tee wheel_build_log
# path to check_wheel_version script (version match + clean version validation)
CHECK_WHEEL_VERSION_SCRIPT=gha-script/check_wheel_version.py

python3 gha-script/build_wheels.py "$WHEEL_SCRIPT" "$PYTHON_VERSION" "$docker_image" "$PKG_DIR_PATH$BUILD_SCRIPT" "$VERSION" "$POST_PROCESS_SCRIPT_PATH" "$CHECK_WHEEL_VERSION_SCRIPT" 2>&1 | tee wheel_build_log
wheel_status=${PIPESTATUS[0]}

log_size=$(stat -c %s wheel_build_log)
Expand Down
169 changes: 169 additions & 0 deletions gha-script/check_wheel_version.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
#!/usr/bin/env python3

"""
check_wheel_version.py
Validates a built wheel's version before post-processing.

Two checks:
1. VERSION MATCH — wheel base version must correspond to GITHUB_PACKAGE_VERSION.
2. CLEAN VERSION — wheel must not contain a git-injected local identifier.

Usage:
python check_wheel_version.py <wheel_filename> <github_package_version>

Requirements: pip install packaging

Exit codes:
0 both checks passed
1 one or both checks failed
"""

import re
import sys

from packaging.specifiers import SpecifierSet
from packaging.utils import InvalidWheelFilename, parse_wheel_filename
from packaging.version import InvalidVersion, Version

# ---------------------------------------------------------------------------
# Compiled patterns
# ---------------------------------------------------------------------------

# Detects a git-hash local segment (6+ hex chars, optional leading 'g').
# Matches: g1892993bc 56be3b5e g6909efdd6
# No match: cpu cuda118 rocm6 cpu.ppc64le
_GIT_HASH_RE = re.compile(r"(?:^|\.)g?[0-9a-f]{6,}(?:\.|$)", re.IGNORECASE)

# Strips operator separators — e.g. pkg==1.2.3 pkg@1.2.3
_RE_OPERATOR = re.compile(r"^[^@=\s]+(?:@|==)(\S+)")

# Strips common textual version prefixes — e.g. v1.2 release-v1.2 rel_1_2 n7 (ffmpeg)
# To add a new prefix pattern, extend this alternation.
_RE_PREFIX = re.compile(r"^(?:release-v|release[-_]|rel_|version_|v|n(?=\d))", re.IGNORECASE)

# Strips a leading package-name segment — e.g. azure-mgmt-batch_18.0.0 jaxlib-v0.4.7 cares-1_19_1
# To add a new package-name pattern, extend this regex.
_RE_PKG_PREFIX = re.compile(r"^[a-zA-Z][\w-]*?[-_]v?(\d[^\n]*)")

# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------

def _is_dirty_local(local: str) -> bool:
"""Return True if *local* contains a setuptools-scm git-hash segment."""
return bool(_GIT_HASH_RE.search(local))


def _normalize_version(github_package_version: str) -> str:
"""Return a bare version number from any common tagging convention.

Strips (in order):
1. Operator separators pkg==1.2.3 pkg@1.2.3
2. Textual prefixes v release-v rel_ version_ n (ffmpeg)
3. Package-name prefix azure-mgmt-batch_18.0.0 jaxlib-v0.4.7
4. Underscore → dot 1_4_39 → 1.4.39

Returns the input unchanged when no rule matches — the caller can detect
this (norm == github_package_version) and hint that a new regex rule may
be needed.
"""
s = github_package_version.strip()
m = _RE_OPERATOR.match(s)
if m:
s = m.group(1)
s = _RE_PREFIX.sub("", s)
m = _RE_PKG_PREFIX.match(s)
if m:
s = m.group(1)
return s.replace("_", ".")


def version_matches(wheel_ver: Version, norm: str) -> bool:
"""Return True if *wheel_ver* corresponds to the already-normalised *norm*.

Tier 1 — SpecifierSet: handles PEP 440 versions including dev/rc/post/local
on the correct base (e.g. 1.14.0.dev0 matches norm '1.14.0').

Tier 2 — Exact equality fallback: for numeric strings that are not valid
PEP 440 (e.g. date-based '20220401').
"""
# Tier 1
try:
spec = SpecifierSet(f"=={Version(norm).base_version}.*", prereleases=True)
return str(wheel_ver) in spec
except InvalidVersion:
pass

# Tier 2
base = wheel_ver.base_version
return base == norm or base.replace(".", "_") == norm.replace(".", "_")

# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------

def main() -> None:
if len(sys.argv) != 3:
print("Usage: check_wheel_version.py <wheel_filename> <github_package_version>")
sys.exit(1)

wheel_filename = sys.argv[1]
github_package_version = sys.argv[2].strip()

try:
_, ver, _, _ = parse_wheel_filename(wheel_filename)
except InvalidWheelFilename as exc:
print(f"ERROR: Cannot parse wheel filename: {exc}")
sys.exit(1)

sep = "=" * 60
print(f"\n{sep}")
print("Wheel Version Validation")
print(sep)
print(f" Wheel : {wheel_filename}")
print(f" Expected : {github_package_version}")
print(f" Got : {ver} (base: {ver.base_version})")
print(sep)

failed = False
norm = _normalize_version(github_package_version)

# Check 1: version match
if version_matches(ver, norm):
print(f" PASS [1] Version match ({ver.base_version} matches {github_package_version})")
else:
print(" FAIL [1] Version mismatch!")
print(f" Expected : {github_package_version}")
print(f" Normalised: {norm}")
print(f" Got : {ver.base_version}")
if norm == github_package_version:
# _normalize_version made no change — likely an unrecognised tag format.
print(f" _normalize_version did not recognise the format of '{github_package_version}'.")
print(" If this is a new tag convention, add a rule to")
print(" _RE_PREFIX or _RE_PKG_PREFIX in check_wheel_version.py.")
else:
print(" The build script produced a different version than requested.")
print(f" Ensure it checks out / pins {github_package_version}")
failed = True

# Check 2: no git-injected local identifier.
# Intentional variant labels (+cpu, +cuda118, +rocm6) are allowed.
if ver.local is None:
print(" PASS [2] Clean version (no local identifier)")
elif _is_dirty_local(str(ver.local)):
print(f" FAIL [2] Git-injected local identifier: +{ver.local}")
print(f" Got : {ver}")
print(f" Expected : {ver.base_version} (no git hash)")
print(" setuptools-scm injected a commit hash because the")
print(" source tree was not at a clean tagged commit.")
print(" Ensure the build script checks out the exact release tag.")
failed = True
else:
print(f" PASS [2] Intentional local variant (+{ver.local})")

print()
sys.exit(1 if failed else 0)


if __name__ == "__main__":
main()
18 changes: 18 additions & 0 deletions gha-script/create_wheel_wrapper.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ PYTHON_VERSION=$1
BUILD_SCRIPT_PATH=${2:-""}
EXTRA_ARGS=${3:-""}
POST_PROCESS_SCRIPT_PATH=${4:-"post_process_wheel.py"}
CHECK_WHEEL_VERSION_SCRIPT=${5:-"check_wheel_version.py"}
CURRENT_DIR=$(pwd)

# install git - required by generate_sha() for all Python versions and UBI versions
Expand Down Expand Up @@ -360,6 +361,23 @@ fi
cd "$CURRENT_DIR"
wheel_final=(*.whl)

# ---------------------------------------------------------------------------
# check_wheel_version: validates the built wheel version against
# GITHUB_PACKAGE_VERSION before any post-processing takes place.
#
# Two checks (both must pass):
# 1. VERSION MATCH — wheel base version must correspond to GITHUB_PACKAGE_VERSION
# Catches: build script checked out the wrong tag, e.g. requested
# v1.14.0 but the package self-reported 1.15.0.dev0.
# 2. CLEAN VERSION — wheel must NOT have a PEP 440 local identifier (+...)
# Catches: setuptools-scm / build system injected a git hash or build
# date because the source tree was not at a clean tagged commit,
# e.g. 0.0.30+56be3b5e.d20250607 or 0.1.3.dev0+ga7d8bbf.d20250607
# ---------------------------------------------------------------------------
if [ -n "$EXTRA_ARGS" ]; then
python "$CHECK_WHEEL_VERSION_SCRIPT" "${wheel_final[0]}" "$EXTRA_ARGS"
fi

# ---------------------------------------------------------------------------
# run_cve_scan: runs generalized_wheel_scanner.py on the built wheel.
#
Expand Down
10 changes: 10 additions & 0 deletions gha-script/read_buildinfo.sh
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,10 @@ config_file='build_info.json'
if [ -f $config_file ]; then
jsonObj=$config_file
build_script=$(jq .build_script $jsonObj)
wheel_name=""
if $(jq 'has("wheel_name")' $jsonObj); then
wheel_name=$(jq -r .wheel_name $jsonObj)
fi

if $(jq 'has("use_non_root_user")' $jsonObj); then
nonRootBuild=$(jq .use_non_root_user $jsonObj)
Expand Down Expand Up @@ -61,6 +65,11 @@ if [ -f $config_file ]; then
build_script=$(jq -c "$version_block.build_script" $config_file)
fi

# version-specific wheel_name
if [[ $(jq -r "$version_block.wheel_name" $config_file) != "null" ]]; then
wheel_name=$(jq -r "$version_block.wheel_name" $config_file)
fi

# version-specific base_docker_image
if [[ $(jq -r "$version_block.base_docker_image" $config_file) != "null" ]]; then
basename=$(jq -r "$version_block.base_docker_image" $config_file)
Expand Down Expand Up @@ -299,6 +308,7 @@ echo "export NON_ROOT_BUILD=\"$nonRootBuild\"" >> $CUR_DIR/var
echo "export TESTED_ON=\"$tested_on\"" >> $CUR_DIR/variable.sh
echo "export AUDITWHEEL_EXCLUDE=\"$AUDITWHEEL_EXCLUDE\"" >> $CUR_DIR/variable.sh
echo "export SKIP_PYTHON_VERSIONS=\"$SKIP_PYTHON_VERSIONS\"" >> $CUR_DIR/variable.sh
echo "export WHEEL_NAME=\"$wheel_name\"" >> $CUR_DIR/variable.sh
# Full array - kept for any downstream consumer that still needs it
echo "export BUILD_SCRIPTS_JSON='$BUILD_SCRIPTS_JSON'" >> $CUR_DIR/variable.sh
# Per-UBI-major named exports - empty string when that UBI version has no script
Expand Down
Loading
Loading