diff --git a/.github/actions/setup-buildx/action.yml b/.github/actions/setup-buildx/action.yml new file mode 100644 index 00000000..0d360cb5 --- /dev/null +++ b/.github/actions/setup-buildx/action.yml @@ -0,0 +1,32 @@ +--- +name: Set up Docker Buildx +description: Boot a BuildKit builder, retrying the BuildKit image pull so a transient Docker Hub failure does not fail the job. + +inputs: + buildkit-image: + description: BuildKit image used by the docker-container driver. + required: false + default: moby/buildkit:buildx-stable-1 + +runs: + using: composite + steps: + - name: Pull BuildKit image + shell: bash + env: + BUILDKIT_IMAGE: ${{ inputs.buildkit-image }} + run: | + for attempt in 1 2 3 4 5; do + if docker pull "$BUILDKIT_IMAGE"; then + exit 0 + fi + echo "::warning::pull of $BUILDKIT_IMAGE failed (attempt $attempt/5)" + sleep "$((attempt * 10))" + done + echo "::error::could not pull $BUILDKIT_IMAGE after 5 attempts" + exit 1 + + - name: Create builder + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4 + with: + driver-opts: image=${{ inputs.buildkit-image }} diff --git a/.github/renovate.json5 b/.github/renovate.json5 index 9426f088..b3346912 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -7,7 +7,6 @@ ':semanticCommits', ':enablePreCommit', ':automergeDigest', - ':automergeBranch', 'helpers:pinGitHubActionDigests', ], dependencyDashboardLabels: [ @@ -54,7 +53,7 @@ 'patch', ], automerge: true, - automergeType: 'branch', + automergeType: 'pr', }, { description: 'Auto merge warpgate patch and minor updates', @@ -104,5 +103,18 @@ datasourceTemplate: 'github-releases', versioningTemplate: 'semver', }, + { + customType: 'regex', + description: 'Update CARGO_LLVM_COV_VERSION environment variable in workflows', + managerFilePatterns: [ + '/^\\.github/workflows/[^/]+\\.ya?ml$/', + ], + matchStrings: [ + 'CARGO_LLVM_COV_VERSION:\\s*["\']?(?[0-9.]+)["\']?', + ], + depNameTemplate: 'cargo-llvm-cov', + datasourceTemplate: 'crate', + versioningTemplate: 'semver', + }, ], } diff --git a/.github/workflows/build-and-push-templates.yaml b/.github/workflows/build-and-push-templates.yaml index 16feba89..aed0d3bd 100644 --- a/.github/workflows/build-and-push-templates.yaml +++ b/.github/workflows/build-and-push-templates.yaml @@ -8,6 +8,16 @@ on: - 'warpgate-templates/**' - 'ansible/**' - '.github/workflows/build-and-push-templates.yaml' + # Template images bake the Rust `ares` binary from these crates; + # rebuild when their source changes too. + - 'ares-cli/**' + - 'ares-core/**' + - 'ares-llm/**' + - 'ares-tools/**' + - 'Cargo.toml' + - 'Cargo.lock' + schedule: + - cron: '0 9 * * 1' workflow_dispatch: inputs: template_filter: @@ -471,7 +481,7 @@ jobs: echo "$HOME/.local/bin" >> "$GITHUB_PATH" - name: Setup Python - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PYTHON_VERSION }} @@ -638,7 +648,7 @@ jobs: cat ~/.config/warpgate/config.yaml - name: Set up Docker Buildx - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4 + uses: ./.github/actions/setup-buildx - name: Register templates with Warpgate run: | @@ -699,7 +709,7 @@ jobs: --digest-dir "$GITHUB_WORKSPACE" \ --verbose \ --cache-from type=registry,ref=ghcr.io/${{ matrix.namespace }}/${{ matrix.name }}:buildcache-${{ matrix.architecture.arch }} \ - --cache-to type=registry,ref=ghcr.io/${{ matrix.namespace }}/${{ matrix.name }}:buildcache-${{ matrix.architecture.arch }},mode=max + --cache-to type=registry,ref=ghcr.io/${{ matrix.namespace }}/${{ matrix.name }}:buildcache-${{ matrix.architecture.arch }},mode=max,ignore-error=true # Check if the specific digest file exists DIGEST_FILE="$GITHUB_WORKSPACE/digest-${{ matrix.name }}-${{ matrix.architecture.arch }}.txt" @@ -879,9 +889,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4 - with: - driver: docker-container + uses: ./.github/actions/setup-buildx - name: Ensure required digests exist run: | @@ -962,7 +970,7 @@ jobs: echo "$HOME/.local/bin" >> "$GITHUB_PATH" - name: Setup Python - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PYTHON_VERSION }} @@ -1129,7 +1137,7 @@ jobs: cat ~/.config/warpgate/config.yaml - name: Set up Docker Buildx - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4 + uses: ./.github/actions/setup-buildx - name: Register templates with Warpgate run: | @@ -1190,7 +1198,7 @@ jobs: --digest-dir "$GITHUB_WORKSPACE" \ --verbose \ --cache-from type=registry,ref=ghcr.io/${{ matrix.namespace }}/${{ matrix.name }}:buildcache-${{ matrix.architecture.arch }} \ - --cache-to type=registry,ref=ghcr.io/${{ matrix.namespace }}/${{ matrix.name }}:buildcache-${{ matrix.architecture.arch }},mode=max + --cache-to type=registry,ref=ghcr.io/${{ matrix.namespace }}/${{ matrix.name }}:buildcache-${{ matrix.architecture.arch }},mode=max,ignore-error=true # Check if the specific digest file exists DIGEST_FILE="$GITHUB_WORKSPACE/digest-${{ matrix.name }}-${{ matrix.architecture.arch }}.txt" @@ -1374,9 +1382,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4 - with: - driver: docker-container + uses: ./.github/actions/setup-buildx - name: Ensure required digests exist run: | @@ -1445,7 +1451,7 @@ jobs: echo "$HOME/.local/bin" >> "$GITHUB_PATH" - name: Setup Python - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PYTHON_VERSION }} @@ -1481,8 +1487,36 @@ jobs: - name: Free up disk space run: | - sudo rm -rf /usr/share/dotnet /opt/ghc "/usr/local/share/boost" /usr/local/lib/android /opt/hostedtoolcache/CodeQL - df -h + echo "Disk space before cleanup:" + df -h / + # The GPU image (CUDA runtime + llvm-18 + hashcat) is ~12GB uncompressed + # and the build triple-counts disk: BuildKit cache + exported tarball + + # reload into the Docker daemon. The default ~30GB free is not enough and + # the load step fails with "no space left on device". Strip the large + # preinstalled toolchains we never use in a container build to reclaim ~25GB. + # Keep /opt/hostedtoolcache/Python — Setup Python populates it. + sudo rm -rf \ + /usr/share/dotnet \ + /usr/share/swift \ + /usr/share/miniconda \ + /opt/ghc \ + /usr/local/.ghcup \ + /usr/local/share/boost \ + /usr/local/share/powershell \ + /usr/local/lib/android \ + /usr/local/lib/node_modules \ + /opt/microsoft \ + /opt/google \ + /opt/hostedtoolcache/CodeQL \ + /opt/hostedtoolcache/go \ + /opt/hostedtoolcache/node \ + /opt/hostedtoolcache/Ruby \ + /opt/hostedtoolcache/PyPy || true + # Drop preinstalled Docker images to reclaim /var/lib/docker space. + docker image prune -af || true + sudo apt-get clean || true + echo "Disk space after cleanup:" + df -h / - name: Add swap space run: | @@ -1562,7 +1596,7 @@ jobs: EOF - name: Set up Docker Buildx - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4 + uses: ./.github/actions/setup-buildx - name: Register templates with Warpgate run: | @@ -1595,7 +1629,7 @@ jobs: --digest-dir "$GITHUB_WORKSPACE" \ --verbose \ --cache-from type=registry,ref=ghcr.io/${{ matrix.namespace }}/${{ matrix.name }}:buildcache-${{ matrix.architecture.arch }} \ - --cache-to type=registry,ref=ghcr.io/${{ matrix.namespace }}/${{ matrix.name }}:buildcache-${{ matrix.architecture.arch }},mode=max + --cache-to type=registry,ref=ghcr.io/${{ matrix.namespace }}/${{ matrix.name }}:buildcache-${{ matrix.architecture.arch }},mode=max,ignore-error=true DIGEST_FILE="$GITHUB_WORKSPACE/digest-${{ matrix.name }}-${{ matrix.architecture.arch }}.txt" if [ -f "$DIGEST_FILE" ]; then cat "$DIGEST_FILE" @@ -1713,9 +1747,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4 - with: - driver: docker-container + uses: ./.github/actions/setup-buildx - name: Ensure required digests exist run: | @@ -1780,7 +1812,7 @@ jobs: echo "$HOME/.local/bin" >> "$GITHUB_PATH" - name: Setup Python - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PYTHON_VERSION }} @@ -1816,8 +1848,36 @@ jobs: - name: Free up disk space run: | - sudo rm -rf /usr/share/dotnet /opt/ghc "/usr/local/share/boost" /usr/local/lib/android /opt/hostedtoolcache/CodeQL - df -h + echo "Disk space before cleanup:" + df -h / + # The GPU image (CUDA runtime + llvm-18 + hashcat) is ~12GB uncompressed + # and the build triple-counts disk: BuildKit cache + exported tarball + + # reload into the Docker daemon. The default ~30GB free is not enough and + # the load step fails with "no space left on device". Strip the large + # preinstalled toolchains we never use in a container build to reclaim ~25GB. + # Keep /opt/hostedtoolcache/Python — Setup Python populates it. + sudo rm -rf \ + /usr/share/dotnet \ + /usr/share/swift \ + /usr/share/miniconda \ + /opt/ghc \ + /usr/local/.ghcup \ + /usr/local/share/boost \ + /usr/local/share/powershell \ + /usr/local/lib/android \ + /usr/local/lib/node_modules \ + /opt/microsoft \ + /opt/google \ + /opt/hostedtoolcache/CodeQL \ + /opt/hostedtoolcache/go \ + /opt/hostedtoolcache/node \ + /opt/hostedtoolcache/Ruby \ + /opt/hostedtoolcache/PyPy || true + # Drop preinstalled Docker images to reclaim /var/lib/docker space. + docker image prune -af || true + sudo apt-get clean || true + echo "Disk space after cleanup:" + df -h / - name: Add swap space run: | @@ -1897,7 +1957,7 @@ jobs: EOF - name: Set up Docker Buildx - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4 + uses: ./.github/actions/setup-buildx - name: Register templates with Warpgate run: | @@ -1930,7 +1990,7 @@ jobs: --digest-dir "$GITHUB_WORKSPACE" \ --verbose \ --cache-from type=registry,ref=ghcr.io/${{ matrix.namespace }}/${{ matrix.name }}:buildcache-${{ matrix.architecture.arch }} \ - --cache-to type=registry,ref=ghcr.io/${{ matrix.namespace }}/${{ matrix.name }}:buildcache-${{ matrix.architecture.arch }},mode=max + --cache-to type=registry,ref=ghcr.io/${{ matrix.namespace }}/${{ matrix.name }}:buildcache-${{ matrix.architecture.arch }},mode=max,ignore-error=true DIGEST_FILE="$GITHUB_WORKSPACE/digest-${{ matrix.name }}-${{ matrix.architecture.arch }}.txt" if [ -f "$DIGEST_FILE" ]; then cat "$DIGEST_FILE" @@ -2052,9 +2112,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4 - with: - driver: docker-container + uses: ./.github/actions/setup-buildx - name: Ensure required digests exist run: | @@ -2164,3 +2222,38 @@ jobs: else echo "All image builds and manifest merges completed successfully" fi + + report-failure: + name: Report Build Failure + runs-on: ubuntu-24.04 + needs: [build-summary] + if: failure() && github.event_name != 'pull_request' + permissions: + contents: read + issues: write + steps: + - name: Open or update the failure issue + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + TITLE: "CI: template builds failing on main" + run: | + existing=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open \ + --search "\"$TITLE\" in:title" --json number --jq '.[0].number // empty') + + if [ -n "$existing" ]; then + echo "Updating existing issue #$existing" + gh issue comment "$existing" --repo "$GITHUB_REPOSITORY" \ + --body "Still failing on \`${GITHUB_SHA:0:7}\` (${GITHUB_EVENT_NAME}): $RUN_URL" + else + echo "Opening new issue" + gh issue create --repo "$GITHUB_REPOSITORY" --title "$TITLE" \ + --body "$(printf '%s\n' \ + "\`Build and Push Templates\` failed on \`main\`." \ + "" \ + "Run: $RUN_URL" \ + "Commit: \`${GITHUB_SHA:0:7}\`" \ + "Trigger: \`${GITHUB_EVENT_NAME}\`" \ + "" \ + "This issue is updated on each subsequent failure. Close it once the build is green.")" + fi diff --git a/.github/workflows/molecule.yaml b/.github/workflows/molecule.yaml deleted file mode 100644 index 1f90f310..00000000 --- a/.github/workflows/molecule.yaml +++ /dev/null @@ -1,467 +0,0 @@ ---- -name: Molecule Test -on: - merge_group: - pull_request: - branches: - - main - - feat/more-attack-cov - types: - - opened - - synchronize - - reopened - paths: - - 'ansible/**' - - '.github/workflows/molecule.yaml' - - '.hooks/requirements.txt' - push: - branches: - - main - paths: - - 'ansible/**' - - '.github/workflows/molecule.yaml' - - '.hooks/requirements.txt' - schedule: - # Runs every Sunday at 4 AM (see https://crontab.guru) - - cron: "0 4 * * 0" - workflow_dispatch: - inputs: - ROLE: - description: 'Role to test' - required: false - default: '' - type: string - SCENARIO: - description: 'Molecule scenario to run (default: default)' - required: false - default: 'default' - type: string - -concurrency: - # Only cancel in-progress runs for PRs, not for main branch or scheduled runs - cancel-in-progress: ${{ github.event_name == 'pull_request' && ! contains(github.event.pull_request.labels.*.name, 'renovate') }} - group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && ! contains(github.event.pull_request.labels.*.name, 'renovate') && github.event.pull_request.number || github.ref }} - -env: - ANSIBLE_FORCE_COLOR: "1" - COLLECTION_NAMESPACE: dreadnode - COLLECTION_NAME: nimbus_range - COLLECTION_PATH: ansible_collections/dreadnode/nimbus_range - REQUIREMENTS_FILE: .hooks/requirements.txt - PY_COLORS: "1" - PYTHON_VERSION: "3.13.7" - ROLE: ${{ github.event.inputs.ROLE }} - SCENARIO: ${{ github.event.inputs.SCENARIO || 'default' }} - ANSIBLE_COLLECTIONS_PATH: ~/.ansible/collections - -permissions: - contents: read - pull-requests: write - -jobs: - detect-changes: - runs-on: ubuntu-latest - outputs: - roles: ${{ steps.filter.outputs.roles }} - matrix: ${{ steps.generate-matrix.outputs.matrix }} - max_parallel: ${{ steps.generate-matrix.outputs.max_parallel }} - test_all: ${{ steps.check-event.outputs.test_all }} - steps: - - name: Set up git repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - fetch-depth: 0 - - - name: Check event type - id: check-event - env: - EVENT_NAME: ${{ github.event_name }} - INPUT_ROLE: ${{ env.ROLE }} - run: | - # Test all on: push to main, schedule, workflow_dispatch without specific inputs, merge_group - if [[ "${EVENT_NAME}" == "push" ]] || \ - [[ "${EVENT_NAME}" == "schedule" ]] || \ - [[ "${EVENT_NAME}" == "merge_group" ]] || \ - [[ "${EVENT_NAME}" == "workflow_dispatch" && -z "${INPUT_ROLE}" ]]; then - echo "test_all=true" >> "$GITHUB_OUTPUT" - else - echo "test_all=false" >> "$GITHUB_OUTPUT" - fi - - - name: Detect changed files - id: filter - if: steps.check-event.outputs.test_all == 'false' - env: - EVENT_NAME: ${{ github.event_name }} - PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} - PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} - run: | - if [[ "${EVENT_NAME}" == "pull_request" ]]; then - BASE="${PR_BASE_SHA}" - HEAD="${PR_HEAD_SHA}" - else - BASE="origin/main" - HEAD="HEAD" - fi - - # Get changed files under ansible/ - CHANGED_FILES=$(git diff --name-only "$BASE"..."$HEAD" -- ansible/) - echo "Changed files:" - echo "$CHANGED_FILES" - - # Extract changed roles (only those with molecule tests) - ROLES=$(echo "$CHANGED_FILES" | grep '^ansible/roles/' | cut -d'/' -f3 | sort -u | tr '\n' ' ') - echo "roles=$ROLES" >> "$GITHUB_OUTPUT" - echo "Changed roles: $ROLES" - - - name: Generate test matrix - id: generate-matrix - env: - EVENT_NAME: ${{ github.event_name }} - TEST_ALL: ${{ steps.check-event.outputs.test_all }} - CHANGED_ROLES: ${{ steps.filter.outputs.roles }} - run: | - # Define roles with molecule tests - ROLES_WITH_MOLECULE=( - "acl_tools" - "base" - "coercion_tools" - "cracking_tools" - "credential_access_tools" - "lateral_movement_tools" - "mythic" - "privesc_tools" - "recon_tools" - ) - - # Define additional scenarios (role:scenario format) - # These only run on schedule/workflow_dispatch (too slow for PR CI) - declare -A ADDITIONAL_SCENARIOS=() - if [[ "${EVENT_NAME}" == "schedule" ]] || \ - [[ "${EVENT_NAME}" == "workflow_dispatch" ]]; then - ADDITIONAL_SCENARIOS=( - ["cracking_tools"]="source-build" - ) - fi - - if [[ "${TEST_ALL}" == "true" ]]; then - # Test all roles that have molecule tests - MATRIX_JSON='[' - FIRST=true - - for role in "${ROLES_WITH_MOLECULE[@]}"; do - if [[ "$FIRST" == "true" ]]; then - FIRST=false - else - MATRIX_JSON+="," - fi - MATRIX_JSON+="{\"name\":\"Role Test - ${role}\",\"path\":\"roles/${role}\",\"scenario\":\"default\"}" - - if [[ -n "${ADDITIONAL_SCENARIOS[$role]}" ]]; then - for scenario in ${ADDITIONAL_SCENARIOS[$role]}; do - MATRIX_JSON+=",{\"name\":\"Role Test - ${role} (${scenario})\",\"path\":\"roles/${role}\",\"scenario\":\"${scenario}\"}" - done - fi - done - - MATRIX_JSON+=']' - else - # Test only changed roles that have molecule tests - ROLES="${CHANGED_ROLES}" - MATRIX_JSON="[" - FIRST=true - - if [[ -n "$ROLES" ]]; then - for role in $ROLES; do - if [[ " ${ROLES_WITH_MOLECULE[*]} " == *" ${role} "* ]]; then - if [[ "$FIRST" == "true" ]]; then - FIRST=false - else - MATRIX_JSON+="," - fi - MATRIX_JSON+="{\"name\":\"Role Test - ${role}\",\"path\":\"roles/${role}\",\"scenario\":\"default\"}" - - if [[ -n "${ADDITIONAL_SCENARIOS[$role]}" ]]; then - for scenario in ${ADDITIONAL_SCENARIOS[$role]}; do - MATRIX_JSON+=",{\"name\":\"Role Test - ${role} (${scenario})\",\"path\":\"roles/${role}\",\"scenario\":\"${scenario}\"}" - done - fi - fi - done - fi - - MATRIX_JSON+=']' - fi - - echo "matrix=$MATRIX_JSON" >> "$GITHUB_OUTPUT" - echo "Matrix to test: $MATRIX_JSON" - - # Calculate dynamic max-parallel based on matrix size - MATRIX_COUNT=$(echo "$MATRIX_JSON" | jq '. | length') - - # Handle empty matrix - if [[ $MATRIX_COUNT -eq 0 ]]; then - echo "max_parallel=1" >> "$GITHUB_OUTPUT" - echo "No tests to run (empty matrix)" - exit 0 - fi - - MAX_PARALLEL=$(( (MATRIX_COUNT * 60) / 100 )) - [[ $MAX_PARALLEL -lt 4 ]] && MAX_PARALLEL=4 - [[ $MAX_PARALLEL -gt 12 ]] && MAX_PARALLEL=12 - - echo "max_parallel=$MAX_PARALLEL" >> "$GITHUB_OUTPUT" - echo "Calculated max-parallel: $MAX_PARALLEL (based on $MATRIX_COUNT tests)" - - validate-inputs: - runs-on: ubuntu-latest - steps: - - name: Set up git repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Validate inputs - env: - INPUT_ROLE: ${{ env.ROLE }} - run: | - ROLES_WITH_MOLECULE=( - "acl_tools" - "base" - "coercion_tools" - "cracking_tools" - "credential_access_tools" - "lateral_movement_tools" - "mythic" - "privesc_tools" - "recon_tools" - ) - - if [[ -n "${INPUT_ROLE}" ]]; then - if [[ ! -d "ansible/roles/${INPUT_ROLE}" ]]; then - echo "::error::Role '${INPUT_ROLE}' not found in ansible/roles/" - exit 1 - fi - if [[ ! " ${ROLES_WITH_MOLECULE[*]} " == *" ${INPUT_ROLE} "* ]]; then - echo "::error::Role '${INPUT_ROLE}' does not have molecule tests" - exit 1 - fi - fi - - role_test: - needs: validate-inputs - if: ${{ github.event.inputs.ROLE != '' }} - runs-on: ubuntu-latest - timeout-minutes: 60 - strategy: - fail-fast: false - max-parallel: 4 - matrix: - include: - - name: ${{ format('Role Test - {0}', github.event.inputs.ROLE) }} - path: ${{ format('roles/{0}', github.event.inputs.ROLE) }} - - steps: - - name: Delete huge unnecessary tools folder - shell: bash - run: | - echo "Initial disk space:" - df -h - rm -rf /opt/hostedtoolcache - echo "Disk space after cleanup:" - df -h - - - name: Checkout git repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - path: ${{ env.COLLECTION_PATH }} - - - name: Set up Python - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 - with: - python-version: ${{ env.PYTHON_VERSION }} - cache: 'pip' - cache-dependency-path: '${{ env.COLLECTION_PATH }}/${{ env.REQUIREMENTS_FILE }}' - - - name: Cache Ansible collections - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/.ansible/collections - key: ${{ runner.os }}-ansible-${{ github.ref }}-${{ hashFiles('**/requirements.yml') }} - - - name: Install dependencies - shell: bash - env: - COLL_PATH: ${{ env.COLLECTION_PATH }} - REQS_FILE: ${{ env.REQUIREMENTS_FILE }} - run: | - python3 -m pip install -r "${COLL_PATH}/${REQS_FILE}" - - - name: Install galaxy dependencies - working-directory: ${{ env.COLLECTION_PATH }}/ansible - shell: bash - env: - ANSIBLE_GALAXY_SERVER_TIMEOUT: "120" - run: | - for i in 1 2 3 4 5; do - ansible-galaxy collection install -r requirements.yml --timeout 120 && break - echo "Attempt $i/5 failed, retrying in $((i * 10))s..." - sleep $((i * 10)) - [ $i -eq 5 ] && echo "All attempts failed" && exit 1 - done - for i in 1 2 3 4 5; do - ansible-galaxy install -r requirements.yml && break - echo "Attempt $i/5 failed, retrying in $((i * 10))s..." - sleep $((i * 10)) - [ $i -eq 5 ] && echo "All attempts failed" && exit 1 - done - - - name: Build and install collection locally - working-directory: ${{ env.COLLECTION_PATH }}/ansible - shell: bash - run: | - ansible-galaxy collection build --force - ansible-galaxy collection install dreadnode-nimbus_range-*.tar.gz -p ~/.ansible/collections --force --pre - - - name: Run molecule test - working-directory: ${{ env.COLLECTION_PATH }}/ansible/${{ matrix.path }} - shell: bash - env: - ANSIBLE_CONFIG: ${{ env.COLLECTION_PATH }}/ansible/ansible.cfg - ANSIBLE_ROLES_PATH: ${{ env.COLLECTION_PATH }}/ansible/roles - MOLECULE_NO_LOG: "false" - MOLECULE_SCENARIO: ${{ env.SCENARIO }} - run: | - set -e - molecule --version - molecule list - - if ! MOLECULE_DEBUG=1 molecule test -s "${MOLECULE_SCENARIO}"; then - echo "Molecule test failed. Collecting debug information..." - - echo "Docker containers:" - docker ps -a - - echo "=== Docker Container Logs ===" - while read -r container; do - echo "=== Logs from container ${container} ===" - docker logs "${container}" 2>&1 - echo "=== End logs for container ${container} ===" - done < <(docker ps -q) - - echo "=== Molecule Logs ===" - while IFS= read -r -d '' log; do - echo "Contents of ${log}:" - cat "${log}" - echo "=== End of ${log} ===" - done < <(find . -name '*.log' -print0) - - exit 1 - fi - - full_test: - needs: [validate-inputs, detect-changes] - if: ${{ github.event.inputs.ROLE == '' && fromJson(needs.detect-changes.outputs.max_parallel) > 0 }} - runs-on: ubuntu-latest - timeout-minutes: 60 - strategy: - fail-fast: false - max-parallel: ${{ fromJson(needs.detect-changes.outputs.max_parallel) }} - matrix: - include: ${{ fromJson(needs.detect-changes.outputs.matrix) }} - - steps: - - name: Delete huge unnecessary tools folder - shell: bash - run: | - echo "Initial disk space:" - df -h - rm -rf /opt/hostedtoolcache - echo "Disk space after cleanup:" - df -h - - - name: Checkout git repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - path: ${{ env.COLLECTION_PATH }} - - - name: Set up Python - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 - with: - python-version: ${{ env.PYTHON_VERSION }} - cache: 'pip' - cache-dependency-path: '${{ env.COLLECTION_PATH }}/${{ env.REQUIREMENTS_FILE }}' - - - name: Cache Ansible collections - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/.ansible/collections - key: ${{ runner.os }}-ansible-${{ github.ref }}-${{ hashFiles('**/requirements.yml') }} - - - name: Install dependencies - shell: bash - env: - COLL_PATH: ${{ env.COLLECTION_PATH }} - REQS_FILE: ${{ env.REQUIREMENTS_FILE }} - run: | - python3 -m pip install -r "${COLL_PATH}/${REQS_FILE}" - - - name: Install galaxy dependencies - working-directory: ${{ env.COLLECTION_PATH }}/ansible - shell: bash - env: - ANSIBLE_GALAXY_SERVER_TIMEOUT: "120" - run: | - for i in 1 2 3 4 5; do - ansible-galaxy collection install -r requirements.yml --timeout 120 && break - echo "Attempt $i/5 failed, retrying in $((i * 10))s..." - sleep $((i * 10)) - [ $i -eq 5 ] && echo "All attempts failed" && exit 1 - done - for i in 1 2 3 4 5; do - ansible-galaxy install -r requirements.yml && break - echo "Attempt $i/5 failed, retrying in $((i * 10))s..." - sleep $((i * 10)) - [ $i -eq 5 ] && echo "All attempts failed" && exit 1 - done - - - name: Build and install collection locally - working-directory: ${{ env.COLLECTION_PATH }}/ansible - shell: bash - run: | - ansible-galaxy collection build --force - ansible-galaxy collection install dreadnode-nimbus_range-*.tar.gz -p ~/.ansible/collections --force --pre - - - name: Run molecule test - working-directory: ${{ env.COLLECTION_PATH }}/ansible/${{ matrix.path }} - shell: bash - env: - ANSIBLE_CONFIG: ${{ env.COLLECTION_PATH }}/ansible/ansible.cfg - MOLECULE_SCENARIO: ${{ matrix.scenario }} - run: | - set -e - molecule --version - molecule list - - SCENARIO="${MOLECULE_SCENARIO:-default}" - - if ! MOLECULE_DEBUG=1 molecule test -s "$SCENARIO"; then - echo "Molecule test failed. Collecting debug information..." - - echo "Docker containers:" - docker ps -a - - echo "=== Docker Container Logs ===" - while read -r container; do - echo "=== Logs from container ${container} ===" - docker logs "${container}" 2>&1 - echo "=== End logs for container ${container} ===" - done < <(docker ps -q) - - echo "=== Molecule Logs ===" - while IFS= read -r -d '' log; do - echo "Contents of ${log}:" - cat "${log}" - echo "=== End of ${log} ===" - done < <(find . -name '*.log' -print0) - - exit 1 - fi diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml index 8efb57ce..171124e9 100644 --- a/.github/workflows/pre-commit.yaml +++ b/.github/workflows/pre-commit.yaml @@ -54,12 +54,12 @@ jobs: persist-credentials: false - name: Set up Python ${{ env.PYTHON_VERSION_ANSIBLE_LINT }} (for ansible-lint pre-commit hook) - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PYTHON_VERSION_ANSIBLE_LINT }} - name: Set up Python ${{ env.PYTHON_VERSION }} - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PYTHON_VERSION }} cache: 'pip' @@ -69,7 +69,7 @@ jobs: run: python3 -m pip install -r .hooks/requirements.txt - name: Set up Go - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: go-version: ${{ env.GO_VERSION }} @@ -114,9 +114,28 @@ jobs: rm -f /tmp/install-task.sh task --version + - name: Prefetch remote Taskfile includes + # Root Taskfile.yaml pulls includes from raw.githubusercontent.com; task's + # default 10s download timeout occasionally trips on transient CDN latency + # and fails the job before any hook runs. Cache with retries + longer + # timeout so the pre-commit step below can then work offline. + run: | + for i in 1 2 3 4 5; do + task -y --download --timeout=60s && exit 0 + echo "Attempt $i/5 failed, retrying in $((i * 10))s..." + sleep $((i * 10)) + done + echo "All prefetch attempts failed" + exit 1 + - name: Run pre-commit id: precommit - run: task -y run-pre-commit + env: + # Rust checks (fmt / clippy / check / test) run in parallel jobs in + # the dedicated 🦀 Rust workflow with their own caches. Skipping them + # here trims ~11 minutes off this job without losing coverage. + SKIP: cargo-fmt,cargo-clippy,cargo-check,cargo-test + run: task -y --timeout=60s pre-commit:run-pre-commit - name: Capture autofix patch id: capture diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index cd22871a..a5492387 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -76,17 +76,22 @@ jobs: for bin in "${BINS[@]}"; do ARCHIVE="${bin}-${TAG}-${{ matrix.target }}" - mkdir -p "${ARCHIVE}" - cp "target/${{ matrix.target }}/release/${bin}" "${ARCHIVE}/" 2>/dev/null || true - if [ -f "${ARCHIVE}/${bin}" ]; then - tar czf "${ARCHIVE}.tar.gz" "${ARCHIVE}" - sha256sum "${ARCHIVE}.tar.gz" > "${ARCHIVE}.tar.gz.sha256" || shasum -a 256 "${ARCHIVE}.tar.gz" > "${ARCHIVE}.tar.gz.sha256" + SRC="target/${{ matrix.target }}/release/${bin}" + + if [ ! -f "${SRC}" ]; then + echo "::error::expected release binary not found: ${SRC}" + exit 1 fi + + mkdir -p "${ARCHIVE}" + cp "${SRC}" "${ARCHIVE}/" + tar czf "${ARCHIVE}.tar.gz" "${ARCHIVE}" + sha256sum "${ARCHIVE}.tar.gz" > "${ARCHIVE}.tar.gz.sha256" || shasum -a 256 "${ARCHIVE}.tar.gz" > "${ARCHIVE}.tar.gz.sha256" rm -rf "${ARCHIVE}" done - name: Upload artifacts - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: binaries-${{ matrix.target }} path: | @@ -141,7 +146,7 @@ jobs: } >> "$GITHUB_OUTPUT" - name: Create GitHub Release - uses: softprops/action-gh-release@718ea10b132b3b2eba29c1007bb80653f286566b # v3.0.1 + uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3.0.2 with: generate_release_notes: true body: | diff --git a/.github/workflows/renovate.yaml b/.github/workflows/renovate.yaml index 8e822c09..ae7a3a29 100644 --- a/.github/workflows/renovate.yaml +++ b/.github/workflows/renovate.yaml @@ -63,7 +63,7 @@ jobs: token: "${{ steps.app-token.outputs.token }}" - name: Set up Python - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PYTHON_VERSION }} @@ -71,12 +71,14 @@ jobs: run: python3 -m pip install pre-commit - name: Renovate - uses: renovatebot/github-action@3064367f740a1a91cca218698a63902689cce200 # v46.1.20 + uses: renovatebot/github-action@316d7cd859606d6039a2182b7d69199e9b036835 # v46.2.1 env: LOG_LEVEL: "${{ inputs.logLevel || 'debug' }}" RENOVATE_AUTODISCOVER: true RENOVATE_AUTODISCOVER_FILTER: "${{ github.repository }}" RENOVATE_DRY_RUN: "${{ inputs.dryRun }}" + # Required: renovate refuses to process forks unless explicitly enabled. + RENOVATE_FORK_PROCESSING: enabled RENOVATE_INTERNAL_CHECKS_FILTER: strict RENOVATE_PLATFORM: github RENOVATE_PLATFORM_COMMIT: true diff --git a/.github/workflows/rust.yaml b/.github/workflows/rust.yaml index dd51722c..bc27370a 100644 --- a/.github/workflows/rust.yaml +++ b/.github/workflows/rust.yaml @@ -32,6 +32,7 @@ concurrency: env: CARGO_TERM_COLOR: always RUST_BACKTRACE: 1 + CARGO_LLVM_COV_VERSION: 0.8.7 permissions: actions: read @@ -79,9 +80,9 @@ jobs: components: llvm-tools-preview - name: Install cargo-llvm-cov - uses: taiki-e/install-action@735e5933943122c5ac182670a935f54a949265c1 # v2 + uses: taiki-e/install-action@6c6fd71fe4fb72c3697d269963d0e15df8adedad # v2 with: - tool: cargo-llvm-cov + tool: cargo-llvm-cov@${{ env.CARGO_LLVM_COV_VERSION }} - name: Cache cargo registry and build uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -156,4 +157,4 @@ jobs: ${{ runner.os }}-cargo-clippy-refs/heads/main- - name: Run clippy - run: cargo clippy --workspace -- -D warnings + run: cargo clippy --workspace --all-targets -- -D warnings diff --git a/.github/workflows/semantic-prs.yaml b/.github/workflows/semantic-prs.yaml index 1dae4b76..f9d4838c 100644 --- a/.github/workflows/semantic-prs.yaml +++ b/.github/workflows/semantic-prs.yaml @@ -10,6 +10,12 @@ on: - edited - synchronize - reopened + workflow_dispatch: + inputs: + pr_number: + description: PR number to validate (used when dispatched manually) + required: true + type: string permissions: pull-requests: read @@ -19,6 +25,22 @@ jobs: name: Validate PR title runs-on: ubuntu-latest steps: - - uses: amannn/action-semantic-pull-request@48f256284bd46cdaab1048c3721360e808335d50 # v6.1.1 + - name: Validate conventional-commit title (pull_request) + if: github.event_name == 'pull_request' + uses: amannn/action-semantic-pull-request@48f256284bd46cdaab1048c3721360e808335d50 # v6.1.1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Validate conventional-commit title (workflow_dispatch) + if: github.event_name == 'workflow_dispatch' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ inputs.pr_number }} + run: | + title=$(gh pr view "$PR_NUMBER" --repo "$GITHUB_REPOSITORY" --json title -q .title) + echo "PR #$PR_NUMBER title: $title" + if echo "$title" | grep -Eq '^(feat|fix|docs|style|refactor|perf|test|build|ci|chore|revert)(\([^)]+\))?!?: .+'; then + echo "Title matches conventional commit format." + else + echo "::error::PR title does not match conventional commit format" + exit 1 + fi diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml index 998f2dc3..c57ef136 100644 --- a/.github/workflows/semgrep.yaml +++ b/.github/workflows/semgrep.yaml @@ -66,6 +66,7 @@ jobs: - name: Upload SARIF to GitHub Security tab if: always() + continue-on-error: true uses: github/codeql-action/upload-sarif@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 with: sarif_file: semgrep-results.sarif diff --git a/.github/workflows/test-template-builds.yaml b/.github/workflows/test-template-builds.yaml index ff893cd4..c34ebd55 100644 --- a/.github/workflows/test-template-builds.yaml +++ b/.github/workflows/test-template-builds.yaml @@ -12,6 +12,7 @@ on: - reopened paths: - 'warpgate-templates/**' + - 'ansible/**' - '.github/workflows/test-template-builds.yaml' permissions: @@ -47,13 +48,49 @@ jobs: id: detect run: | # Get list of changed files in templates/ - CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD -- warpgate-templates/) + CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD -- warpgate-templates/templates/) echo "Changed files:" echo "$CHANGED_FILES" # Extract unique template names from changed paths - CHANGED_TEMPLATES=$(echo "$CHANGED_FILES" | grep -oP 'warpgate-templates/\K[^/]+' | sort -u) + CHANGED_TEMPLATES=$(echo "$CHANGED_FILES" | grep -oP 'warpgate-templates/templates/\K[^/]+' | sort -u) + + CHANGED_ANSIBLE=$(git diff --name-only origin/${{ github.base_ref }}...HEAD -- ansible/) + + if [ -n "$CHANGED_ANSIBLE" ]; then + echo "" + echo "Changed ansible files:" + echo "$CHANGED_ANSIBLE" + + SHARED_ANSIBLE_CHANGE=false + if echo "$CHANGED_ANSIBLE" | grep -qvE '^ansible/playbooks/ares/[^/]+\.yml$'; then + SHARED_ANSIBLE_CHANGE=true + fi + + echo "" + echo "Templates pulled in by ansible changes (shared=$SHARED_ANSIBLE_CHANGE):" + for template_yaml in warpgate-templates/templates/*/warpgate.yaml; do + [ -f "$template_yaml" ] || continue + grep -q "type: ansible" "$template_yaml" || continue + + ansible_template=$(basename "$(dirname "$template_yaml")") + + if [ "$SHARED_ANSIBLE_CHANGE" = "true" ]; then + echo " $ansible_template: shared ansible change" + CHANGED_TEMPLATES=$(printf '%s\n%s' "$CHANGED_TEMPLATES" "$ansible_template") + continue + fi + + playbook=$(grep -m1 "playbook_path:" "$template_yaml" | sed 's|.*/||') + if echo "$CHANGED_ANSIBLE" | grep -q "playbooks/ares/${playbook}$"; then + echo " $ansible_template: runs changed playbook $playbook" + CHANGED_TEMPLATES=$(printf '%s\n%s' "$CHANGED_TEMPLATES" "$ansible_template") + fi + done + fi + + CHANGED_TEMPLATES=$(echo "$CHANGED_TEMPLATES" | grep -v '^$' | sort -u) echo "" echo "Changed templates:" @@ -82,7 +119,7 @@ jobs: CHANGED_BASE_LIST="[]" for template_name in $CHANGED_TEMPLATES; do - template_dir="warpgate-templates/$template_name" + template_dir="warpgate-templates/templates/$template_name" if [ ! -f "${template_dir}/warpgate.yaml" ]; then echo "Warning: $template_name has no warpgate.yaml, skipping" @@ -250,7 +287,7 @@ jobs: echo "$HOME/.local/bin" >> "$GITHUB_PATH" - name: Setup Python - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PYTHON_VERSION }} @@ -357,10 +394,9 @@ jobs: EOF - name: Set up Docker Buildx - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4 + uses: ./.github/actions/setup-buildx with: - driver-opts: | - image=moby/buildkit:latest + buildkit-image: moby/buildkit:latest - name: Test build ${{ matrix.name }} (amd64) run: | @@ -370,13 +406,10 @@ jobs: echo "Building ${{ matrix.name }} for amd64..." - # Build and push to registry with test tag - # We push to make the image available, then pull locally for artifact creation warpgate build warpgate.yaml \ --arch amd64 \ --registry ghcr.io/${{ matrix.namespace }} \ --tag test-${{ github.run_id }} \ - --push \ --verbose echo "Build successful for ${{ matrix.name }}" @@ -385,17 +418,10 @@ jobs: - name: Free up disk space after build run: | - # Remove buildx cache and unused images to make room for docker save docker buildx prune -af || true - docker system prune -af || true + docker system prune -f || true df -h / - - name: Pull image for local use - run: | - IMAGE_NAME="ghcr.io/${{ matrix.namespace }}/${{ matrix.name }}:test-${{ github.run_id }}" - echo "Pulling $IMAGE_NAME..." - docker pull "$IMAGE_NAME" - - name: Save image for dependent templates run: | IMAGE_NAME="ghcr.io/${{ matrix.namespace }}/${{ matrix.name }}:test-${{ github.run_id }}" @@ -418,7 +444,7 @@ jobs: # =========================================================================== # Phase 2: Build dependent templates - # These templates use base images from our registry (ghcr.io/dreadnode/*) + # These templates use base images from our registry (ghcr.io/l50/*) # If their base template was changed in this PR, load it from artifacts # =========================================================================== test-build-dependent: @@ -450,7 +476,7 @@ jobs: echo "$HOME/.local/bin" >> "$GITHUB_PATH" - name: Setup Python - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PYTHON_VERSION }} @@ -557,10 +583,9 @@ jobs: EOF - name: Set up Docker Buildx - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4 + uses: ./.github/actions/setup-buildx with: - driver-opts: | - image=moby/buildkit:latest + buildkit-image: moby/buildkit:latest - name: Check if base template was changed id: check-base diff --git a/.github/workflows/validate-templates.yaml b/.github/workflows/validate-templates.yaml index d82f51df..2ce1eb19 100644 --- a/.github/workflows/validate-templates.yaml +++ b/.github/workflows/validate-templates.yaml @@ -115,6 +115,13 @@ jobs: AZURE_GALLERY_NAME: placeholder-gallery AZURE_IDENTITY_ID: /subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/placeholder-rg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/placeholder-uami AZURE_VM_SIZE: Standard_D4s_v3 + PROXMOX_NODE: placeholder-node + PROXMOX_SOURCE_TEMPLATE: placeholder-template + PROXMOX_STORAGE: placeholder-storage + PROXMOX_POOL: '' + PROXMOX_CI_PASSWORD: placeholder + PROXMOX_CI_SSH_KEY: placeholder + PROXMOX_SSH_PRIVATE_KEY: placeholder run: | failed=0 while IFS= read -r template; do @@ -252,7 +259,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Setup Python - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PYTHON_VERSION }} @@ -305,7 +312,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Setup Python - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PYTHON_VERSION }} diff --git a/ares-cli/src/config.rs b/ares-cli/src/config.rs index b46bebe9..09bb0d17 100644 --- a/ares-cli/src/config.rs +++ b/ares-cli/src/config.rs @@ -369,4 +369,141 @@ mod tests { let result = replace_model_in_yaml(yaml, "recon", "wrong-model", "new-model"); assert!(result.contains("model: \"new-model\"")); } + + const SHIPPED_CONFIG: &str = include_str!("../../config/ares.yaml"); + + fn shipped_config_copy() -> (tempfile::TempDir, String) { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("ares.yaml"); + std::fs::write(&path, SHIPPED_CONFIG).unwrap(); + let path = path.to_str().unwrap().to_string(); + (dir, path) + } + + #[test] + fn resolve_config_path_accepts_an_existing_file() { + let (_dir, path) = shipped_config_copy(); + assert_eq!( + resolve_config_path(Some(path.clone())).unwrap(), + std::path::PathBuf::from(&path) + ); + } + + #[test] + fn resolve_config_path_rejects_a_missing_file() { + let dir = tempfile::tempdir().unwrap(); + let missing = dir.path().join("absent.yaml"); + let err = resolve_config_path(Some(missing.to_str().unwrap().to_string())).unwrap_err(); + assert!(err.to_string().contains("Config file not found")); + } + + #[test] + fn shipped_config_parses_and_validates() { + let (_dir, path) = shipped_config_copy(); + config_validate(Some(path)).unwrap(); + } + + #[test] + fn shipped_config_renders_both_show_modes() { + let (_dir, path) = shipped_config_copy(); + config_show(Some(path.clone()), true).unwrap(); + config_show(Some(path), false).unwrap(); + } + + #[test] + fn set_model_rewrites_one_role_and_leaves_the_rest() { + let (_dir, path) = shipped_config_copy(); + let before = AresConfig::load(std::path::Path::new(&path)).unwrap(); + let untouched: Vec<(String, String)> = before + .agents + .iter() + .filter(|(role, _)| role.as_str() != "recon") + .map(|(role, agent)| (role.clone(), agent.model.clone())) + .collect(); + + config_set_model( + Some(path.clone()), + Some("recon".to_string()), + "test-model".to_string(), + false, + ) + .unwrap(); + + let after = AresConfig::load(std::path::Path::new(&path)).unwrap(); + assert_eq!(after.agents["recon"].model, "test-model"); + for (role, model) in untouched { + assert_eq!( + after.agents[&role].model, model, + "role {role} was rewritten" + ); + } + } + + #[test] + fn set_model_preserves_comments_and_step_budgets() { + let (_dir, path) = shipped_config_copy(); + let before = AresConfig::load(std::path::Path::new(&path)).unwrap(); + + config_set_model( + Some(path.clone()), + Some("recon".to_string()), + "test-model".to_string(), + false, + ) + .unwrap(); + + let rewritten = std::fs::read_to_string(&path).unwrap(); + assert_eq!( + rewritten + .lines() + .filter(|l| l.trim_start().starts_with('#')) + .count(), + SHIPPED_CONFIG + .lines() + .filter(|l| l.trim_start().starts_with('#')) + .count(), + ); + let after = AresConfig::load(std::path::Path::new(&path)).unwrap(); + assert_eq!( + after.agents["recon"].max_steps, + before.agents["recon"].max_steps + ); + } + + #[test] + fn set_model_all_rewrites_every_role() { + let (_dir, path) = shipped_config_copy(); + config_set_model(Some(path.clone()), None, "test-model".to_string(), true).unwrap(); + + let after = AresConfig::load(std::path::Path::new(&path)).unwrap(); + assert!(!after.agents.is_empty()); + for (role, agent) in &after.agents { + assert_eq!(agent.model, "test-model", "role {role} kept its old model"); + } + } + + #[test] + fn set_model_rejects_an_unknown_role() { + let (_dir, path) = shipped_config_copy(); + let err = config_set_model( + Some(path.clone()), + Some("nonexistent".to_string()), + "test-model".to_string(), + false, + ) + .unwrap_err(); + + assert!(err.to_string().contains("Unknown role 'nonexistent'")); + assert_eq!(std::fs::read_to_string(&path).unwrap(), SHIPPED_CONFIG); + } + + #[test] + fn set_model_requires_a_role_when_all_is_unset() { + let (_dir, path) = shipped_config_copy(); + let err = config_set_model(Some(path.clone()), None, "test-model".to_string(), false) + .unwrap_err(); + + assert!(err.to_string().contains("Role argument is required")); + assert_eq!(std::fs::read_to_string(&path).unwrap(), SHIPPED_CONFIG); + } } diff --git a/ares-cli/src/ops/loot/format/hosts.rs b/ares-cli/src/ops/loot/format/hosts.rs index 0b2f6b01..6a7747b3 100644 --- a/ares-cli/src/ops/loot/format/hosts.rs +++ b/ares-cli/src/ops/loot/format/hosts.rs @@ -494,4 +494,147 @@ mod tests { let map = hostname_by_ip(&[make_host("192.168.58.10", "DC01.CONTOSO.LOCAL.")]); assert_eq!(map.get("192.168.58.10").unwrap(), "dc01.contoso.local"); } + + fn with_services(mut host: Host, services: &[&str]) -> Host { + host.services = services.iter().map(|s| (*s).to_string()).collect(); + host + } + + fn dedup(hosts: &[Host]) -> Vec { + dedup_hosts(hosts, &HashMap::new(), &HashMap::new()) + } + + #[test] + fn dedup_drops_cidr_rows_and_empty_ips() { + let hosts = [ + make_host("192.168.58.0/24", "subnet"), + make_host("", ""), + make_host("192.168.58.10", "dc01.contoso.local"), + ]; + + let result = dedup(&hosts); + assert_eq!(result.len(), 1); + assert_eq!(result[0].ip, "192.168.58.10"); + } + + #[test] + fn dedup_merges_rows_sharing_an_ip() { + let hosts = [ + with_services(make_host("192.168.58.10", "dc01"), &["445/tcp"]), + with_services(make_host("192.168.58.10", ""), &["445/tcp", "389/tcp"]), + ]; + + let result = dedup(&hosts); + assert_eq!(result.len(), 1); + assert_eq!(result[0].services, vec!["445/tcp", "389/tcp"]); + } + + #[test] + fn dedup_upgrades_a_short_name_to_the_fqdn() { + let hosts = [ + make_host("192.168.58.10", "dc01"), + make_host("192.168.58.10", "dc01.contoso.local"), + ]; + + assert_eq!(dedup(&hosts)[0].hostname, "dc01.contoso.local"); + } + + #[test] + fn dedup_keeps_the_fqdn_when_the_short_name_arrives_second() { + let hosts = [ + make_host("192.168.58.10", "dc01.contoso.local"), + make_host("192.168.58.10", "dc01"), + ]; + + assert_eq!(dedup(&hosts)[0].hostname, "dc01.contoso.local"); + } + + #[test] + fn dedup_is_dc_is_sticky_across_merges() { + let mut dc = make_host("192.168.58.10", "dc01.contoso.local"); + dc.is_dc = true; + let hosts = [dc, make_host("192.168.58.10", "dc01.contoso.local")]; + + assert!(dedup(&hosts)[0].is_dc); + } + + #[test] + fn dedup_fills_an_empty_os_but_never_overwrites_one() { + let mut first = make_host("192.168.58.10", "dc01.contoso.local"); + first.os = "Windows Server 2019".to_string(); + let mut second = make_host("192.168.58.10", "dc01.contoso.local"); + second.os = "Windows Server 2022".to_string(); + + assert_eq!(dedup(&[first, second.clone()])[0].os, "Windows Server 2019"); + + let mut blank = make_host("192.168.58.10", "dc01.contoso.local"); + blank.os = String::new(); + assert_eq!(dedup(&[blank, second])[0].os, "Windows Server 2022"); + } + + #[test] + fn dedup_unions_roles_without_duplicating() { + let mut first = make_host("192.168.58.10", "dc01.contoso.local"); + first.roles = vec!["dc".to_string()]; + let mut second = make_host("192.168.58.10", "dc01.contoso.local"); + second.roles = vec!["dc".to_string(), "ca".to_string()]; + + assert_eq!(dedup(&[first, second])[0].roles, vec!["dc", "ca"]); + } + + #[test] + fn dedup_folds_a_hostname_only_row_into_the_matching_ip_row() { + let hosts = [ + make_host("192.168.58.10", "dc01.contoso.local"), + with_services(make_host("dc01.contoso.local", ""), &["445/tcp"]), + ]; + + let result = dedup(&hosts); + assert_eq!( + result.len(), + 1, + "hostname-only row should not become its own entry" + ); + assert_eq!(result[0].ip, "192.168.58.10"); + assert_eq!(result[0].services, vec!["445/tcp"]); + } + + #[test] + fn dedup_keeps_an_unmatched_hostname_row_only_when_it_has_services() { + let with_svc = [with_services( + make_host("web01.contoso.local", ""), + &["80/tcp"], + )]; + let result = dedup(&with_svc); + assert_eq!(result.len(), 1); + assert_eq!(result[0].hostname, "web01.contoso.local"); + assert!(result[0].ip.is_empty()); + + let without_svc = [make_host("web01.contoso.local", "")]; + assert!(dedup(&without_svc).is_empty()); + } + + #[test] + fn dedup_marks_known_domain_controllers_and_backfills_the_fqdn() { + let hosts = [make_host("192.168.58.10", "")]; + let netbios = HashMap::from([("DC01".to_string(), "dc01.contoso.local".to_string())]); + let dcs = HashMap::from([("contoso.local".to_string(), "192.168.58.10".to_string())]); + + let result = dedup_hosts(&hosts, &netbios, &dcs); + assert!(result[0].is_dc); + assert_eq!(result[0].hostname, "dc01.contoso.local"); + } + + #[test] + fn dedup_sorts_by_ip() { + let hosts = [ + make_host("192.168.58.30", "ws01.contoso.local"), + make_host("192.168.58.10", "dc01.contoso.local"), + make_host("192.168.58.20", "sql01.contoso.local"), + ]; + + let result = dedup(&hosts); + let ips: Vec<&str> = result.iter().map(|h| h.ip.as_str()).collect(); + assert_eq!(ips, ["192.168.58.10", "192.168.58.20", "192.168.58.30"]); + } } diff --git a/ares-cli/src/ops/loot/snapshot.rs b/ares-cli/src/ops/loot/snapshot.rs index 343764c8..3dcf39e2 100644 --- a/ares-cli/src/ops/loot/snapshot.rs +++ b/ares-cli/src/ops/loot/snapshot.rs @@ -129,3 +129,189 @@ pub(crate) fn print_diff(prev: &LootSnapshot, curr: &LootSnapshot) { println!(" [share] {host}/{name}"); } } + +#[cfg(test)] +mod tests { + use super::*; + + use ares_core::models::{Credential, Hash, Host, Share, User}; + use serde_json::json; + + fn state() -> SharedRedTeamState { + SharedRedTeamState::new("op-test-001".to_string()) + } + + fn host(hostname: &str, ip: &str) -> Host { + serde_json::from_value(json!({ "hostname": hostname, "ip": ip })).unwrap() + } + + fn user(username: &str, domain: &str) -> User { + serde_json::from_value(json!({ "username": username, "domain": domain })).unwrap() + } + + fn credential(username: &str, domain: &str, password: &str) -> Credential { + serde_json::from_value( + json!({ "username": username, "domain": domain, "password": password }), + ) + .unwrap() + } + + fn hash(username: &str, domain: &str, hash_type: &str, hash_value: &str) -> Hash { + serde_json::from_value(json!({ + "username": username, + "domain": domain, + "hash_type": hash_type, + "hash_value": hash_value, + })) + .unwrap() + } + + fn share(host: &str, name: &str) -> Share { + serde_json::from_value(json!({ "host": host, "name": name })).unwrap() + } + + #[test] + fn empty_state_yields_an_empty_snapshot() { + let snap = loot_snapshot(&state()); + assert!(snap.domains.is_empty()); + assert!(snap.host_keys.is_empty()); + assert!(snap.user_keys.is_empty()); + assert!(snap.cred_keys.is_empty()); + assert!(snap.hash_keys.is_empty()); + assert!(snap.share_keys.is_empty()); + } + + #[test] + fn domains_are_trimmed_lowercased_and_deduped() { + let mut s = state(); + s.all_domains = vec![ + "CONTOSO.local".to_string(), + " contoso.local ".to_string(), + "fabrikam.local".to_string(), + ]; + + let snap = loot_snapshot(&s); + assert_eq!(snap.domains.len(), 2); + assert!(snap.domains.contains("contoso.local")); + assert!(snap.domains.contains("fabrikam.local")); + } + + #[test] + fn blank_domains_are_dropped() { + let mut s = state(); + s.all_domains = vec![ + String::new(), + " ".to_string(), + "contoso.local".to_string(), + ]; + + let snap = loot_snapshot(&s); + assert_eq!( + snap.domains, + HashSet::from(["contoso.local".to_string()]), + "whitespace-only domains must not become empty-string keys" + ); + } + + #[test] + fn user_keys_normalize_domain_and_username() { + let mut s = state(); + s.all_users = vec![ + user("Alice", "CONTOSO.local"), + user(" alice ", " contoso.local "), + ]; + + let snap = loot_snapshot(&s); + assert_eq!( + snap.user_keys, + HashSet::from([("contoso.local".to_string(), "alice".to_string())]) + ); + } + + #[test] + fn credential_keys_normalize_identity_but_keep_the_password_verbatim() { + let mut s = state(); + s.all_credentials = vec![credential("Alice", "CONTOSO.local", "P@ssw0rd!")]; + + let snap = loot_snapshot(&s); + assert_eq!( + snap.cred_keys, + HashSet::from([( + "contoso.local".to_string(), + "alice".to_string(), + "P@ssw0rd!".to_string(), + )]), + "case-folding the password would merge distinct credentials" + ); + } + + #[test] + fn credentials_differing_only_by_password_stay_separate() { + let mut s = state(); + s.all_credentials = vec![ + credential("alice", "contoso.local", "P@ssw0rd!"), + credential("alice", "contoso.local", "p@ssw0rd!"), + ]; + + assert_eq!(loot_snapshot(&s).cred_keys.len(), 2); + } + + #[test] + fn hash_keys_are_fully_lowercased() { + let mut s = state(); + s.all_hashes = vec![hash( + "Alice", + "CONTOSO.local", + "NTLM", + "AAD3B435B51404EEAAD3B435B51404EE", + )]; + + let snap = loot_snapshot(&s); + assert_eq!( + snap.hash_keys, + HashSet::from([( + "contoso.local".to_string(), + "alice".to_string(), + "ntlm".to_string(), + "aad3b435b51404eeaad3b435b51404ee".to_string(), + )]) + ); + } + + #[test] + fn host_and_share_keys_are_kept_verbatim() { + let mut s = state(); + s.all_hosts = vec![host("DC01.contoso.local", "192.168.58.10")]; + s.all_shares = vec![share("192.168.58.10", "SYSVOL")]; + + let snap = loot_snapshot(&s); + assert_eq!( + snap.host_keys, + HashSet::from([( + "DC01.contoso.local".to_string(), + "192.168.58.10".to_string() + )]) + ); + assert_eq!( + snap.share_keys, + HashSet::from([("192.168.58.10".to_string(), "SYSVOL".to_string())]) + ); + } + + #[test] + fn repeated_entries_collapse_to_one_key_each() { + let mut s = state(); + s.all_hosts = vec![ + host("dc01.contoso.local", "192.168.58.10"), + host("dc01.contoso.local", "192.168.58.10"), + ]; + s.all_shares = vec![ + share("192.168.58.10", "SYSVOL"), + share("192.168.58.10", "SYSVOL"), + ]; + + let snap = loot_snapshot(&s); + assert_eq!(snap.host_keys.len(), 1); + assert_eq!(snap.share_keys.len(), 1); + } +}