diff --git a/.github/workflows/update-pages.yml b/.github/workflows/update-pages.yml index d1afb40c9a..f17d369816 100644 --- a/.github/workflows/update-pages.yml +++ b/.github/workflows/update-pages.yml @@ -49,7 +49,7 @@ jobs: run: | mkdir -p gh-pages git fetch --depth=1 origin gh-pages - for cache_path in github/commitActivity github/commitActivityHashes github/prMetrics; do + for cache_path in github/commitActivity github/commitActivityHashes github/prMetrics azure; do if git cat-file -e "origin/gh-pages:${cache_path}"; then git archive origin/gh-pages "${cache_path}" | tar -x -C gh-pages fi @@ -57,6 +57,11 @@ jobs: - name: Collect data env: + DASHBOARD_AZURE_SIGNING_CACHE_ONLY: ${{ github.event_name == 'pull_request' }} + AZURE_SIGNING_RESOURCE_ID: ${{ secrets.AZURE_SIGNING_RESOURCE_ID }} + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} DASHBOARD_AUR_REPOS: sunshine,sunshine-bin,sunshine-git CODECOV_TOKEN: ${{ secrets.CODECOV_API_TOKEN }} DISCORD_INVITE: ${{ secrets.DISCORD_INVITE }} diff --git a/README.md b/README.md index c2ddb872be..ed94dab0be 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,64 @@ A dashboard for viewing LizardByte repository data inside a Jekyll static site. +## Azure code signing metrics + +The optional Azure Code Signing section displays Artifact Signing (formerly Trusted Signing) +completed signing requests: UTC month-to-date and last-30-day totals, plus daily history. +Counts cover the entire signing account. Azure's standard `SignCompleted` metric does not include +repository, file, certificate profile, failure-rate, or signing-duration dimensions; these counts +are not billing records. Azure reporting can be delayed and the current day is incomplete. +Missing metric samples are not presented as confirmed zero usage. + +### Collection costs + +The collector refreshes the current UTC day and recent unsettled days, with a three-hour cache and +no automatic retries. Each update also backfills at most seven missing historical days, starting +with the oldest dates in Azure's available 90-day window. Once a complete day has had two full +days of reporting grace and is fetched successfully, it is finalized and never fetched again. +Historical gaps are queried separately, so a backfill request never spans finalized days. +Finalized history is retained indefinitely in the existing `gh-pages` branch, without Azure storage. +Successfully queried days with no numeric samples remain unknown, rather than becoming zero; +partial month and 30-day totals are labeled while history fills in. + +The scheduled job runs eight times per day: one metric query per update when caught up, or at +most two while backfilling (at most 496 queries in a 31-day month for one account). Site visitors +read the generated JSON and never query Azure. +Pull-request builds use only the Azure data restored from the published `gh-pages` cache and make +no Azure API calls, even when signing secrets are available. Their preview displays cached counts +when available; without published Azure data, the section stays hidden. +No diagnostic settings, Azure Storage, Log Analytics, Event Hubs, custom metrics, or Azure alerts +are created or required. + +Microsoft currently lists unlimited standard platform metric ingestion as free and the first +1,000,000 metric query API calls per month as included. This allowance is shared with other consumers +in the Azure subscription; exceeding it can incur charges. With no other monitoring consumers, +this collector's scheduled usage fits comfortably within the included allowance. Your existing +signing plan and any signature overage charges still apply independently of this dashboard. +See [Azure Monitor pricing](https://azure.microsoft.com/en-us/pricing/details/monitor/) and +[Artifact Signing's supported metric](https://learn.microsoft.com/en-us/azure/azure-monitor/reference/supported-metrics/microsoft-codesigning-codesigningaccounts-metrics). + +### Setup + +1. Set the repository or organization secret `AZURE_SIGNING_RESOURCE_ID` to the full existing account ID: + `/subscriptions//resourceGroups//providers/Microsoft.CodeSigning/codeSigningAccounts/`. + In the Azure portal, open the signing account, select **Overview → JSON View**, and copy its **ID**. + `AZURE_SIGNING_ACCOUNT` supplies only the final account name; it is not the full resource ID. + `AZURE_SIGNING_CERT_PROFILE` is not needed for these account-wide metrics. + The resource ID does not grant access by itself, but using a secret keeps the subscription and + resource identifiers private and enables GitHub Actions log masking. +2. Make the secrets `AZURE_TENANT_ID`, `AZURE_CLIENT_ID`, and `AZURE_CLIENT_SECRET` available to this + repository. These are the same credential names used by the organization's signing workflows. + The service principal needs read access to metrics on this account; **Monitoring Reader** scoped + to the signing account supplies that permission. The signing role alone may not allow metric reads. +3. Run the Update workflow or wait for its next scheduled run. + +Local collection accepts the same names in the environment or ignored `.env` file. +Collection stays disabled and the section stays hidden until the resource ID is configured. +Only counts, dates, and collection status appear in the dashboard data; credentials and raw Azure +responses are never published. A failed window retains its previous data, is eligible for retry +after three hours, and does not discard other successfully collected windows. + ## Testing ### Python unit tests diff --git a/gh-pages-template/assets/js/dashboard.js b/gh-pages-template/assets/js/dashboard.js index 9456d7bdf5..3068bd54c4 100644 --- a/gh-pages-template/assets/js/dashboard.js +++ b/gh-pages-template/assets/js/dashboard.js @@ -620,12 +620,66 @@ function renderDocsChart(repos) { }, false), CONFIG); } +// Azure Artifact Signing account totals +function renderAzureSigning(data) { + const section = document.getElementById('azure-signing'); + if (!section) return; + const enabled = Boolean(data && data.status !== 'disabled'); + section.hidden = !enabled; + const nav = document.getElementById('azure-signing-nav'); + if (nav) nav.hidden = !enabled; + if (!enabled) return; + + const summary = document.getElementById('azure-signing-summary'); + summary.replaceChildren(); + const status = document.getElementById('azure-signing-status'); + if (!data.daily.some(point => point.completed !== null)) { + status.textContent = data.status === 'error' + ? 'Signing metrics are temporarily unavailable.' + : 'Azure has not reported signing counts for this period.'; + document.getElementById('chart-azure-signing').hidden = true; + return; + } + const updated = new Date(data.collected_at).toUTCString(); + status.textContent = data.status === 'error' + ? `Some metrics could not be refreshed. Available data last updated: ${updated}.` + : `Metrics collected: ${updated}.`; + for (const [value, label] of [ + [data.month_to_date, `Month to Date (UTC)${data.month_to_date_complete ? '' : ' — partial history'}`], + [data.last_30_days, `Last 30 Days (UTC)${data.last_30_days_complete ? '' : ' — partial history'}`], + ]) { + const card = document.createElement('div'); + card.className = 'col-6 mb-3 text-center'; + const number = document.createElement('h3'); + number.className = 'fw-bold'; + number.textContent = value === null ? 'Unavailable' : value.toLocaleString(); + const caption = document.createElement('small'); + caption.className = 'text-muted'; + caption.textContent = label; + card.append(number, caption); + summary.append(card); + } + document.getElementById('chart-azure-signing').hidden = false; + Plotly.newPlot('chart-azure-signing', [{ + x: data.daily.map(point => point.date), + y: data.daily.map(point => point.completed), + type: 'bar', + marker: { color: '#28a9e6' }, + hovertemplate: '%{x}: %{y} completed requests', + }], themeLayout({ + xaxis: { title: { text: 'Date (UTC)' }, type: 'date' }, + yaxis: { title: { text: 'Completed Requests' }, rangemode: 'tozero' }, + margin: { t: 30, r: 20, b: 60, l: 60 }, + }), CONFIG); +} + // Main async function loadDashboard() { const loadingEl = document.getElementById('loading-msg'); const contentEl = document.getElementById('dashboard-content'); try { - const [repos, prs, metadata, coverageHistory, commitActivity, starHistory, codeScanningHistory] = await Promise.all([ + const [repos, prs, metadata, coverageHistory, commitActivity, starHistory, + codeScanningHistory, azureSigning] = await Promise.all([ fetchJSON('repos.json'), fetchJSON('prs.json'), fetchJSON('metadata.json'), @@ -633,6 +687,7 @@ async function loadDashboard() { fetchJSON('commit_activity.json').catch(() => []), fetchJSON('star_history.json').catch(() => []), fetchJSON('code_scanning_history.json').catch(() => []), + fetchJSON('azure_signing.json').catch(() => null), ]); const active = activeRepos(repos); @@ -671,6 +726,7 @@ async function loadDashboard() { renderLanguageCharts(active); renderCommitActivityChart(commitActivity, active); renderDocsChart(active); + renderAzureSigning(azureSigning); } catch (err) { if (loadingEl) loadingEl.innerHTML = @@ -705,6 +761,7 @@ if (typeof module !== 'undefined' && module.exports) { renderLanguageCharts, renderCommitActivityChart, renderDocsChart, + renderAzureSigning, loadDashboard, }; } diff --git a/gh-pages-template/index.html b/gh-pages-template/index.html index 4f37cd67a6..b0022612fe 100644 --- a/gh-pages-template/index.html +++ b/gh-pages-template/index.html @@ -64,9 +64,21 @@
  • Commit Activity
  • Languages
  • Docs
    • + + + +

    Star Gazers

    diff --git a/src/azure_signing.py b/src/azure_signing.py new file mode 100644 index 0000000000..5808eceb3b --- /dev/null +++ b/src/azure_signing.py @@ -0,0 +1,243 @@ +"""Incrementally collect existing Artifact Signing platform metrics.""" + +import hashlib +import json +import math +import os +import re +from datetime import datetime, timedelta, timezone + +import requests + +from src import helpers +from src.logger import log + +RESOURCE_ID_ENV = 'AZURE_SIGNING_RESOURCE_ID' +CACHE_PATH = ('azure', 'signing.json') +REFRESH_INTERVAL = timedelta(hours=3) +HISTORY_DAYS = 90 +BACKFILL_DAYS = 7 +SETTLE_DAYS = 2 +RESOURCE_ID_PATTERN = re.compile( + r'/subscriptions/[\w-]+/resourceGroups/[\w.()-]+/' + r'providers/Microsoft\.CodeSigning/codeSigningAccounts/[\w-]+', re.IGNORECASE, +) + + +def _load_cache(base_dir: str) -> dict: + """Read cached metrics, or return an unconfigured state.""" + try: + with open(os.path.join(base_dir, *CACHE_PATH)) as f: + data = json.load(f) + if isinstance(data, dict) and isinstance(data.get('daily'), list): + return data + except (OSError, ValueError): + pass + return {'status': 'disabled', 'daily': []} + + +def _dates(start: datetime, end: datetime) -> list[str]: + """Return UTC dates in a half-open interval (including a partial final day).""" + count = math.ceil((end - start).total_seconds() / 86400) + return [(start + timedelta(days=i)).date().isoformat() for i in range(count)] + + +def load_data(base_dir: str) -> dict: + """Publish counts and reporting coverage, without Azure identifiers or cache state.""" + cache = _load_cache(base_dir) + data = {key: cache[key] for key in ('status', 'daily', 'collected_at', 'attempted_at') if key in cache} + if not cache.get('collected_at'): + return data + now = datetime.fromisoformat(cache['collected_at']) + days = {p['date']: p['completed'] for p in cache['daily']} + midnight = now.replace(hour=0, minute=0, second=0, microsecond=0) + today_end = midnight + timedelta(days=1) + for name, start in ( + ('month_to_date', midnight.replace(day=1)), + ('last_30_days', today_end - timedelta(days=30)), + ): + values = [days.get(date) for date in _dates(start, today_end)] + known = [value for value in values if value is not None] + data[name] = sum(known) if known else None + data[f'{name}_complete'] = len(known) == len(values) + return data + + +def _query_windows(cache: dict, now: datetime) -> list[tuple[datetime, datetime]]: + """Refresh unsettled days and backfill one oldest missing contiguous week.""" + midnight = now.replace(hour=0, minute=0, second=0, microsecond=0) + # Include one day that can be finalized after at least two full days of reporting grace. + recent_start = midnight - timedelta(days=SETTLE_DAYS + 1) + finalized = set(cache.get('finalized_dates', [])) + while recent_start.date().isoformat() in finalized: + recent_start += timedelta(days=1) + windows = [(recent_start, now)] + history_start = midnight - timedelta(days=HISTORY_DAYS - 1) + historical_dates = _dates(history_start, midnight - timedelta(days=SETTLE_DAYS + 1)) + missing = [date for date in historical_dates if date not in finalized] + if missing: + start = datetime.fromisoformat(missing[0]).replace(tzinfo=timezone.utc) + end = start + timedelta(days=1) + while (end - start).days < BACKFILL_DAYS and end < midnight - timedelta(days=SETTLE_DAYS + 1): + if end.date().isoformat() in finalized: + break + end += timedelta(days=1) + windows.append((start, end)) + return windows + + +def _get_token() -> str: + """Obtain a management token without retries or logging authentication data.""" + tenant_id = os.environ['AZURE_TENANT_ID'] + if not re.fullmatch(r'[\w.-]+', tenant_id): + raise ValueError('Invalid Azure tenant ID') + response = requests.post( + f'https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token', + data={ + 'client_id': os.environ['AZURE_CLIENT_ID'], + 'client_secret': os.environ['AZURE_CLIENT_SECRET'], + 'grant_type': 'client_credentials', + 'scope': 'https://management.azure.com/.default', + }, + timeout=30, + allow_redirects=False, + ) + if response.status_code != 200: + raise ValueError('Azure authentication failed') + return response.json()['access_token'] + + +def _fetch_metrics(resource_id: str, token: str, start: datetime, end: datetime) -> dict: + """Read one bounded metric window with no automatic retries.""" + response = requests.get( + f'https://management.azure.com{resource_id}/providers/Microsoft.Insights/metrics', + headers={'Authorization': f'Bearer {token}'}, + params={ + 'api-version': '2023-10-01', + 'metricnamespace': 'Microsoft.CodeSigning/codeSigningAccounts', + 'metricnames': 'SignCompleted', + 'aggregation': 'Total', + 'interval': 'P1D', + 'AutoAdjustTimegrain': 'true', + 'timespan': f'{start.isoformat()}/{end.isoformat()}', + }, + timeout=30, + allow_redirects=False, + ) + if response.status_code != 200: + raise ValueError('Azure metrics request failed') + return response.json() + + +def _metric_points(payload: dict): + """Yield samples from successful signing metrics.""" + metrics = [metric for metric in payload['value'] if metric['name']['value'] == 'SignCompleted'] + if not metrics: + raise ValueError('Signing metric missing from response') + for metric in metrics: + if metric.get('errorCode', 'Success') != 'Success': + raise ValueError('Azure reported a metric error') + for series in metric['timeseries']: + yield from series['data'] + + +def _signing_sample(point: dict, start: datetime, end: datetime) -> tuple[str, int] | None: + """Validate a known count and return its UTC date within the requested window.""" + total = point.get('total') + if total is None: + return None + if isinstance(total, bool) or not isinstance(total, (int, float)): + raise ValueError('Invalid signing count') + if not math.isfinite(total) or total < 0 or not float(total).is_integer(): + raise ValueError('Invalid signing count') + timestamp = datetime.fromisoformat(point['timeStamp']) + if timestamp.tzinfo is None: + raise ValueError('Metric timestamp must include a timezone') + timestamp = timestamp.astimezone(timezone.utc) + if start <= timestamp < end: + return timestamp.date().isoformat(), int(total) + return None + + +def _daily_counts(payload: dict, start: datetime, end: datetime) -> dict: + """Sum samples by UTC day; absent telemetry remains unknown rather than zero.""" + days = dict.fromkeys(_dates(start, end)) + for point in _metric_points(payload): + sample = _signing_sample(point, start, end) + if sample is not None: + date, total = sample + days[date] = (days[date] or 0) + total + return days + + +def _collect(cache: dict, resource_id: str, now: datetime) -> dict: + """Merge successful windows by date, retaining finalized history indefinitely.""" + token = _get_token() + days = {p['date']: p['completed'] for p in cache['daily']} + finalized = set(cache.get('finalized_dates', [])) + finalize_before = now.date() - timedelta(days=SETTLE_DAYS) + failed = False + collected_at = cache.get('collected_at') + for start, end in _query_windows(cache, now): + try: + counts = _daily_counts(_fetch_metrics(resource_id, token, start, end), start, end) + except (requests.RequestException, ValueError, KeyError, TypeError): + failed = True + continue + for date, total in counts.items(): + # A temporarily absent current sample must not erase a previously reported count. + if total is not None or date not in days: + days[date] = total + if date < finalize_before.isoformat(): + finalized.add(date) + collected_at = now.isoformat() + return { + **cache, + 'status': 'error' if failed else 'ready', + 'collected_at': collected_at, + 'daily': [{'date': date, 'completed': total} for date, total in sorted(days.items())], + 'finalized_dates': sorted(finalized), + } + + +def _validate_configuration(resource_id: str) -> None: + """Require a signing account resource ID and credentials before collecting.""" + if not RESOURCE_ID_PATTERN.fullmatch(resource_id): + raise ValueError('Invalid Artifact Signing resource ID') + if not all(os.getenv(key) for key in ('AZURE_TENANT_ID', 'AZURE_CLIENT_ID', 'AZURE_CLIENT_SECRET')): + raise ValueError('Azure credentials are missing') + + +def _attempt_is_recent(cache: dict, now: datetime) -> bool: + """Throttle successful and failed attempts within the refresh interval.""" + if not cache.get('attempted_at'): + return False + attempted_at = datetime.fromisoformat(cache['attempted_at']) + return timedelta(0) <= now - attempted_at < REFRESH_INTERVAL + + +def update(base_dir: str) -> None: + """Refresh optional metrics; preserve successful data and throttle every attempt.""" + if os.getenv('DASHBOARD_AZURE_SIGNING_CACHE_ONLY') == 'true': + return + resource_id = os.getenv(RESOURCE_ID_ENV, '').strip().rstrip('/') + cache = _load_cache(base_dir) + now = datetime.now(timezone.utc) + if not resource_id: + data = {'status': 'disabled', 'daily': []} + else: + resource_hash = hashlib.sha256(resource_id.lower().encode()).hexdigest() + if cache.get('resource_hash') != resource_hash: + cache = {'status': 'disabled', 'daily': []} + try: + _validate_configuration(resource_id) + if _attempt_is_recent(cache, now): + return + data = _collect(cache, resource_id, now) + except (requests.RequestException, ValueError, KeyError, TypeError): + data = {**cache, 'status': 'error'} + if data['status'] == 'error': + log.warning('Azure signing metrics unavailable; check configuration, permissions, and connectivity.') + data.update(resource_hash=resource_hash, attempted_at=now.isoformat()) + + helpers.write_json_files(file_path=os.path.join(base_dir, 'azure', 'signing'), data=data) diff --git a/src/builder.py b/src/builder.py index 472072affe..1c415d502a 100644 --- a/src/builder.py +++ b/src/builder.py @@ -9,6 +9,7 @@ from src import BASE_DIR, TEMPLATE_DIR from src import helpers from src import pr_metrics +from src import azure_signing from src.logger import log @@ -306,6 +307,7 @@ def write_json(filename, data): write_json('star_history.json', star_history) write_json('code_scanning_history.json', code_scanning_history) write_json('pr_metrics.json', pr_metric_caches) + write_json('azure_signing.json', azure_signing.load_data(BASE_DIR)) now = datetime.now(timezone.utc) write_json('metadata.json', { 'updated_at': now.isoformat(), diff --git a/src/updater.py b/src/updater.py index 31d1c7f3a6..0a75492f0f 100644 --- a/src/updater.py +++ b/src/updater.py @@ -17,6 +17,7 @@ from src import BASE_DIR from src import helpers from src import pr_metrics +from src import azure_signing from src.logger import log COMMIT_ACTIVITY_READY = 'ready' @@ -1114,6 +1115,7 @@ def append_thread_if_env_set( def update(): # Threads that are fully independent of each other and of GitHub data. independent_threads = [] + independent_threads.append(Thread(name='azure-signing', target=azure_signing.update, kwargs={'base_dir': BASE_DIR})) append_thread_if_env_set( env_vars=['DASHBOARD_AUR_REPOS'], diff --git a/tests/dashboard.test.js b/tests/dashboard.test.js index f77c076f26..a2642044fa 100644 --- a/tests/dashboard.test.js +++ b/tests/dashboard.test.js @@ -26,6 +26,12 @@ function buildDom() {
    + + `; } @@ -279,6 +285,44 @@ describe('dashboard.js', () => { expect(globalThis.Plotly.newPlot).toHaveBeenCalled(); }); + test('renderAzureSigning hides unconfigured metrics and tolerates missing containers', () => { + mod.renderAzureSigning(null); + expect(document.getElementById('azure-signing').hidden).toBe(true); + mod.renderAzureSigning({ status: 'disabled', daily: [] }); + document.getElementById('azure-signing-nav').remove(); + mod.renderAzureSigning(null); + document.getElementById('azure-signing').remove(); + mod.renderAzureSigning(null); + expect(globalThis.Plotly.newPlot).not.toHaveBeenCalled(); + }); + + test('renderAzureSigning distinguishes unknown data, zero usage, and partial history', () => { + mod.renderAzureSigning({ status: 'ready', daily: [{ date: '2026-10-01', completed: null }] }); + expect(document.getElementById('azure-signing-status').textContent).toContain('has not reported'); + expect(document.getElementById('chart-azure-signing').hidden).toBe(true); + mod.renderAzureSigning({ status: 'error', daily: [] }); + expect(document.getElementById('azure-signing-status').textContent).toContain('temporarily unavailable'); + const data = { + status: 'ready', collected_at: '2026-10-01T12:00:00Z', + daily: [{ date: '2026-10-01', completed: 0 }], + month_to_date: 0, last_30_days: null, + month_to_date_complete: true, last_30_days_complete: false, + }; + mod.renderAzureSigning(data); + expect(document.getElementById('azure-signing').hidden).toBe(false); + expect(document.getElementById('azure-signing-nav').hidden).toBe(false); + expect(document.getElementById('chart-azure-signing').hidden).toBe(false); + expect(document.getElementById('azure-signing-summary').textContent).toContain('Unavailable'); + expect(globalThis.Plotly.newPlot.mock.calls[0][1][0].y).toEqual([0]); + data.status = 'error'; + data.month_to_date_complete = false; + data.last_30_days_complete = true; + mod.renderAzureSigning(data); + expect(document.getElementById('azure-signing-status').textContent).toContain('could not be refreshed'); + expect(document.querySelectorAll('#azure-signing-summary h3')).toHaveLength(2); + expect(document.getElementById('azure-signing-summary').textContent).toContain('partial history'); + }); + test('renderCoverageChart handles empty and non-empty', () => { mod.renderCoverageChart([{ name: 'x', coverage: 0 }]); expect(globalThis.Plotly.newPlot).toHaveBeenCalledTimes(0); @@ -348,6 +392,12 @@ describe('dashboard.js', () => { const commits = [{ repo: 'repo-a', week: '2026-01-01', total: 1 }]; const stars = [{ repo: 'repo-a', date: '2026-01-01', stars: 3 }]; const codeScanningHistory = [{ repo: 'repo-a', date: '2026-03-19', open: 4 }]; + const azureSigning = { + status: 'ready', collected_at: '2026-03-20T00:00:00Z', + daily: [{ date: '2026-03-19', completed: 12 }], + month_to_date: 12, last_30_days: 12, + month_to_date_complete: false, last_30_days_complete: false, + }; globalThis.fetch.mockImplementation(async (url) => { if (url.endsWith('repos.json')) return { ok: true, json: async () => repos }; @@ -357,12 +407,15 @@ describe('dashboard.js', () => { if (url.endsWith('commit_activity.json')) return { ok: true, json: async () => commits }; if (url.endsWith('star_history.json')) return { ok: true, json: async () => stars }; if (url.endsWith('code_scanning_history.json')) return { ok: true, json: async () => codeScanningHistory }; + if (url.endsWith('azure_signing.json')) return { ok: true, json: async () => azureSigning }; return { ok: false, status: 404 }; }); await mod.loadDashboard(); expect(document.getElementById('loading-msg').style.display).toBe('none'); expect(document.getElementById('dashboard-content').style.display).toBe(''); + expect(document.getElementById('azure-signing').hidden).toBe(false); + expect(globalThis.Plotly.newPlot.mock.calls.some(([id]) => id === 'chart-azure-signing')).toBe(true); document.getElementById('loading-msg').remove(); document.getElementById('dashboard-content').remove(); diff --git a/tests/unit/test_azure_signing.py b/tests/unit/test_azure_signing.py new file mode 100644 index 0000000000..7693c81eca --- /dev/null +++ b/tests/unit/test_azure_signing.py @@ -0,0 +1,347 @@ +"""Prove incremental signing collection, cost bounds, and reporting accuracy.""" + +import hashlib +import json +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace + +import pytest +import requests + +from src import azure_signing as signing + +RESOURCE = '/subscriptions/sub/resourceGroups/rg/providers/Microsoft.CodeSigning/codeSigningAccounts/account' +NOW = datetime(2026, 10, 7, 12, tzinfo=timezone.utc) + + +def payload(*points, error='Success'): + return {'value': [ + {'name': {'value': 'Other'}, 'timeseries': []}, + {'name': {'value': 'SignCompleted'}, 'errorCode': error, 'timeseries': [{'data': list(points)}]}, + ]} + + +def point(date, count): + return {'timeStamp': date, 'total': count} + + +def write_cache(tmp_path, data): + path = tmp_path / 'azure' / 'signing.json' + path.parent.mkdir(exist_ok=True) + path.write_text(json.dumps(data), encoding='utf-8') + return path + + +@pytest.fixture +def configured(monkeypatch): + for key, value in { + signing.RESOURCE_ID_ENV: RESOURCE, + 'AZURE_TENANT_ID': 'tenant', + 'AZURE_CLIENT_ID': 'client', + 'AZURE_CLIENT_SECRET': 'secret-value', + }.items(): + monkeypatch.setenv(key, value) + + class Clock(datetime): + current = NOW + + @classmethod + def now(cls, tz=None): + return cls.current + + monkeypatch.setattr(signing, 'datetime', Clock) + return Clock + + +def test_disabled_and_invalid_cache(tmp_path, monkeypatch): + monkeypatch.delenv(signing.RESOURCE_ID_ENV, raising=False) + assert signing.load_data(str(tmp_path)) == {'status': 'disabled', 'daily': []} + path = write_cache(tmp_path, {'status': 'ready', 'daily': [], 'secret': 'must-not-publish'}) + assert signing.load_data(str(tmp_path)) == {'status': 'ready', 'daily': []} + for contents in ('invalid json', '[]', '{}'): + path.write_text(contents) + assert signing.load_data(str(tmp_path)) == {'status': 'disabled', 'daily': []} + signing.update(str(tmp_path)) + assert json.loads(path.read_text()) == {'status': 'disabled', 'daily': []} + + +@pytest.mark.parametrize('has_cache', [False, True]) +def test_cache_only_preview_preserves_data_without_requests( + tmp_path, monkeypatch, configured, requests_mock, has_cache): + monkeypatch.setenv('DASHBOARD_AZURE_SIGNING_CACHE_ONLY', 'true') + path = tmp_path / 'azure' / 'signing.json' + if has_cache: + write_cache(tmp_path, { + 'status': 'ready', 'resource_hash': hashlib.sha256(RESOURCE.lower().encode()).hexdigest(), + 'collected_at': (NOW - timedelta(days=1)).isoformat(), + 'attempted_at': (NOW - timedelta(days=1)).isoformat(), + 'daily': [{'date': '2026-10-06', 'completed': 7}], + 'finalized_dates': ['2026-10-06'], + }) + original = path.read_bytes() + + signing.update(str(tmp_path)) + + assert requests_mock.call_count == 0 + if has_cache: + assert path.read_bytes() == original + assert signing.load_data(str(tmp_path))['daily'] == [{'date': '2026-10-06', 'completed': 7}] + else: + assert not path.exists() + assert signing.load_data(str(tmp_path)) == {'status': 'disabled', 'daily': []} + + +def test_public_totals_and_partial_history(tmp_path): + write_cache(tmp_path, { + 'status': 'ready', 'resource_hash': 'private-cache-key', 'finalized_dates': ['2026-09-30'], + 'collected_at': NOW.isoformat(), 'attempted_at': NOW.isoformat(), + 'daily': [ + {'date': '2026-08-01', 'completed': 100}, + {'date': '2026-09-30', 'completed': 3}, + {'date': '2026-10-06', 'completed': None}, + {'date': '2026-10-07', 'completed': 0}, + ], + }) + data = signing.load_data(str(tmp_path)) + assert 'resource_hash' not in data + assert 'finalized_dates' not in data + assert data['month_to_date'] == 0 + assert data['last_30_days'] == 3 + assert not data['month_to_date_complete'] + assert not data['last_30_days_complete'] + + # A new month must not include the previous month in its total. + write_cache(tmp_path, { + 'status': 'ready', 'collected_at': '2026-11-01T12:00:00+00:00', + 'daily': [{'date': '2026-10-31', 'completed': 9}], + }) + assert signing.load_data(str(tmp_path))['month_to_date'] is None + write_cache(tmp_path, { + 'status': 'ready', 'collected_at': '2026-10-31T12:00:00+00:00', + 'daily': [{'date': '2026-10-31', 'completed': 9}], + }) + assert signing.load_data(str(tmp_path))['month_to_date'] == 9 + + +def test_query_windows_never_cross_finalized_history(): + current, backfill = signing._query_windows({}, NOW) + assert current == (datetime(2026, 10, 4, tzinfo=timezone.utc), NOW) + assert (backfill[1] - backfill[0]).days == 7 + assert (NOW.date() - backfill[0].date()).days == 89 + oldest = backfill[0] + finalized = [(oldest + timedelta(days=i)).date().isoformat() for i in (0, 2)] + ['2026-10-04'] + current, backfill = signing._query_windows({'finalized_dates': finalized}, NOW) + assert current[0].date().isoformat() == '2026-10-05' + assert backfill == (oldest + timedelta(days=1), oldest + timedelta(days=2)) + + finalized = signing._dates(oldest, datetime(2026, 10, 5, tzinfo=timezone.utc)) + assert signing._query_windows({'finalized_dates': finalized}, NOW) == [current] + finalized.remove('2026-10-03') + assert signing._query_windows({'finalized_dates': finalized}, NOW)[1] == ( + datetime(2026, 10, 3, tzinfo=timezone.utc), datetime(2026, 10, 4, tzinfo=timezone.utc), + ) + + +def test_daily_counts_adjusted_grain_timezone_and_unknown_samples(): + start = datetime(2026, 10, 4, tzinfo=timezone.utc) + data = payload( + point('2026-10-04T00:00:00Z', 2), point('2026-10-04T01:00:00Z', 3), + point('2026-10-04T23:30:00-02:00', 4), point('2026-10-06T00:00:00Z', None), + point('2026-10-07T00:00:00Z', 0), point('2026-10-03T23:59:00Z', 99), + point(NOW.isoformat(), 99), + ) + data['value'][1]['timeseries'].append({'data': [point('2026-10-04T03:00:00Z', 1)]}) + assert signing._daily_counts(data, start, NOW) == { + '2026-10-04': 6, '2026-10-05': 4, '2026-10-06': None, '2026-10-07': 0, + } + assert signing._dates(start, start) == [] + assert signing._daily_counts(payload(), start, NOW)['2026-10-04'] is None + + +@pytest.mark.parametrize('count', [True, '3', float('nan'), float('inf'), -1, 0.5]) +def test_invalid_count_is_rejected(count): + data = payload(point('2026-10-07T00:00:00Z', count)) + start = NOW - timedelta(days=1) + with pytest.raises(ValueError, match='Invalid signing count'): + signing._daily_counts(data, start, NOW) + + +def test_unavailable_or_malformed_metrics_are_not_finalized(): + start = NOW - timedelta(days=1) + for data in ({'value': []}, payload(error='Error'), payload(point('2026-10-07T00:00:00', 1))): + with pytest.raises(ValueError): + signing._daily_counts(data, start, NOW) + + +def test_http_contract_and_no_credential_disclosure(requests_mock, configured): + token_url = 'https://login.microsoftonline.com/tenant/oauth2/v2.0/token' + metrics_url = f'https://management.azure.com{RESOURCE}/providers/Microsoft.Insights/metrics' + requests_mock.post(token_url, json={'access_token': 'private-token'}) + requests_mock.get(metrics_url, json=payload()) + token = signing._get_token() + start = NOW - timedelta(days=1) + assert signing._fetch_metrics(RESOURCE, token, start, NOW) == payload() + assert requests_mock.call_count == 2 + request = requests_mock.last_request + assert request.headers['Authorization'] == 'Bearer private-token' + assert request.qs['metricnames'] == ['signcompleted'] + assert request.qs['aggregation'] == ['total'] + assert request.qs['timespan'] == [f'{start.isoformat()}/{NOW.isoformat()}'.lower()] + assert 'private-token' not in request.url + + requests_mock.post(token_url, status_code=401, text='private-token secret-value') + with pytest.raises(ValueError, match='Azure authentication failed') as error: + signing._get_token() + assert 'secret-value' not in str(error.value) + requests_mock.get(metrics_url, status_code=403) + with pytest.raises(ValueError, match='Azure metrics request failed'): + signing._fetch_metrics(RESOURCE, token, start, NOW) + + +def test_bad_tenant_cannot_redirect_credentials(monkeypatch, configured): + monkeypatch.setenv('AZURE_TENANT_ID', 'tenant/../../evil') + with pytest.raises(ValueError, match='Invalid Azure tenant ID'): + signing._get_token() + + +def test_incremental_refresh_rollover_and_archival(monkeypatch, tmp_path, configured): + calls = [] + count = 2 + monkeypatch.setattr(signing, '_get_token', lambda: 'token') + + def fetch(resource_id, token, start, end): + calls.append((start, end)) + return payload(*(point(f'{date}T00:00:00Z', count) for date in signing._dates(start, end))) + + monkeypatch.setattr(signing, '_fetch_metrics', fetch) + signing.update(str(tmp_path)) + first = signing._load_cache(str(tmp_path)) + assert len(calls) == 2 + assert len(first['finalized_dates']) == 8 + assert '2026-10-04' in first['finalized_dates'] + assert '2026-10-05' not in first['finalized_dates'] + + # Repeated builds within the polling interval consume no Azure requests. + signing.update(str(tmp_path)) + assert len(calls) == 2 + + configured.current += timedelta(hours=3) + count = 5 + signing.update(str(tmp_path)) + second = signing._load_cache(str(tmp_path)) + assert len(calls) == 4 + assert calls[2][0].date().isoformat() == '2026-10-05' + first_week_end = calls[1][1] + assert calls[3][0] == first_week_end + days = {p['date']: p['completed'] for p in second['daily']} + assert days['2026-10-04'] == 2 # Archived day stays unchanged. + assert days['2026-10-07'] == 5 # Replacement, never 2 + 5. + + configured.current = datetime(2026, 10, 8, 12, tzinfo=timezone.utc) + signing.update(str(tmp_path)) + third = signing._load_cache(str(tmp_path)) + assert '2026-10-05' in third['finalized_dates'] + assert len(third['daily']) > len(second['daily']) + + # Old history survives beyond Azure's retention window and the current query skips it. + third['daily'].insert(0, {'date': '2025-01-01', 'completed': 50}) + third['finalized_dates'] = signing._dates( + configured.current.replace(hour=0) - timedelta(days=89), + configured.current.replace(hour=0) - timedelta(days=2), + ) + write_cache(tmp_path, third) + configured.current += timedelta(hours=3) + before = len(calls) + signing.update(str(tmp_path)) + assert len(calls) == before + 1 + assert signing._load_cache(str(tmp_path))['daily'][0] == {'date': '2025-01-01', 'completed': 50} + + +def test_failed_backfill_keeps_current_and_retries_gap(monkeypatch, tmp_path, configured): + monkeypatch.setattr(signing, '_get_token', lambda: 'token') + windows = [] + + def fetch(resource_id, token, start, end): + windows.append((start, end)) + if end != configured.current: + raise requests.Timeout('secret-value private-token') + return payload(point('2026-10-07T00:00:00Z', 8)) + + monkeypatch.setattr(signing, '_fetch_metrics', fetch) + signing.update(str(tmp_path)) + data = signing.load_data(str(tmp_path)) + assert data['status'] == 'error' + assert data['month_to_date'] == 8 + assert not data['month_to_date_complete'] + assert 'secret-value' not in json.dumps(data) + assert windows[1][0].date().isoformat() not in signing._load_cache(str(tmp_path))['finalized_dates'] + configured.current += timedelta(hours=3) + signing.update(str(tmp_path)) + assert windows[3] == windows[1] + + +def test_absent_recent_samples_preserve_previous_counts(monkeypatch): + monkeypatch.setattr(signing, '_get_token', lambda: 'token') + monkeypatch.setattr(signing, '_fetch_metrics', lambda *args: payload()) + data = signing._collect({'daily': [{'date': '2026-10-07', 'completed': 9}]}, RESOURCE, NOW) + assert next(p['completed'] for p in data['daily'] if p['date'] == '2026-10-07') == 9 + assert next(p['completed'] for p in data['daily'] if p['date'] == '2026-10-06') is None + + +def test_current_failure_can_still_backfill(monkeypatch): + monkeypatch.setattr(signing, '_get_token', lambda: 'token') + + def fetch(resource_id, token, start, end): + if end == NOW: + return {'value': []} + return payload(point(start.isoformat(), 3)) + + monkeypatch.setattr(signing, '_fetch_metrics', fetch) + result = signing._collect({'daily': []}, RESOURCE, NOW) + assert result['status'] == 'error' + assert result['collected_at'] == NOW.isoformat() + assert len(result['finalized_dates']) == 7 + + +@pytest.mark.parametrize('failure', ['credentials', 'resource', 'authentication', 'cache_timestamp']) +def test_update_failures_preserve_matching_cache(monkeypatch, tmp_path, configured, failure): + cache = { + 'status': 'ready', 'daily': [{'date': '2026-10-01', 'completed': 4}], + 'collected_at': (NOW - timedelta(days=1)).isoformat(), + 'resource_hash': hashlib.sha256(RESOURCE.lower().encode()).hexdigest(), + } + if failure == 'credentials': + monkeypatch.delenv('AZURE_CLIENT_SECRET') + elif failure == 'resource': + monkeypatch.setenv(signing.RESOURCE_ID_ENV, 'https://evil.example/resource') + elif failure == 'cache_timestamp': + cache['attempted_at'] = 'not-a-date' + else: + def bad_token(): + raise requests.Timeout('secret-value') + monkeypatch.setattr(signing, '_get_token', bad_token) + path = write_cache(tmp_path, cache) + signing.update(str(tmp_path)) + saved = json.loads(path.read_text()) + assert saved['status'] == 'error' + assert saved['attempted_at'] == NOW.isoformat() + if failure == 'resource': + assert saved['daily'] == [] # Never reuse another account's history. + else: + assert saved['daily'] == cache['daily'] + signing.update(str(tmp_path)) + + +def test_request_options_disable_redirects_and_retries(monkeypatch, configured): + calls = [] + + def request(url, **kwargs): + calls.append((url, kwargs)) + return SimpleNamespace(status_code=200, json=lambda: {'access_token': 'token'}) + + monkeypatch.setattr(signing.requests, 'post', request) + monkeypatch.setattr(signing.requests, 'get', request) + signing._get_token() + signing._fetch_metrics(RESOURCE, 'token', NOW - timedelta(days=1), NOW) + assert len(calls) == 2 + assert all(options['timeout'] == 30 and options['allow_redirects'] is False for _, options in calls) diff --git a/tests/unit/test_builder.py b/tests/unit/test_builder.py index 4f27ced2fd..d038648401 100644 --- a/tests/unit/test_builder.py +++ b/tests/unit/test_builder.py @@ -242,6 +242,18 @@ def now(cls, tz=None): metrics = json.loads((data_dir / 'pr_metrics.json').read_text(encoding='utf-8')) assert metrics == {'demo': pr_metric_cache} + assert json.loads((data_dir / 'azure_signing.json').read_text()) == {'status': 'disabled', 'daily': []} + _write_json(base / 'azure' / 'signing.json', { + 'status': 'ready', 'collected_at': fixed_now.isoformat(), + 'resource_hash': 'internal-cache-state', 'finalized_dates': ['2026-01-04'], + 'daily': [{'date': '2026-01-04', 'completed': 8}], + }) + builder.build() + signing = json.loads((data_dir / 'azure_signing.json').read_text()) + assert signing['month_to_date'] == 8 + assert not signing['month_to_date_complete'] + assert 'resource_hash' not in signing + assert 'finalized_dates' not in signing assert 'Pull Request Metrics' in (template / 'pr-metrics' / 'index.md').read_text(encoding='utf-8') assert 'PR Metrics - demo' in (template / 'pr-metrics' / 'demo.md').read_text(encoding='utf-8') diff --git a/tests/unit/test_updater.py b/tests/unit/test_updater.py index d4520d4d90..9b5efef10b 100644 --- a/tests/unit/test_updater.py +++ b/tests/unit/test_updater.py @@ -765,8 +765,10 @@ def join(self): updater.update() assert 'github' in started + assert 'azure-signing' in started assert 'codecov' in started assert 'github' in joined + assert 'azure-signing' in joined assert 'codecov' in joined assert 'activate' in started assert 'deactivate' in joined