diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..5076716 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,197 @@ +name: CI + +on: + push: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + CARGO_BUILD_JOBS: 3 + CARGO_TERM_COLOR: always + +jobs: + style: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@7e38f4b43b4db5c8dd498af069a4f6196df1d067 + with: + toolchain: stable + components: rustfmt,clippy + - name: Install tools + run: | + sudo apt-get update + sudo apt-get install -y nasm pipx + pipx install clang-format==23.1.1 + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + - name: Format and lint + run: | + ./scripts/stylecheck.sh + git diff --exit-code + + tests: + runs-on: ubuntu-24.04 + timeout-minutes: 25 + strategy: + fail-fast: false + matrix: + asm: ['true', 'false'] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: halidecx/wpd-test-data + ref: f8c31341db3ab4400f048a96e7b3736fed303b34 + path: wpd-test-data + persist-credentials: false + - uses: dtolnay/rust-toolchain@7e38f4b43b4db5c8dd498af069a4f6196df1d067 + with: + toolchain: stable + - name: Install tools + run: | + sudo apt-get update + sudo apt-get install -y pipx ninja-build nasm cmake + pipx install meson==1.12.1 + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + - name: Tests, C API, parity and checkasm + env: + ASM: ${{ matrix.asm }} + run: | + meson setup build -Denable_asm="$ASM" -Dtrim_dsp=false \ + -Dtestdata_tests=true -Dlibwebp=subproject -Dwuffs=disabled + if [ "$ASM" = true ]; then + meson test -C build --list | grep -q checkasm + fi + meson test -C build --print-errorlogs --num-processes 3 + + scripts: + runs-on: ubuntu-24.04 + timeout-minutes: 45 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: halidecx/wpd-test-data + ref: f8c31341db3ab4400f048a96e7b3736fed303b34 + path: wpd-test-data + persist-credentials: false + - uses: dtolnay/rust-toolchain@7e38f4b43b4db5c8dd498af069a4f6196df1d067 + with: + toolchain: stable + - name: Install tools + run: | + sudo apt-get update + sudo apt-get install -y pipx ninja-build nasm cmake webp + pipx install meson==1.12.1 + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + - name: Build assembly and fallback tools + run: | + meson setup build -Dtrim_dsp=false -Dtestdata_tests=true \ + -Dlibwebp=subproject -Dwuffs=disabled + meson compile -C build -j 3 libwebpdec + meson setup build-noasm -Denable_asm=false \ + -Dlibwebp=disabled -Dwuffs=disabled + meson compile -C build-noasm -j 3 + - name: Existing correctness checks + run: | + ./scripts/testdata.sh + ./scripts/animcheck.sh + ./scripts/threadcheck.sh + ./scripts/md5check.sh ./build-noasm/wpd ./build/wpd + ./scripts/clicheck.sh ./build-noasm/wpd ./build/wpd + ./scripts/rac32.sh --print-errorlogs --num-processes 3 + + c-sanitizers: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: halidecx/wpd-test-data + ref: f8c31341db3ab4400f048a96e7b3736fed303b34 + path: wpd-test-data + persist-credentials: false + - uses: dtolnay/rust-toolchain@7e38f4b43b4db5c8dd498af069a4f6196df1d067 + with: + toolchain: stable + - name: Install tools + run: | + sudo apt-get update + sudo apt-get install -y pipx ninja-build nasm cmake + pipx install meson==1.12.1 + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + - name: C harnesses and damaged-input smoke + run: | + ./scripts/sanitize.sh --print-errorlogs --num-processes 3 + ./scripts/fuzz.sh 8 wpd-test-data/odd_lossy.webp \ + wpd-test-data/odd_a_lossy.webp wpd-test-data/palette_rgb.webp \ + wpd-test-data/mixed_codecs.webp + + nightly: + runs-on: ubuntu-24.04 + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + check: [rustsan, tsan, miri, fuzz] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: halidecx/wpd-test-data + ref: f8c31341db3ab4400f048a96e7b3736fed303b34 + path: wpd-test-data + persist-credentials: false + - uses: dtolnay/rust-toolchain@7e38f4b43b4db5c8dd498af069a4f6196df1d067 + with: + toolchain: nightly + components: rust-src,miri + - name: Install tools + run: | + sudo apt-get update + sudo apt-get install -y pipx ninja-build nasm cmake clang + pipx install meson==1.12.1 + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + - name: Nightly check + env: + CHECK: ${{ matrix.check }} + run: | + case "$CHECK" in + rustsan) + meson setup build -Dlibwebp=disabled -Dwuffs=disabled + meson compile -C build -j 3 + ./scripts/rustsan.sh + ;; + tsan) ./scripts/tsan.sh ;; + miri) ./scripts/miri.sh --lib container::tests ;; + fuzz) + cargo install cargo-fuzz --version 0.13.2 --locked + ./scripts/fuzz-smoke.sh + ;; + esac + - name: Save fuzz failures + if: failure() && matrix.check == 'fuzz' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: fuzz-failures + path: fuzz/artifacts + if-no-files-found: ignore diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..c83f2b6 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,13 @@ +# Changelog + +## Unreleased — 0.2.0 + +### Build and CI + +- Repair the end-to-end fuzz target's decoder options initializer so all four + coverage-guided targets build again. +- Correct the optional Wuffs dependency name in the Meson build. +- Add one GitHub Actions workflow for tests, rustfmt/clippy, fuzz target builds, + seeded smoke runs, and the existing correctness and sanitizer checks. +- Bound fuzz-harness pictures to one megapixel so mutated dimensions fit the + smoke run's memory budget without changing decoder limits. diff --git a/README.md b/README.md index 4da01c4..a789d16 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,24 @@ Test data is maintained at into the `wpd/` root. `./scripts/testdata.sh` runs end-to-end assembly and fallback checks. +GitHub Actions runs assembly and fallback tests, checkasm, format/lint checks, +the correctness scripts, C and Rust sanitizers, and a container Miri smoke +check. The corpus revision is pinned in `.github/workflows/ci.yml`. CI compares +the assembly and fallback tools with `md5check.sh` and `clicheck.sh`; comparing +an older release still needs an explicit baseline binary. Timing scripts +(`bench.sh` and `cmpbench.sh`) remain manual because shared CI runners do not +provide stable performance measurements. + +`./scripts/fuzz-smoke.sh [seconds-per-target] [corpus-directory]` builds every +fuzz target and runs each for 15 seconds by default. It requires nightly Rust, +Python 3 and cargo-fuzz 0.13.2. It derives container and raw VP8/VP8L seeds from +the test corpus without changing its WebP files, and leaves generated seeds and +failure artifacts under `fuzz/`. Longer fuzzing and the full safe-core Miri +suite (`./scripts/miri.sh`) are useful local checks before releases. The +decoding harnesses bound pictures to 1,048,576 pixels so mutated dimensions fit +the smoke run's memory budget; larger pictures remain in ordinary corpus tests. +This is a harness limit, not a decoder limit. + For libwebp parity testing and benchmarking against alternative WebP decoders, build the optional third-party test binaries: diff --git a/fuzz/budget.rs b/fuzz/budget.rs new file mode 100644 index 0000000..fe1e0c6 --- /dev/null +++ b/fuzz/budget.rs @@ -0,0 +1,13 @@ +pub const MAX_PIXELS: u32 = 1 << 20; + +pub fn fits(data: &[u8]) -> bool { + // Keep malformed headers in coverage; only a successfully read size can + // exceed the harness budget. Raw codecs have no decoder options limit. + wpd::api::info(data).map_or(true, |info| { + wpd::api::Options { + frame_size_limit: MAX_PIXELS, + ..Default::default() + } + .fits(info.width, info.height) + }) +} diff --git a/fuzz/fuzz_targets/container.rs b/fuzz/fuzz_targets/container.rs index ff1825b..85373cf 100644 --- a/fuzz/fuzz_targets/container.rs +++ b/fuzz/fuzz_targets/container.rs @@ -1,4 +1,3 @@ - #![no_main] use libfuzzer_sys::fuzz_target; diff --git a/fuzz/fuzz_targets/e2e.rs b/fuzz/fuzz_targets/e2e.rs index 9832126..70c0a75 100644 --- a/fuzz/fuzz_targets/e2e.rs +++ b/fuzz/fuzz_targets/e2e.rs @@ -11,6 +11,9 @@ use wpd_capi::decoder::{wpd_decode_into, WPDOutputBuffer}; use wpd_capi::frame::{WPDFrame, WPDOutputPlane}; use wpd_capi::options::WPDDecoderOptions; +#[path = "../budget.rs"] +mod budget; + const FORMATS: [Format; 16] = [ Format::Yuv420p, Format::Yuva420p, @@ -40,6 +43,7 @@ fn decode_options(data: &[u8]) -> (Options, bool) { let flags = byte(data, 1); let subframe = flags & 4 != 0; let mut options = Options { + frame_size_limit: budget::MAX_PIXELS, n_threads: [1, 2, 3, 8][usize::from(flags >> 6)], bypass_filtering: flags & 8 != 0, no_fancy_upsampling: flags & 16 != 0, @@ -123,6 +127,8 @@ fn decode_external(data: &[u8], options: Options) { flip: i32::from(options.flip), reserved: 0, n_threads: options.n_threads, + reserved2: 0, + frame_size_limit: options.frame_size_limit, }; let mut frame = WPDFrame { struct_size: mem::size_of::(), @@ -154,6 +160,9 @@ fn decode_external(data: &[u8], options: Options) { } fuzz_target!(|data: &[u8]| { + if !budget::fits(data) { + return; + } let Some(&first) = data.first() else { return; }; diff --git a/fuzz/fuzz_targets/vp8.rs b/fuzz/fuzz_targets/vp8.rs index dbe3ec7..f2794f6 100644 --- a/fuzz/fuzz_targets/vp8.rs +++ b/fuzz/fuzz_targets/vp8.rs @@ -1,10 +1,15 @@ - #![no_main] use libfuzzer_sys::fuzz_target; use wpd::vp8::Decoder; +#[path = "../budget.rs"] +mod budget; + fuzz_target!(|data: &[u8]| { + if !budget::fits(data) { + return; + } let mut decoder = Decoder::new(); let _ = decoder.decode_frame(data); diff --git a/fuzz/fuzz_targets/vp8l.rs b/fuzz/fuzz_targets/vp8l.rs index 48367bd..fc91499 100644 --- a/fuzz/fuzz_targets/vp8l.rs +++ b/fuzz/fuzz_targets/vp8l.rs @@ -1,14 +1,19 @@ - #![no_main] use libfuzzer_sys::fuzz_target; use wpd::vp8l::{AlphaDst, Decoder, Target}; +#[path = "../budget.rs"] +mod budget; + fuzz_target!(|data: &[u8]| { if data.len() < 2 { return; } let (head, payload) = data.split_at(2); + if !budget::fits(payload) { + return; + } let mut decoder = Decoder::new(); let alpha_chunk = head[0] & 1 != 0; diff --git a/meson.build b/meson.build index f3eb390..66af0e2 100644 --- a/meson.build +++ b/meson.build @@ -517,7 +517,7 @@ custom_target( message('imagewebpdec: enabled (build explicitly with target imagewebpdec)') wuffs_dep = dependency( - '', + 'wuffs', fallback: ['wuffs', 'wuffs_dep'], required: get_option('wuffs'), ) diff --git a/scripts/fuzz-smoke.sh b/scripts/fuzz-smoke.sh new file mode 100755 index 0000000..9ae87e1 --- /dev/null +++ b/scripts/fuzz-smoke.sh @@ -0,0 +1,65 @@ +#!/bin/bash -eu + +SECONDS_PER_TARGET="${1:-15}" +CORPUS="${2:-wpd-test-data}" + +case "$SECONDS_PER_TARGET" in + ''|*[!0-9]*|0*) echo "fuzz-smoke.sh: seconds must be positive" >&2; exit 1 ;; +esac + +# Keep generated seeds separate from saved developer corpora and artifacts. +mkdir -p fuzz/corpus/ci/{container,e2e,vp8,vp8l} fuzz/artifacts +python3 - "$CORPUS" <<'PY' +from pathlib import Path +import sys + +corpus = Path(sys.argv[1]) +out = Path("fuzz/corpus/ci") +files = sorted(corpus.glob("*.webp")) +if not files: + sys.exit(f"no WebP files found in {corpus}") + +def chunks(data): + offset = 0 + while offset + 8 <= len(data): + tag = data[offset:offset + 4] + size = int.from_bytes(data[offset + 4:offset + 8], "little") + end = offset + 8 + size + if end > len(data): + break + payload = data[offset + 8:end] + if tag == b"ANMF": + yield from chunks(payload[16:]) + else: + yield tag, payload + offset = end + (size & 1) + +for path in files: + data = path.read_bytes() + # Large seeds make a smoke run spend its budget replaying images. + if len(data) > 65536: + continue + for target in ("container", "e2e"): + (out / target / path.stem).write_bytes(data) + for index, (tag, payload) in enumerate(chunks(data[12:])): + name = f"{path.stem}-{index}" + if tag == b"VP8 ": + (out / "vp8" / name).write_bytes(payload) + elif tag == b"VP8L": + # The target uses two leading bytes to select ARGB and threading. + for threads in (0, 1): + (out / "vp8l" / f"{name}-{threads}").write_bytes( + bytes((0, threads)) + payload) + +for target in ("container", "e2e", "vp8", "vp8l"): + if not any((out / target).iterdir()): + sys.exit(f"no seeds prepared for {target}") +PY + +# With no target argument cargo-fuzz builds every declared target. +cargo +nightly fuzz build -O --debug-assertions +for target in container vp8 vp8l e2e; do + cargo +nightly fuzz run -O --debug-assertions "$target" \ + "fuzz/corpus/ci/$target" -- -max_total_time="$SECONDS_PER_TARGET" \ + -max_len=65536 -timeout=5 -rss_limit_mb=1024 -seed=1 +done