diff --git a/src/support/file_lock.h b/src/support/file_lock.h new file mode 100644 index 00000000000..9f42a69fff4 --- /dev/null +++ b/src/support/file_lock.h @@ -0,0 +1,59 @@ +/* + * Copyright 2026 WebAssembly Community Group participants + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// +// RAII file lock for cross-process synchronization. +// + +#ifndef wasm_support_file_lock_h +#define wasm_support_file_lock_h + +#include + +#ifndef _WIN32 +#include +#include +#include +#endif + +namespace wasm { + +struct FileLock { +#ifndef _WIN32 + int fd = -1; + FileLock(const std::string& path) { + fd = open(path.c_str(), O_RDWR | O_CREAT, 0666); + if (fd >= 0) { + flock(fd, LOCK_EX); + } + } + ~FileLock() { + if (fd >= 0) { + flock(fd, LOCK_UN); + close(fd); + } + } +#else + FileLock(const std::string&) {} +#endif + + FileLock(const FileLock&) = delete; + FileLock& operator=(const FileLock&) = delete; +}; + +} // namespace wasm + +#endif // wasm_support_file_lock_h diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt index 91fea46b1df..4a24d506ec0 100644 --- a/src/tools/CMakeLists.txt +++ b/src/tools/CMakeLists.txt @@ -1,6 +1,7 @@ include_directories(fuzzing) FILE(GLOB fuzzing_HEADERS fuzzing/*h) set(fuzzing_SOURCES + fuzzing/fuzz-stats.cpp fuzzing/fuzzing.cpp fuzzing/heap-types.cpp fuzzing/parameters.cpp diff --git a/src/tools/fuzzing/fuzz-stats.cpp b/src/tools/fuzzing/fuzz-stats.cpp new file mode 100644 index 00000000000..7f25f38a112 --- /dev/null +++ b/src/tools/fuzzing/fuzz-stats.cpp @@ -0,0 +1,298 @@ +/* + * Copyright 2026 WebAssembly Community Group participants + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tools/fuzzing/fuzz-stats.h" +#include "support/file_lock.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace wasm { + +namespace { + +std::string getStatsFilename() { + if (const char* env = getenv("BINARYEN_FUZZ_STATS")) { + if (env[0] != '\0') { + std::string str(env); + if (str == "1" || str == "true" || str == "on" || str == "yes") { + return "fuzz-stats.txt"; + } + return str; + } + } + return ""; +} + +struct PatternStatsRecord { + uint64_t occurrences = 0; + uint64_t inModules = 0; + uint64_t inFunctions = 0; +}; + +std::mutex eventsMutex; +std::map> pendingEvents; + +void saveStats(bool hasModuleStats, + uint64_t numFunctions, + const std::map* occurrences, + const std::map* funcMatches) { + std::string filename = getStatsFilename(); + if (filename.empty()) { + return; + } + + std::map> localEvents; + { + std::lock_guard lock(eventsMutex); + localEvents.swap(pendingEvents); + } + + if (!hasModuleStats && localEvents.empty()) { + return; + } + + // Lock file for safe concurrent updates across processes. + FileLock lock(filename + ".lock"); + + // Read existing statistics from file if present. + uint64_t totalModules = 0; + uint64_t totalFunctions = 0; + std::map statsMap; + std::map> eventStatsMap; + + enum class Section { None, Patterns, Events }; + + { + std::ifstream in(filename); + if (in.is_open()) { + std::string line; + Section section = Section::None; + std::string currentEvent; + while (std::getline(in, line)) { + if (!line.empty() && line.back() == '\r') { + line.pop_back(); + } + if (line.empty() || line[0] == '#') { + continue; + } + if (line == "Patterns:") { + section = Section::Patterns; + continue; + } + if (line == "Events:") { + section = Section::Events; + currentEvent.clear(); + continue; + } + if (section == Section::Events) { + if (line.rfind(" ", 0) == 0) { + std::stringstream ss(line); + int outcome = 0; + char colon = 0; + uint64_t count = 0; + if (!currentEvent.empty() && (ss >> outcome >> colon) && + colon == ':' && (ss >> count)) { + eventStatsMap[currentEvent][outcome] = count; + } + } else if (line.rfind(" ", 0) == 0) { + currentEvent = line.substr(2); + if (!currentEvent.empty() && currentEvent.back() == ':') { + currentEvent.pop_back(); + } + } + continue; + } + + std::stringstream ss(line); + std::string key; + if (ss >> key) { + if (key == "Modules:") { + ss >> totalModules; + } else if (key == "Functions:") { + ss >> totalFunctions; + } else if (section == Section::Patterns) { + std::string name = key; + uint64_t occurrencesCount = 0; + std::string perModStr, perFuncStr; + uint64_t inMods = 0; + std::string pctModStr; + uint64_t inFuncs = 0; + std::string pctFuncStr; + if (ss >> occurrencesCount >> perModStr >> perFuncStr >> inMods >> + pctModStr >> inFuncs >> pctFuncStr) { + statsMap[name] = {occurrencesCount, inMods, inFuncs}; + } else { + statsMap[name].occurrences = occurrencesCount; + } + } + } + } + } + } + + // Accumulate current run into overall statistics. + if (hasModuleStats) { + totalModules += 1; + totalFunctions += numFunctions; + + if (occurrences && funcMatches) { + for (const auto& [name, count] : *occurrences) { + auto& record = statsMap[name]; + record.occurrences += count; + record.inFunctions += funcMatches->at(name); + if (count > 0) { + record.inModules += 1; + } + } + } + } + + for (const auto& [eventName, outcomes] : localEvents) { + for (const auto& [outcome, count] : outcomes) { + eventStatsMap[eventName][outcome] += count; + } + } + + // Write updated statistics to a temporary file and atomically rename. + std::string tmpFilename = filename + ".tmp"; + { + std::ofstream out(tmpFilename); + if (!out.is_open()) { + return; + } + + out << "# Binaryen Fuzzing Statistics\n"; + out << "Modules: " << totalModules << "\n"; + out << "Functions: " << totalFunctions << "\n"; + + if (!statsMap.empty()) { + out << "\nPatterns:\n"; + out << "# " << std::left << std::setw(30) << "Name" << std::right + << std::setw(12) << "Occurrences" << std::setw(14) << "Per Module" + << std::setw(14) << "Per Function" << std::setw(10) << "Modules" + << std::setw(12) << "% Modules" << std::setw(12) << "Functions" + << std::setw(14) << "% Functions" + << "\n"; + + for (const auto& [name, data] : statsMap) { + double perMod = + totalModules > 0 ? (double)data.occurrences / totalModules : 0.0; + double perFunc = + totalFunctions > 0 ? (double)data.occurrences / totalFunctions : 0.0; + double pctMod = + totalModules > 0 ? (100.0 * data.inModules) / totalModules : 0.0; + double pctFunc = totalFunctions > 0 + ? (100.0 * data.inFunctions) / totalFunctions + : 0.0; + + std::ostringstream pctModStream, pctFuncStream; + pctModStream << std::fixed << std::setprecision(2) << pctMod << "%"; + pctFuncStream << std::fixed << std::setprecision(2) << pctFunc << "%"; + + out << " " << std::left << std::setw(30) << name << std::right + << std::setw(12) << data.occurrences << std::setw(14) << std::fixed + << std::setprecision(4) << perMod << std::setw(14) << std::fixed + << std::setprecision(4) << perFunc << std::setw(10) + << data.inModules << std::setw(12) << pctModStream.str() + << std::setw(12) << data.inFunctions << std::setw(14) + << pctFuncStream.str() << "\n"; + } + } + + if (!eventStatsMap.empty()) { + out << "\nEvents:\n"; + for (const auto& [eventName, outcomes] : eventStatsMap) { + uint64_t totalEventCount = 0; + for (const auto& [outcome, count] : outcomes) { + totalEventCount += count; + } + out << " " << eventName << ":\n"; + for (const auto& [outcome, count] : outcomes) { + double pct = + totalEventCount > 0 ? (100.0 * count) / totalEventCount : 0.0; + std::ostringstream pctStream; + pctStream << std::fixed << std::setprecision(2) << pct << "%"; + out << " " << std::left << std::setw(8) + << (std::to_string(outcome) + ":") << std::right << std::setw(12) + << count << " (" << std::setw(7) << pctStream.str() << ")\n"; + } + } + } + } + + rename(tmpFilename.c_str(), filename.c_str()); + + if (getenv("BINARYEN_FUZZ_STATS_VERBOSE")) { + std::cerr << "Fuzz stats updated in " << filename + << " (Total Modules: " << totalModules + << ", Total Functions: " << totalFunctions << ")\n"; + } +} + +struct EventFlusher { + ~EventFlusher() { saveStats(false, 0, nullptr, nullptr); } +} eventFlusher; + +} // namespace + +namespace FuzzStats { + +bool isEnabled() { + static const bool enabled = !getStatsFilename().empty(); + return enabled; +} + +int recordEvent(const std::string& name, int outcome) { + if (!isEnabled()) { + return outcome; + } + std::lock_guard lock(eventsMutex); + pendingEvents[name][outcome]++; + return outcome; +} + +int recordEvent(const std::string& name, + const char* file, + int line, + int outcome) { + if (!isEnabled()) { + return outcome; + } + const char* filename = file; + for (const char* p = file; *p; ++p) { + if (*p == '/' || *p == '\\') { + filename = p + 1; + } + } + return recordEvent(name + " (" + filename + ":" + std::to_string(line) + ")", + outcome); +} + +void save(uint64_t numFunctions, + const std::map& occurrences, + const std::map& funcMatches) { + saveStats(true, numFunctions, &occurrences, &funcMatches); +} + +} // namespace FuzzStats + +} // namespace wasm diff --git a/src/tools/fuzzing/fuzz-stats.h b/src/tools/fuzzing/fuzz-stats.h new file mode 100644 index 00000000000..c3de6106f86 --- /dev/null +++ b/src/tools/fuzzing/fuzz-stats.h @@ -0,0 +1,102 @@ +/* + * Copyright 2026 WebAssembly Community Group participants + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef wasm_tools_fuzzing_fuzz_stats_h +#define wasm_tools_fuzzing_fuzz_stats_h + +#include "wasm-traversal.h" + +#include +#include + +namespace wasm { + +class Module; + +namespace FuzzStats { + +// Record the outcome of a random decision or event during fuzzing. +// If statistics collection is not enabled (via BINARYEN_FUZZ_STATS), this does +// nothing. Returns `outcome` so it can be used inline in expressions. +int recordEvent(const std::string& name, int outcome); + +int recordEvent(const std::string& name, + const char* file, + int line, + int outcome); + +// Check whether statistics collection is enabled. +bool isEnabled(); + +// Save collected statistics to the stats file. +void save(uint64_t numFunctions, + const std::map& occurrences, + const std::map& funcMatches); + +// CRTP base class for visitors that collect fuzzing pattern statistics. +template +struct PatternCollectorBase : public PostWalker { + // Counts within the current function: pattern name -> count + std::map currentFuncCounts; + + // Pattern stats for this module: pattern name -> total occurrences + std::map occurrences; + // pattern name -> number of functions with >= 1 occurrence + std::map funcMatches; + uint64_t numFunctions = 0; + + void record(const std::string& name) { currentFuncCounts[name]++; } + + void visitFunction(Function* func) { + if (func->imported() || !func->body) { + return; + } + numFunctions++; + for (const auto& [name, count] : currentFuncCounts) { + if (count > 0) { + occurrences[name] += count; + funcMatches[name]++; + } + } + currentFuncCounts.clear(); + } + + // Walk the module and save collected statistics if enabled. + void collect(Module& wasm) { + if (!isEnabled()) { + return; + } + for (const auto& func : wasm.functions) { + if (!func->imported() && func->body) { + this->walkFunction(func.get()); + } + } + save(numFunctions, occurrences, funcMatches); + } +}; + +} // namespace FuzzStats + +using FuzzStats::recordEvent; + +#define RECORD_EVENT(name, outcome) \ + ::wasm::FuzzStats::recordEvent((name), __FILE__, __LINE__, (outcome)) + +#define RECORD_FUZZ_EVENT(name, outcome) RECORD_EVENT(name, outcome) + +} // namespace wasm + +#endif // wasm_tools_fuzzing_fuzz_stats_h diff --git a/src/tools/fuzzing/fuzzing.cpp b/src/tools/fuzzing/fuzzing.cpp index 65444bc6355..d906b706c5f 100644 --- a/src/tools/fuzzing/fuzzing.cpp +++ b/src/tools/fuzzing/fuzzing.cpp @@ -31,10 +31,51 @@ #include "wasm-io.h" #include "wasm-type.h" +#include "tools/fuzzing/fuzz-stats.h" + namespace wasm { namespace { +struct FuzzStatsCollector + : public FuzzStats::PatternCollectorBase { + // Collect the occurrences of various cast instructions. Casts are + // particularly important for fuzzing. Meant as a sample for running + // experiments collecting other interesting patterns. + void visitBrOn(BrOn* curr) { + switch (curr->op) { + case BrOnNull: + record("br_on_null"); + break; + case BrOnNonNull: + record("br_on_non_null"); + break; + case BrOnCast: + record("br_on_cast"); + break; + case BrOnCastFail: + record("br_on_cast_fail"); + break; + case BrOnCastDescEq: + record("br_on_cast_desc_eq"); + break; + case BrOnCastDescEqFail: + record("br_on_cast_desc_eq_fail"); + break; + } + } + + void visitRefCast(RefCast* curr) { + if (curr->desc) { + record("ref_cast_desc_eq"); + } else { + record("ref_cast"); + } + } + + void visitRefTest(RefTest* curr) { record("ref_test"); } +}; + std::vector getLoggableTypes(const FeatureSet& features) { std::vector loggableTypes = { Type::i32, Type::i64, Type::f32, Type::f64}; @@ -418,6 +459,8 @@ void TranslateToFuzzReader::build() { if (againstJS) { mutateJSBoundary(); } + + FuzzStatsCollector().collect(wasm); } void TranslateToFuzzReader::setupMemory() { diff --git a/test/unit/test_fuzz_stats.py b/test/unit/test_fuzz_stats.py new file mode 100644 index 00000000000..a3ab0a48b3f --- /dev/null +++ b/test/unit/test_fuzz_stats.py @@ -0,0 +1,178 @@ +import os +import platform +import tempfile +import unittest + +from scripts.test import shared + +from . import utils + + +def parse_stats(content): + stats = { + 'modules': None, + 'functions': None, + 'patterns': {}, + } + section = None + for raw_line in content.strip().splitlines(): + line = raw_line.strip() + if not line or line.startswith('#'): + continue + if line == 'Patterns:': + section = 'patterns' + continue + if line == 'Events:': + section = 'events' + continue + if section == 'patterns': + parts = line.split() + name = parts[0] + stats['patterns'][name] = { + 'occurrences': int(parts[1]), + 'modules': int(parts[4]), + 'functions': int(parts[6]), + } + continue + if line.startswith('Modules:'): + stats['modules'] = int(line.split(':')[1].strip()) + elif line.startswith('Functions:'): + stats['functions'] = int(line.split(':')[1].strip()) + return stats + + +class FuzzStatsTest(utils.BinaryenTestCase): + def test_stats_file_created(self): + with tempfile.TemporaryDirectory() as temp_dir: + stats_path = os.path.join(temp_dir, 'stats.txt') + random_data = self.input_path('random_data.txt') + env = dict(os.environ, BINARYEN_FUZZ_STATS=stats_path) + + shared.run_process( + shared.WASM_OPT + [ + '-ttf', random_data, '-q', '-o', os.devnull, + ], + env=env, + ) + + self.assertTrue(os.path.exists(stats_path)) + with open(stats_path) as f: + content = f.read() + + self.assertIn('# Binaryen Fuzzing Statistics', content) + stats = parse_stats(content) + self.assertEqual(stats['modules'], 1) + self.assertIsNotNone(stats['functions']) + + @unittest.skipIf(platform.system() == 'Windows', + 'Windows line endings affect random data PRNG seed') + def test_stats_file_updated_across_invocations(self): + with tempfile.TemporaryDirectory() as temp_dir: + stats_path = os.path.join(temp_dir, 'stats.txt') + wat_path = os.path.join(temp_dir, 'test.wat') + with open(wat_path, 'w') as f: + f.write( + """(module + (func $foo (param $x (ref null any)) (result (ref null any)) + (drop (ref.test (ref null any) (local.get $x))) + (drop (ref.cast (ref null any) (local.get $x))) + (local.get $x) + ) +)""", + ) + + random_data = self.input_path('random_data.txt') + env = dict(os.environ, BINARYEN_FUZZ_STATS=stats_path) + cmd = shared.WASM_OPT + [ + '-ttf', + random_data, + f'--initial-fuzz={wat_path}', + '--all-features', + '-q', + '-o', + os.devnull, + ] + + # First invocation + shared.run_process(cmd, env=env) + with open(stats_path) as f: + stats1 = parse_stats(f.read()) + + self.assertEqual(stats1['modules'], 1) + self.assertEqual(stats1['functions'], 1) + self.assertIn('ref_cast', stats1['patterns']) + self.assertIn('ref_test', stats1['patterns']) + self.assertEqual(stats1['patterns']['ref_cast']['occurrences'], 1) + self.assertEqual(stats1['patterns']['ref_cast']['modules'], 1) + self.assertEqual(stats1['patterns']['ref_test']['occurrences'], 1) + self.assertEqual(stats1['patterns']['ref_test']['modules'], 1) + + # Second invocation + shared.run_process(cmd, env=env) + with open(stats_path) as f: + stats2 = parse_stats(f.read()) + + self.assertEqual(stats2['modules'], 2) + self.assertEqual(stats2['functions'], 2) + self.assertEqual(stats2['patterns']['ref_cast']['occurrences'], 2) + self.assertEqual(stats2['patterns']['ref_cast']['modules'], 2) + self.assertEqual(stats2['patterns']['ref_test']['occurrences'], 2) + self.assertEqual(stats2['patterns']['ref_test']['modules'], 2) + + # Third invocation without initial-fuzz (empty function count) + empty_cmd = shared.WASM_OPT + [ + '-ttf', + random_data, + '-q', + '-o', + os.devnull, + ] + shared.run_process(empty_cmd, env=env) + with open(stats_path) as f: + stats3 = parse_stats(f.read()) + + self.assertEqual(stats3['modules'], 3) + self.assertEqual(stats3['functions'], 2) + # Pattern occurrences shouldn't change for module with no casts + self.assertEqual(stats3['patterns']['ref_cast']['occurrences'], 2) + self.assertEqual(stats3['patterns']['ref_cast']['modules'], 2) + + def test_stats_not_created_when_disabled(self): + with tempfile.TemporaryDirectory() as temp_dir: + stats_path = os.path.join(temp_dir, 'stats.txt') + random_data = self.input_path('random_data.txt') + env = { + k: v for k, v in os.environ.items() + if k != 'BINARYEN_FUZZ_STATS' + } + + shared.run_process( + shared.WASM_OPT + [ + '-ttf', random_data, '-q', '-o', os.devnull, + ], + env=env, + cwd=temp_dir, + ) + + self.assertFalse(os.path.exists(stats_path)) + default_stats = os.path.join(temp_dir, 'fuzz-stats.txt') + self.assertFalse(os.path.exists(default_stats)) + + def test_default_filename(self): + with tempfile.TemporaryDirectory() as temp_dir: + random_data = self.input_path('random_data.txt') + env = dict(os.environ, BINARYEN_FUZZ_STATS='1') + + shared.run_process( + shared.WASM_OPT + [ + '-ttf', random_data, '-q', '-o', os.devnull, + ], + env=env, + cwd=temp_dir, + ) + + default_stats = os.path.join(temp_dir, 'fuzz-stats.txt') + self.assertTrue(os.path.exists(default_stats)) + with open(default_stats) as f: + stats = parse_stats(f.read()) + self.assertEqual(stats['modules'], 1)