It would be good to track fuzz “stability” and determinism, and fix any issues.
Is there an easy way to generate a table for this metric for each fuzz target, maybe as a side effect of CI, or in another way?
edit: oss-fuzz link https://oss-fuzz.com/fuzzer-stats?group_by=by-fuzzer&fuzzer=afl&job=afl_asan_bitcoin-core&project=bitcoin-core
How is the “stability” metric calculated?
“Stability is measured by how many percent of the edges in the target are “stable”. Sending the same input again and again should take the exact same path through the target every time. If that is the case, the stability is 100%.” - https://github.com/AFLplusplus/AFLplusplus/blob/stable/docs/FAQ.md
oss-fuzz has a table for the afl++ jobs
Nice. Though, I wonder if there is something public available. Similar to the coverage report (https://github.com/bitcoin/bitcoin/blame/fcdb39d3ee17015776c0759e4742334a962219db/doc/fuzzing.md#L350) or the inputs zip (https://github.com/bitcoin-core/qa-assets/blob/38d7a06e9544bada01d558e6f85129334c228076/download_oss_fuzz_inputs.py#L35)
Nice. Though, I wonder if there is something public available.
I’m not aware of something like that hosted by oss-fuzz, all the per fuzzer stats seem to require auth.
I’ve been primarily fuzzing with afl++ lately, I can look at hosting some stats from that.
The latest build log from https://oss-fuzz-build-logs.storage.googleapis.com/index.html#bitcoin-core says:
0Step [#4](/bitcoin-bitcoin/4/) - "build-check-afl-address-x86_64": Retrying failed fuzz targets sequentially 4
1Step [#4](/bitcoin-bitcoin/4/) - "build-check-afl-address-x86_64": INFO: performing bad build checks for /tmp/not-out/tmpjtp3xr64/process_message
2Step [#4](/bitcoin-bitcoin/4/) - "build-check-afl-address-x86_64": INFO: performing bad build checks for /tmp/not-out/tmpjtp3xr64/process_messages
3Step [#4](/bitcoin-bitcoin/4/) - "build-check-afl-address-x86_64": INFO: performing bad build checks for /tmp/not-out/tmpjtp3xr64/tx_pool
4Step [#4](/bitcoin-bitcoin/4/) - "build-check-afl-address-x86_64": INFO: performing bad build checks for /tmp/not-out/tmpjtp3xr64/tx_pool_standard
5Step [#4](/bitcoin-bitcoin/4/) - "build-check-afl-address-x86_64": Broken fuzz targets 4
However, I don’t think this is the cause. Pretty sure this has happened for years, but I haven’t checked.
Current screenshot, for reference
Just dropping an extremely hacky/ugly patch that could be useful to collect the combined debug log while iterating over all fuzz input files in one process and then split that debug log again for each fuzz input file and show the diff:
0diff --git a/a.py b/a.py
1new file mode 100644
2index 0000000000..a28b602891
3--- /dev/null
4+++ b/a.py
5@@ -0,0 +1,27 @@
6+import os
7+import sys
8+
9+in_a=sys.argv[1]
10+out_a=[]
11+for basename_a in [in_a,in_a.replace('.a.','.b.')]:
12+ with open(basename_a, 'r') as f:
13+ content = f.read()
14+
15+ segments = content.split('e3d9b719b03df812e55de8df6414a3aa_MAGIC_')
16+
17+ # If the file started with the split token, the first segment will be empty
18+ for idx, seg in enumerate(segments):
19+ if not seg.strip():
20+ continue # skip empty segments
21+
22+ filename=f'{basename_a}.{seg[:16]}.txt'
23+ with open(filename, 'w') as fout:
24+ fout.write(seg)
25+
26+ print(f'Wrote segment to {filename}')
27+ out_a.append(filename)
28+
29+for fn in out_a:
30+ import subprocess
31+ subprocess.run(['git', '--no-pager','diff', '--no-index', '--color-moved=dimmed-zebra', '-U1', fn, fn.replace('.a.','.b.')])
32+
33diff --git a/contrib/devtools/deterministic-fuzz-coverage/src/main.rs b/contrib/devtools/deterministic-fuzz-coverage/src/main.rs
34index 3eeb121db0..73192e5af5 100644
35--- a/contrib/devtools/deterministic-fuzz-coverage/src/main.rs
36+++ b/contrib/devtools/deterministic-fuzz-coverage/src/main.rs
37@@ -127,9 +127,14 @@ fn deterministic_coverage(
38 entries.sort_by_key(|entry| entry.file_name());
39 let run_single = |run_id: char, entry: &Path, thread_id: usize| -> Result<PathBuf, AppError> {
40 let cov_txt_path = build_dir.join(format!("fuzz_det_cov.show.t{thread_id}.{run_id}.txt"));
41+ let debug_log = build_dir.join(format!("fuzz_det_cov.debug.log.t{thread_id}.{run_id}.txt"));
42 let profraw_file = build_dir.join(format!("fuzz_det_cov.t{thread_id}.{run_id}.profraw"));
43 let profdata_file = build_dir.join(format!("fuzz_det_cov.t{thread_id}.{run_id}.profdata"));
44 {
45+ use std::process::Stdio;
46+
47+ let log_file = File::create(&debug_log)
48+ .map_err(|e| format!("failed to open debug_log: {e}"))?;
49 let output = {
50 let mut cmd = Command::new(fuzz_exe);
51 if using_libfuzzer {
52@@ -140,6 +145,7 @@ fn deterministic_coverage(
53 .env("LLVM_PROFILE_FILE", &profraw_file)
54 .env("FUZZ", fuzz_target)
55 .arg(entry)
56+ .stdout(Stdio::from(log_file))
57 .output()
58 .map_err(|e| format!("fuzz failed: {e}"))?;
59 if !output.status.success() {
60@@ -219,13 +225,6 @@ The coverage was not deterministic between runs.
61 if !entry.is_file() {
62 Err(format!("{} should be a file", entry.display()))?;
63 }
64- let cov_txt_base = run_single('a', &entry, thread_id)?;
65- let cov_txt_repeat = run_single('b', &entry, thread_id)?;
66- check_diff(
67- &cov_txt_base,
68- &cov_txt_repeat,
69- &format!("The fuzz target input was {}.", entry.display()),
70- )?;
71 Ok(())
72 };
73 thread::scope(|s| -> AppResult {
74diff --git a/src/logging.cpp b/src/logging.cpp
75index 5f055566ef..bd54ee4886 100644
76--- a/src/logging.cpp
77+++ b/src/logging.cpp
78@@ -301,7 +301,7 @@ std::string BCLog::Logger::LogTimestampStr(SystemClock::time_point now, std::chr
79 return strStamped;
80
81 const auto now_seconds{std::chrono::time_point_cast<std::chrono::seconds>(now)};
82- strStamped = FormatISO8601DateTime(TicksSinceEpoch<std::chrono::seconds>(now_seconds));
83+ // strStamped = FormatISO8601DateTime(TicksSinceEpoch<std::chrono::seconds>(now_seconds));
84 if (m_log_time_micros && !strStamped.empty()) {
85 strStamped.pop_back();
86 strStamped += strprintf(".%06dZ", Ticks<std::chrono::microseconds>(now - now_seconds));
87diff --git a/src/test/fuzz/fuzz.cpp b/src/test/fuzz/fuzz.cpp
88index 96ff3b55fa..3fe9ee2797 100644
89--- a/src/test/fuzz/fuzz.cpp
90+++ b/src/test/fuzz/fuzz.cpp
91@@ -183,6 +183,7 @@ static bool read_stdin(std::vector<uint8_t>& data)
92 #if defined(PROVIDE_FUZZ_MAIN_FUNCTION) && !defined(__AFL_LOOP)
93 static bool read_file(fs::path p, std::vector<uint8_t>& data)
94 {
95+ std::cout << "e3d9b719b03df812e55de8df6414a3aa_MAGIC_" << fs::PathToString(p.filename()) << std::endl;
96 uint8_t buffer[1024];
97 FILE* f = fsbridge::fopen(p, "rb");
98 if (f == nullptr) return false;
99diff --git a/src/test/fuzz/process_message.cpp b/src/test/fuzz/process_message.cpp
100index 207daaccc5..ae8c5b4807 100644
101--- a/src/test/fuzz/process_message.cpp
102+++ b/src/test/fuzz/process_message.cpp
103@@ -56,7 +56,9 @@ void initialize_process_message()
104 static const auto testing_setup{
105 MakeNoLogFileContext<TestingSetup>(
106 /*chain_type=*/ChainType::REGTEST,
107- {}),
108+ {
109+ .extra_args = {"-printtoconsole", "-debug=1","-debugexclude=bench"},
110+ }),
111 };
112 g_setup = testing_setup.get();
113 ResetChainman(*g_setup);
114diff --git a/src/test/fuzz/process_messages.cpp b/src/test/fuzz/process_messages.cpp
115index fbb0d5f470..f24ca0ae29 100644
116--- a/src/test/fuzz/process_messages.cpp
117+++ b/src/test/fuzz/process_messages.cpp
118@@ -46,7 +46,9 @@ void initialize_process_messages()
119 static const auto testing_setup{
120 MakeNoLogFileContext<TestingSetup>(
121 /*chain_type=*/ChainType::REGTEST,
122- {}),
123+ {
124+ .extra_args = {"-printtoconsole", "-debug=1","-debugexclude=bench"},
125+ }),
126 };
127 g_setup = testing_setup.get();
128 ResetChainman(*g_setup);
edit: oss-fuzz link https://oss-fuzz.com/fuzzer-stats?group_by=by-fuzzer&fuzzer=afl&job=afl_asan_bitcoin-core&project=bitcoin-core
Currently the link just returns a stability of 100 for all fuzz targets, so something is probably off at the oss-fuzz side.