[IBD] coins: increase default UTXO flush batch size to 32 MiB #31645

 0# Set up a clean demo environment
 1mkdir -p demo && rm -rfd demo
 2
 3# Build Bitcoin Core
 4cmake -B build -DCMAKE_BUILD_TYPE=Release && cmake --build build -j$(nproc)
 5
 6# Start bitcoind with minimal settings without mempool and internet connection
 7build/bin/bitcoind -datadir=demo -stopatheight=1
 8build/bin/bitcoind -datadir=demo -blocksonly=1 -connect=0 -dbcache=30000 -daemon
 9
10# Load the AssumeUTXO snapshot, making sure the path is correct
11# Expected output includes `"coins_loaded": 184821030`
12build/bin/bitcoin-cli -datadir=demo -rpcclienttimeout=0 loadtxoutset ~/utxo-880000.dat
13
14# Stop the daemon and verify snapshot flushes in the logs
15build/bin/bitcoin-cli -datadir=demo stop
16grep "FlushSnapshotToDisk: completed" demo/debug.log

 0COMMITS="e6bfd95d5012fa1d91f83bf4122cb292afd6277f af653f321b135a59e38794b537737ed2f4a0040b"; \
 1STOP=900000; DBCACHE=10000; \
 2CC=gcc; CXX=g++; \
 3BASE_DIR="/mnt/my_storage"; DATA_DIR="$BASE_DIR/BitcoinData"; LOG_DIR="$BASE_DIR/logs"; \
 4(echo ""; for c in $COMMITS; do git fetch -q origin $c && git log -1 --pretty='%h %s' $c || exit 1; done; echo "") && \
 5hyperfine \
 6  --sort command \
 7  --runs 1 \
 8  --export-json "$BASE_DIR/rdx-$(sed -E 's/(\w{8})\w+ ?/\1-/g;s/-$//'<<<"$COMMITS")-$STOP-$DBCACHE-$CC.json" \
 9  --parameter-list COMMIT ${COMMITS// /,} \
10  --prepare "killall bitcoind 2>/dev/null; rm -f $DATA_DIR/debug.log; git checkout {COMMIT}; git clean -fxd; git reset --hard && \
11    cmake -B build -G Ninja -DCMAKE_BUILD_TYPE=Release && ninja -C build bitcoind && \
12    ./build/bin/bitcoind -datadir=$DATA_DIR -stopatheight=$STOP -dbcache=1000 -printtoconsole=0; sleep 10" \
13  --cleanup "cp $DATA_DIR/debug.log $LOG_DIR/debug-{COMMIT}-$(date +%s).log" \
14  "COMPILER=$CC ./build/bin/bitcoind -datadir=$DATA_DIR -stopatheight=$STOP -dbcache=$DBCACHE -reindex-chainstate -blocksonly -connect=0 -printtoconsole=0"
15
16e6bfd95d50 Merge bitcoin-core/gui#881: Move `FreespaceChecker` class into its own module
17af653f321b coins: derive `batch_write_bytes` from `-dbcache` when unspecified
18
19Benchmark 1: COMPILER=gcc ./build/bin/bitcoind -datadir=/mnt/my_storage/BitcoinData -stopatheight=900000 -dbcache=10000 -reindex-chainstate -blocksonly -connect=0 -printtoconsole=0 (COMMIT = e6bfd95d5012fa1d91f83bf4122cb292afd6277f)
20  Time (abs ≡):        25016.346 s               [User: 30333.911 s, System: 826.463 s]
21 
22Benchmark 2: COMPILER=gcc ./build/bin/bitcoind -datadir=/mnt/my_storage/BitcoinData -stopatheight=900000 -dbcache=10000 -reindex-chainstate -blocksonly -connect=0 -printtoconsole=0 (COMMIT = af653f321b135a59e38794b537737ed2f4a0040b)
23  Time (abs ≡):        24801.283 s               [User: 30328.665 s, System: 834.110 s]
24 
25Relative speed comparison
26        1.01          COMPILER=gcc ./build/bin/bitcoind -datadir=/mnt/my_storage/BitcoinData -stopatheight=900000 -dbcache=10000 -reindex-chainstate -blocksonly -connect=0 -printtoconsole=0 (COMMIT = e6bfd95d5012fa1d91f83bf4122cb292afd6277f)
27        1.00          COMPILER=gcc ./build/bin/bitcoind -datadir=/mnt/my_storage/BitcoinData -stopatheight=900000 -dbcache=10000 -reindex-chainstate -blocksonly -connect=0 -printtoconsole=0 (COMMIT = af653f321b135a59e38794b537737ed2f4a0040b)

 0set -e
 1
 2killall bitcoind 2>/dev/null || true
 3cd /mnt/my_storage/bitcoin
 4mkdir -p demo
 5
 6for i in 1 2; do
 7  for dbcache in 440 5000 30000; do
 8    for dbbatchsize in 4194304 8388608 16777216 33554432 67108864 134217728 268435456; do
 9      cmake -B build -DCMAKE_BUILD_TYPE=Release > /dev/null 2>&1
10      cmake --build build -j"$(nproc)" > /dev/null 2>&1
11
12      build/bin/bitcoin-cli -datadir=demo stop 2>/dev/null || true
13      killall bitcoind 2>/dev/null || true
14      sleep 10
15
16      rm -rfd demo/chainstate demo/chainstate_snapshot
17      build/bin/bitcoind -datadir=demo -stopatheight=1 -printtoconsole=0 && rm -f demo/debug.log
18
19      echo "Starting bitcoind with dbcache=$dbcache"
20      build/bin/bitcoind -datadir=demo -blocksonly=1 -connect=0 -dbcache="$dbcache" -dbbatchsize="$dbbatchsize" -daemon -printtoconsole=0
21      sleep 10
22
23      echo "Loading UTXO snapshot..."
24      build/bin/bitcoin-cli -datadir=demo -rpcclienttimeout=0 loadtxoutset /mnt/my_storage/utxo-880000.dat | grep -q '"coins_loaded": 184821030' || { echo "ERROR: Wrong number of coins loaded"; exit 1; }
25      build/bin/bitcoin-cli -datadir=demo stop
26      killall bitcoind 2>/dev/null || true
27      sleep 10
28
29      out_file="results_i_${i}_dbcache_${dbcache}_dbbatchsize_${dbbatchsize}.log"
30      echo "Collecting logs in ${out_file}"
31      grep "FlushSnapshotToDisk: completed" demo/debug.log | tee -a "${out_file}"
32      echo "---" >> "${out_file}"
33
34      echo "Done with i=${i}, dbcache=${dbcache}, dbbatchsize=${dbbatchsize}"
35      echo
36    done
37  done
38done
39
40echo "All runs complete. Logs are saved in results_dbcache*.log files."

 0import re
 1import sys
 2
 3
 4def parse_bitcoin_debug_log(file_path):
 5    results = []
 6
 7    flush_sum = 0.0
 8    flush_count = 0
 9
10    version_pattern = re.compile(r"Bitcoin Core version")
11    flush_pattern = re.compile(r'FlushSnapshotToDisk: completed \(([\d.]+)ms\)')
12
13    def finalize_current_block():
14        nonlocal flush_sum, flush_count
15        if flush_count > 0:
16            results.append((flush_sum, flush_count))
17        flush_sum = 0.0
18        flush_count = 0
19
20    try:
21        with open(file_path, 'r') as file:
22            for line in file:
23                if version_pattern.search(line):
24                    finalize_current_block()
25                    continue
26
27                match_flush = flush_pattern.search(line)
28                if match_flush:
29                    flush_ms = float(match_flush.group(1))
30                    flush_sum += flush_ms
31                    flush_count += 1
32    except Exception as e:
33        print(f"Error reading file: {e}")
34        sys.exit(1)
35
36    finalize_current_block()
37
38    return results
39
40
41if __name__ == "__main__":
42    if len(sys.argv) < 2:
43        print("Usage: python3 script.py <path_to_debug_log>")
44        sys.exit(1)
45
46    file_path = sys.argv[1]
47    parsed_results = parse_bitcoin_debug_log(file_path)
48
49    for total_flush_time, total_flush_calls in parsed_results:
50        print(f"{total_flush_time:.2f},{total_flush_calls}")

  0diff --git a/src/txdb.cpp b/src/txdb.cpp
  1--- a/src/txdb.cpp	(revision d249a353be58868d41d2a7c57357038ffd779eba)
  2+++ b/src/txdb.cpp	(revision bae884969d35469320ed9967736eb15b5d87edff)
  3@@ -90,7 +90,81 @@
  4     return vhashHeadBlocks;
  5 }
  6
  7+/*
  8+ * Author:  David Robert Nadeau
  9+ * Site:    http://NadeauSoftware.com/
 10+ * License: Creative Commons Attribution 3.0 Unported License
 11+ *          http://creativecommons.org/licenses/by/3.0/deed.en_US
 12+ */
 13+#if defined(_WIN32)
 14+#include <windows.h>
 15+#include <psapi.h>
 16+
 17+#elif defined(__unix__) || defined(__unix) || defined(unix) || (defined(__APPLE__) && defined(__MACH__))
 18+#include <unistd.h>
 19+#include <sys/resource.h>
 20+
 21+#if defined(__APPLE__) && defined(__MACH__)
 22+#include <mach/mach.h>
 23+
 24+#elif (defined(_AIX) || defined(__TOS__AIX__)) || (defined(__sun__) || defined(__sun) || defined(sun) && (defined(__SVR4) || defined(__svr4__)))
 25+#include <fcntl.h>
 26+#include <procfs.h>
 27+
 28+#elif defined(__linux__) || defined(__linux) || defined(linux) || defined(__gnu_linux__)
 29+#include <stdio.h>
 30+
 31+#endif
 32+
 33+#else
 34+#error "Cannot define  getCurrentRSS( ) for an unknown OS."
 35+#endif
 36+
 37+/**
 38+ * Returns the current resident set size (physical memory use) measured
 39+ * in bytes, or zero if the value cannot be determined on this OS.
 40+ */
 41+size_t getCurrentRSS( )
 42+{
 43+#if defined(_WIN32)
 44+    /* Windows -------------------------------------------------- */
 45+    PROCESS_MEMORY_COUNTERS info;
 46+    GetProcessMemoryInfo( GetCurrentProcess( ), &info, sizeof(info) );
 47+    return (size_t)info.WorkingSetSize;
 48+
 49+#elif defined(__APPLE__) && defined(__MACH__)
 50+    /* OSX ------------------------------------------------------ */
 51+    struct mach_task_basic_info info;
 52+    mach_msg_type_number_t infoCount = MACH_TASK_BASIC_INFO_COUNT;
 53+    if ( task_info( mach_task_self( ), MACH_TASK_BASIC_INFO,
 54+        (task_info_t)&info, &infoCount ) != KERN_SUCCESS )
 55+        return (size_t)0L;      /* Can't access? */
 56+    return (size_t)info.resident_size;
 57+
 58+#elif defined(__linux__) || defined(__linux) || defined(linux) || defined(__gnu_linux__)
 59+    /* Linux ---------------------------------------------------- */
 60+    long rss = 0L;
 61+    FILE* fp = NULL;
 62+    if ( (fp = fopen( "/proc/self/statm", "r" )) == NULL )
 63+        return (size_t)0L;      /* Can't open? */
 64+    if ( fscanf( fp, "%*s%ld", &rss ) != 1 )
 65+    {
 66+        fclose( fp );
 67+        return (size_t)0L;      /* Can't read? */
 68+    }
 69+    fclose( fp );
 70+    return (size_t)rss * (size_t)sysconf( _SC_PAGESIZE);
 71+
 72+#else
 73+    /* AIX, BSD, Solaris, and Unknown OS ------------------------ */
 74+    return (size_t)0L;          /* Unsupported. */
 75+#endif
 76+}
 77+
 78 bool CCoinsViewDB::BatchWrite(CoinsViewCacheCursor& cursor, const uint256 &hashBlock) {
 79+    const auto start = std::chrono::steady_clock::now();
 80+    size_t max_mem{getCurrentRSS()};
 81+
 82     CDBBatch batch(*m_db);
 83     size_t count = 0;
 84     size_t changed = 0;
 85@@ -129,7 +203,11 @@
 86         it = cursor.NextAndMaybeErase(*it);
 87         if (batch.SizeEstimate() > m_options.batch_write_bytes) {
 88             LogDebug(BCLog::COINDB, "Writing partial batch of %.2f MiB\n", batch.SizeEstimate() * (1.0 / 1048576.0));
 89+
 90+            max_mem = std::max(max_mem, getCurrentRSS());
 91             m_db->WriteBatch(batch);
 92+            max_mem = std::max(max_mem, getCurrentRSS());
 93+
 94             batch.Clear();
 95             if (m_options.simulate_crash_ratio) {
 96                 static FastRandomContext rng;
 97@@ -146,8 +224,16 @@
 98     batch.Write(DB_BEST_BLOCK, hashBlock);
 99
100     LogDebug(BCLog::COINDB, "Writing final batch of %.2f MiB\n", batch.SizeEstimate() * (1.0 / 1048576.0));
101+
102+    max_mem = std::max(max_mem, getCurrentRSS());
103     bool ret = m_db->WriteBatch(batch);
104+    max_mem = std::max(max_mem, getCurrentRSS());
105+
106     LogDebug(BCLog::COINDB, "Committed %u changed transaction outputs (out of %u) to coin database...\n", (unsigned int)changed, (unsigned int)count);
107+    if (changed > 0) {
108+        const auto end{std::chrono::steady_clock::now()};
109+        LogInfo("BatchWrite took=%dms, maxMem=%dMiB", duration_cast<std::chrono::milliseconds>(end - start).count(), max_mem >> 20);
110+    }
111     return ret;
112 }

  0import os
  1import re
  2import shutil
  3import statistics
  4import subprocess
  5import time
  6import datetime
  7import argparse
  8import matplotlib.pyplot as plt  # python3.12 -m pip install matplotlib --break-system-packages
  9
 10# Regex to parse logs
 11BATCHWRITE_REGEX = re.compile(r"^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z) BatchWrite took=(\d+)ms, maxMem=(\d+)MiB")
 12
 13
 14def parse_log(archive):
 15    """Parse the log file to extract elapsed times, flush times, and memory usage."""
 16    start_time = None
 17    elapsed, batchwrite_times, usage_snapshots = [], [], []
 18    with open(archive, "r") as f:
 19        for line in f:
 20            if m := BATCHWRITE_REGEX.search(line):
 21                dt = datetime.datetime.strptime(m.group(1), "%Y-%m-%dT%H:%M:%SZ")
 22                if start_time is None:
 23                    start_time = dt
 24                elapsed.append((dt - start_time).total_seconds())
 25                batchwrite_times.append(int(m.group(2)))
 26                usage_snapshots.append(int(m.group(3)))
 27    return elapsed, batchwrite_times, usage_snapshots
 28
 29
 30def plot_results(results, output_dir):
 31    """Create separate plots for flush times and memory usage."""
 32    if len(results) != 2:
 33        print("plot_results() requires exactly 2 runs for comparison.")
 34        return
 35
 36    (dbbatch0, elapsed0, flush0, mem0) = results[0]
 37    (dbbatch1, elapsed1, flush1, mem1) = results[1]
 38
 39    # Compute percentage differences
 40    avg_flush0, avg_flush1 = statistics.mean(flush0), statistics.mean(flush1)
 41    max_mem0, max_mem1 = max(mem0), max(mem1)
 42    flush_improvement = round(((avg_flush0 - avg_flush1) / avg_flush0) * 100, 1)
 43    mem_increase = round(((max_mem1 - max_mem0) / max_mem0) * 100, 1)
 44
 45    # Plot flush times
 46    plt.figure(figsize=(16, 8))
 47    plt.plot(elapsed0, flush0, color="red", linestyle="-", label=f"Flush Times (dbbatch={dbbatch0})")
 48    plt.axhline(y=avg_flush0, color="red", linestyle="--", alpha=0.5, label=f"Mean ({dbbatch0})={avg_flush0:.1f}ms")
 49    plt.plot(elapsed1, flush1, color="orange", linestyle="-", label=f"Flush Times (dbbatch={dbbatch1})")
 50    plt.axhline(y=avg_flush1, color="orange", linestyle="--", alpha=0.5, label=f"Mean ({dbbatch1})={avg_flush1:.1f}ms")
 51    plt.title(f"Flush Times (dbbatch {dbbatch0} vs {dbbatch1}) — {abs(flush_improvement)}% {'faster' if flush_improvement > 0 else 'slower'}")
 52    plt.xlabel("Elapsed Time (seconds)")
 53    plt.ylabel("Flush Times (ms)")
 54    plt.legend()
 55    plt.grid(True)
 56    plt.tight_layout()
 57    flush_out_file = os.path.join(output_dir, "plot_flush_times.png")
 58    plt.savefig(flush_out_file)
 59    print(f"Flush Times plot saved as {flush_out_file}")
 60    plt.close()
 61
 62    # Plot memory usage
 63    plt.figure(figsize=(16, 8))
 64    plt.plot(elapsed0, mem0, color="blue", linestyle="-", label=f"Memory (dbbatch={dbbatch0})")
 65    plt.axhline(y=max_mem0, color="blue", linestyle="--", alpha=0.5, label=f"Max Mem ({dbbatch0})={max_mem0}MiB")
 66    plt.plot(elapsed1, mem1, color="green", linestyle="-", label=f"Memory (dbbatch={dbbatch1})")
 67    plt.axhline(y=max_mem1, color="green", linestyle="--", alpha=0.5, label=f"Max Mem ({dbbatch1})={max_mem1}MiB")
 68    plt.title(f"Memory Usage (dbbatch {dbbatch0} vs {dbbatch1}) — {abs(mem_increase)}% {'higher' if mem_increase > 0 else 'lower'}")
 69    plt.xlabel("Elapsed Time (seconds)")
 70    plt.ylabel("Memory Usage (MiB)")
 71    plt.legend()
 72    plt.grid(True)
 73    plt.tight_layout()
 74    mem_out_file = os.path.join(output_dir, "plot_memory_usage.png")
 75    plt.savefig(mem_out_file)
 76    print(f"Memory Usage plot saved as {mem_out_file}")
 77    plt.close()
 78
 79
 80def loadtxoutset(dbbatchsize, datadir, bitcoin_cli, bitcoind, utxo_file):
 81    """Load the UTXO set and run the Bitcoin node."""
 82    archive = os.path.join(datadir, f"results_dbbatch-{dbbatchsize}.log")
 83
 84    # Skip if logs already exist
 85    if os.path.exists(archive):
 86        print(f"Log file {archive} already exists. Skipping loadtxoutset for dbbatchsize={dbbatchsize}.")
 87        return
 88
 89    os.makedirs(datadir, exist_ok=True)
 90    debug_log = os.path.join(datadir, "debug.log")
 91
 92    try:
 93        print("Cleaning up previous run")
 94        for subdir in ["chainstate", "chainstate_snapshot"]:
 95            shutil.rmtree(os.path.join(datadir, subdir), ignore_errors=True)
 96
 97        print("Preparing UTXO load")
 98        subprocess.run([bitcoind, f"-datadir={datadir}", "-stopatheight=1"], cwd=bitcoin_core_path)
 99        os.remove(debug_log)
100
101        print(f"Starting bitcoind with dbbatchsize={dbbatchsize}")
102        subprocess.run([bitcoind, f"-datadir={datadir}", "-daemon", "-blocksonly=1", "-connect=0", f"-dbbatchsize={dbbatchsize}", f"-dbcache={440}"], cwd=bitcoin_core_path)
103        time.sleep(5)
104
105        print("Loading UTXO set")
106        subprocess.run([bitcoin_cli, f"-datadir={datadir}", "loadtxoutset", utxo_file], cwd=bitcoin_core_path)
107    except Exception as e:
108        print(f"Error during loadtxoutset for dbbatchsize={dbbatchsize}: {e}")
109        raise
110    finally:
111        print("Stopping bitcoind...")
112        subprocess.run([bitcoin_cli, f"-datadir={datadir}", "stop"], cwd=bitcoin_core_path, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
113        time.sleep(5)
114
115    shutil.copy2(debug_log, archive)
116    print(f"Archived logs to {archive}")
117
118
119if __name__ == "__main__":
120    # Parse script arguments
121    parser = argparse.ArgumentParser(description="Benchmark Bitcoin dbbatchsize configurations.")
122    parser.add_argument("--utxo-file", required=True, help="Path to the UTXO snapshot file.")
123    parser.add_argument("--bitcoin-core-path", required=True, help="Path to the Bitcoin Core project directory.")
124    args = parser.parse_args()
125
126    utxo_file = args.utxo_file
127    bitcoin_core_path = args.bitcoin_core_path
128    datadir = os.path.join(bitcoin_core_path, "demo")
129    debug_log = os.path.join(datadir, "debug.log")
130    bitcoin_cli = os.path.join(bitcoin_core_path, "build/src/bitcoin-cli")
131    bitcoind = os.path.join(bitcoin_core_path, "build/src/bitcoind")
132
133    # Build Bitcoin Core
134    print("Building Bitcoin Core...")
135    subprocess.run(["cmake", "-B", "build", "-DCMAKE_BUILD_TYPE=Release"], cwd=bitcoin_core_path, check=True)
136    subprocess.run(["cmake", "--build", "build", "-j", str(os.cpu_count())], cwd=bitcoin_core_path, check=True)
137
138    # Run tests for each dbbatchsize
139    results = []
140    for dbbatchsize in [16777216, 67108864]:  # Original and proposed
141        loadtxoutset(dbbatchsize, datadir, bitcoin_cli, bitcoind, utxo_file)
142        archive = os.path.join(datadir, f"results_dbbatch-{dbbatchsize}.log")
143        elapsed, batchwrite_times, usage_snapshots = parse_log(archive)
144        results.append((dbbatchsize, elapsed, batchwrite_times, usage_snapshots))
145
146    # Plot results
147    plot_results(results, bitcoin_core_path)
148    print("All configurations processed.")

0$ ./build/bin/bitcoind -help-debug | grep -A2 dbbatch
1  -dbbatchsize
2       Maximum database write batch size in bytes (default: 67108864)

 0mkdir demo
 1build/bin/bitcoind -datadir=demo -stopatheight=1
 2build/bin/bitcoind -datadir=demo -daemon -blocksonly=1 -stopatheight=840001
 3build/bin/bitcoin-cli -datadir=demo -rpcclienttimeout=0 loadtxoutset ~/utxo-840000.dat
 4mkdir demo-backup
 5cp -r demo/* demo-backup
 6
 7valgrind --tool=massif build/bin/bitcoind -datadir=demo -daemon -blocksonly=1 -dbbatchsize=16777216 -stopatheight=850000
 8
 9rm -rf demo
10mkdir demo
11cp -r demo-backup/* demo
12
13valgrind --tool=massif build/bin/bitcoind -datadir=demo -daemon -blocksonly=1 -dbbatchsize=67108864 -stopatheight=850000

 0--------------------------------------------------------------------------------
 1Command:            build/bin/bitcoind -datadir=../demo -daemon -blocksonly=1 -dbbatchsize=16777216 -stopatheight=850000
 2Massif arguments:   (none)
 3ms_print arguments: massif.out.674977
 4--------------------------------------------------------------------------------
 5
 6
 7    MB
 8811.4^        #                                                               
 9     |     :: #                                                       :       
10     |     :  #                                                       :       
11     |     :  #             ::               ::                       :       
12     |  :  :  # @           :                :           :   :        :       
13     |  :  :  # @          ::  :   ::        :   @@      :   :   :    :  ::  @
14     |  :  : :# @   :  ::  ::  :   :        ::   @   :   :   :   :    :  :   @
15     | ::  : :# @ :::  :   ::  :   :  ::    ::   @   :   :  ::   :  :::  :   @
16     | ::  : :# @ : :  :  :::  :   :  :   :::: ::@   :  ::  ::   :  : :  :   @
17     | ::  : :#:@:: :  :  :::  : :::  :   : :: : @   :  ::  :: :::  : :  :   @
18     | ::::: :#:@:: :  :  ::: @: : :  :   : :: : @   :  ::  :: : :  : ::::   @
19     | ::: : :#:@:: :::: :::: @: : : ::   : :: : @   :  :::::: : :::: :: : ::@
20     | ::: : :#:@:: :: : :::: @: : : :: ::: :: : @   :@@::: :::: :: : :: : : @
21     | ::: : :#:@:: :: : :::: @::: : :: : : :: : @ :::@ ::: :::: :: : :: : : @
22     | ::: : :#:@:: :: : :::: @::: : :: : : :: : @ : :@ ::: :::: :: : :: : : @
23     | ::: : :#:@:: :: : :::: @::: : :: : : :: : @ : :@ ::: :::: :: : :: : : @
24     | ::: : :#:@:: :: : :::: @::: : :: : : :: : @ : :@ ::: :::: :: : :: : : @
25     | ::: : :#:@:: :: : :::: @::: : :: : : :: : @ : :@ ::: :::: :: : :: : : @
26     | ::: : :#:@:: :: : :::: @::: : :: : : :: : @ : :@ ::: :::: :: : :: : : @
27     | ::: : :#:@:: :: : :::: @::: : :: : : :: : @ : :@ ::: :::: :: : :: : : @
28   0 +----------------------------------------------------------------------->Ti
29     0                                                                   8.610

 0--------------------------------------------------------------------------------
 1Command:            build/bin/bitcoind -datadir=../demo -daemon -blocksonly=1 -dbbatchsize=67108864 -stopatheight=850000
 2Massif arguments:   (none)
 3ms_print arguments: massif.out.682470
 4--------------------------------------------------------------------------------
 5
 6
 7    GB
 81.005^          ##                                                            
 9     |          #                        ::                                   
10     |          #               :        :   ::                               
11     |          #               :        :   :                        :       
12     |          #               :        :   :                        :       
13     |          #               :        :   :                        :      :
14     |          #               :        :   :        :               :      :
15     |  :  ::   #      :    :   :        :  ::   :    : ::   :   ::   :   :: :
16     |  :  : @  #      :    :  ::  :   :::  ::   :    : :    :   :  :::   :  @
17     |  :  : @  # ::   :    : :::  :   : : :::  ::  ::: :  :::  ::  : : :::  @
18     |  :  : @  # :   ::  ::: :::  :   : : :::  ::  : : :  : :  ::  : : : : :@
19     | ::::: @::# :   ::  : : :::  :  :: : ::: :::  : :::  : :  :: :: ::: : :@
20     | ::: : @: # :   ::  : : ::::::  :: : ::: :::  : :::  : :  :: :: ::: : :@
21     | ::: : @: # : ::::  : :::::: :  :: : ::: :::::: :::  : ::::: :: ::: : :@
22     | ::: : @: # : : ::::: :::::: ::::: : ::: :::: : ::: :: :: :: :: ::: : :@
23     | ::: : @: # : : ::: : :::::: :: :: : ::: :::: : ::: :: :: :: :: ::: : :@
24     | ::: : @: # : : ::: : :::::: :: :: : ::: :::: : ::: :: :: :: :: ::: : :@
25     | ::: : @: # : : ::: : :::::: :: :: : ::: :::: : ::: :: :: :: :: ::: : :@
26     | ::: : @: # : : ::: : :::::: :: :: : ::: :::: : ::: :: :: :: :: ::: : :@
27     | ::: : @: # : : ::: : :::::: :: :: : ::: :::: : ::: :: :: :: :: ::: : :@
28   0 +----------------------------------------------------------------------->Ti
29     0                                                                   8.477

 0diff --git a/src/txdb.cpp b/src/txdb.cpp
 1index 1622039d63..ef73378ce6 100644
 2--- a/src/txdb.cpp
 3+++ b/src/txdb.cpp
 4@@ -91,6 +91,7 @@ std::vector<uint256> CCoinsViewDB::GetHeadBlocks() const {
 5 }
 6 
 7 bool CCoinsViewDB::BatchWrite(CoinsViewCacheCursor& cursor, const uint256 &hashBlock) {
 8+    const auto start = std::chrono::steady_clock::now();
 9     CDBBatch batch(*m_db);
10     size_t count = 0;
11     size_t changed = 0;
12@@ -147,7 +148,9 @@ bool CCoinsViewDB::BatchWrite(CoinsViewCacheCursor& cursor, const uint256 &hashB
13 
14     LogDebug(BCLog::COINDB, "Writing final batch of %.2f MiB\n", batch.SizeEstimate() * (1.0 / 1048576.0));
15     bool ret = m_db->WriteBatch(batch);
16+    const auto end{std::chrono::steady_clock::now()};
17     LogDebug(BCLog::COINDB, "Committed %u changed transaction outputs (out of %u) to coin database...\n", (unsigned int)changed, (unsigned int)count);
18+    LogDebug(BCLog::COINDB, "BatchWrite: completed in %dms\n", duration_cast<std::chrono::milliseconds>(end - start).count());
19     return ret;
20 }

 0mkdir demo
 1build/bin/bitcoind -datadir=demo -stopatheight=1
 2build/bin/bitcoind -datadir=demo -daemon -blocksonly=1 -stopatheight=840001
 3build/bin/bitcoin-cli -datadir=demo -rpcclienttimeout=0 loadtxoutset ~/utxo-840000.dat
 4mkdir demo-backup
 5cp -r demo/* demo-backup
 6
 7build/bin/bitcoind -datadir=demo -daemon -blocksonly=1 -dbbatchsize=16777216 -stopatheight=850000 -debug=coindb
 8cp demo/debug.log defaultcache_16MiB_dbbatchsize.log
 9
10rm -rf demo
11mkdir demo
12cp -r demo-backup/* demo
13
14build/bin/bitcoind -datadir=demo -daemon -blocksonly=1 -dbbatchsize=67108864 -stopatheight=850000 -debug=coindb
15cp demo/debug.log defaultcache_64MiB_dbbatchsize.log
16
17// Parse log files using https://github.com/Eunovo/31645-data/blob/main/extract_batch_write_data.py
18cat defaultcache_64MiB_dbbatchsize.log | grep "BatchWrite" -B 1 | xargs ./extract_batch_write_data.py > defaultcache_64MiB_dbbatchsize.result

 0Logfile: defaultcache_16MiB_dbbatchsize.log
 1--------------------------------------------
 2Average num_txouts_per_ms: 1018.2494977251025
 3--------------------------------------------
 4num_txouts,write_times_ms,num_txouts_per_ms
 52862776,2830,1011.5816254416961
 62934562,2851,1029.3097158891617
 72921143,2753,1061.0762804213584
 82940809,2861,1027.8954910870325
 92948759,2870,1027.4421602787456
102979042,2967,1004.0586450960566
112956426,2917,1013.5159410353102
123048466,3060,996.2307189542483
132978184,2940,1012.9877551020409
142944033,2945,999.6716468590832
152973528,2970,1001.1878787878788
162970816,2889,1028.3198338525442
172982352,2898,1029.1069703243616
182960008,2776,1066.28530259366
193016540,2952,1021.8631436314363
203007010,3033,991.4309264754369
212976775,2982,998.2478202548625
223084773,3085,999.92641815235
232069630,2016,1026.6021825396826

 0Logfile: defaultcache_64MiB_dbbatchsize.log
 1--------------------------------------------
 2Average num_txouts_per_ms: 735.7374156918239
 3--------------------------------------------
 4num_txouts,write_times_ms,num_txouts_per_ms
 52862776,3881,737.6387528987375
 62895729,3942,734.5837138508372
 72941575,3704,794.161717062635
 82941158,3924,749.5305810397554
 92965901,4129,718.3097602325018
102978043,4095,727.2388278388279
112946445,4041,729.1375897055184
123034443,4319,702.5799953692984
132993584,4154,720.6509388541165
142947744,4113,716.6895210308777
152979400,4072,731.679764243615
162968346,3819,777.2573972244043
172991885,3932,760.9066632756867
182960911,3844,770.2682101977107
192942739,4000,735.68475
203031336,4282,707.9252685660906
213007839,4118,730.4125789218067
223057094,4257,718.133427296218
232137921,2985,716.2214405360133

 0COMMITS="998386d4462f5e06412303ba559791da83b913fb 5cefc4bbadaa06f22089dd7991fac17ed5b283ca"; \
 1STOP_HEIGHT=888888; DBCACHE=1000; \
 2C_COMPILER=gcc; CXX_COMPILER=g++; \
 3DATA_DIR="/mnt/my_storage/BitcoinData"; \
 4LOG_DIR="/mnt/my_storage/logs"; \
 5mkdir -p $LOG_DIR; \
 6for COMMIT in $COMMITS; do \
 7  git fetch --all -q && git fetch origin $COMMIT && git log -1 --oneline $COMMIT && \
 8  killall bitcoind || true && \
 9  rm -rf $DATA_DIR/* && \
10  git checkout $COMMIT && \
11  git clean -fxd && \
12  git reset --hard && \
13  cmake -B build -DCMAKE_BUILD_TYPE=Release -DENABLE_WALLET=OFF -DCMAKE_C_COMPILER=$C_COMPILER -DCMAKE_CXX_COMPILER=$CXX_COMPILER && \
14  cmake --build build -j$(nproc) --target bitcoind && \
15  ./build/bin/bitcoind -datadir=$DATA_DIR -stopatheight=1 -printtoconsole=1 || true && \
16  valgrind --tool=massif --time-unit=ms \
17    --massif-out-file="$LOG_DIR/massif-$COMMIT-$STOP_HEIGHT-$DBCACHE.out" \
18    ./build/bin/bitcoind -datadir=$DATA_DIR \
19    -stopatheight=$STOP_HEIGHT \
20    -dbcache=$DBCACHE \
21    -blocksonly=1 \
22    -printtoconsole=1 && \
23  cp $DATA_DIR/debug.log "$LOG_DIR/debug-$COMMIT-$STOP_HEIGHT-$DBCACHE.log" && \
24  ms_print "$LOG_DIR/massif-$COMMIT-$STOP_HEIGHT-$DBCACHE.out" | tee "$LOG_DIR/massif-$COMMIT-$STOP_HEIGHT-$DBCACHE.txt"; \
25done

0static int64_t GetDefaultDbBatchSize(int64_t dbcache_bytes) {
1    return std::min<int64_t>(
2        MAX_DB_CACHE_BATCH, // 512 MiB upper limit
3        std::max<int64_t>(
4            MIN_DB_CACHE,   // 4 MiB lower limit
5            dbcache_bytes * DEFAULT_KERNEL_CACHE_BATCH_SIZE / DEFAULT_KERNEL_CACHE
6        )
7    );
8}

0           4 |          16.0 |     832
1          10 |          16.0 |     832
2          45 |          16.0 |     832
3         100 |          16.0 |     832
4         450 |          16.0 |     832
5        1000 |          35.6 |     375
6        2000 |          71.1 |     188
7        4500 |         160.0 |      84
8       10000 |         256.0 |      52
9       45000 |         256.0 |      52

0         450 |     0.753 |    0.755 |       0.001 |       0.2% | 16 MiB → 16 MiB
1        1000 |     1.240 |    1.313 |       0.073 |       5.9% | 16 MiB → 35 MiB
2        4500 |     4.564 |    5.109 |       0.545 |      11.9% | 16 MiB → 128 MiB
3       10000 |     9.785 |   10.770 |       0.985 |      10.1% | 16 MiB → 256 MiB

 0rm -rf ~/.bitcoin && time ./build/bin/bitcoind -connect=workstation.lan -dbcache=45000 -stopatheight=910000
 1
 2real	238m57.330s
 3user	527m48.667s
 4sys	19m49.001s
 5
 6rm -rf ~/.bitcoin && time ./build/bin/bitcoind -connect=workstation.lan -dbcache=450 -stopatheight=910000
 7
 8real	266m16.473s
 9user	636m6.859s
10sys	32m59.252s
11
12repeated:
13
14real	273m25.548s
15user	640m5.442s
16sys	31m58.674s
17
18rm -rf ~/.bitcoin && time ./build/bin/bitcoind -connect=workstation.lan -dbcache=45000 -stopatheight=910000
19
20real	228m12.943s
21user	509m52.682s
22sys	17m10.000s
23
24rm -rf ~/.bitcoin && time ./build/bin/bitcoind -connect=workstation.lan -dbcache=450 -stopatheight=910000
25
26real	261m47.071s
27user	631m42.243s
28sys	31m34.241s

 0rm -rf ~/.bitcoin && time ./build/bin/bitcoind -connect=workstation.lan -dbcache=45000 -stopatheight=910000
 1
 2real	246m56.108s
 3user	522m46.642s
 4sys	20m24.343s
 5
 6again:
 7
 8real	232m41.467s
 9user	508m7.763s
10sys	19m16.589s
11
12again:
13
14real	242m17.316s
15user	506m25.625s
16sys	18m45.029s

0rm -rf ~/.bitcoin && time ./build/bin/bitcoind -connect=workstation.lan -dbcache=45000 -stopatheight=910000
1
2real	232m37.784s
3user	490m58.944s
4sys	18m3.817s

0450 runs:
1266+273+262 = 267mins
2
345'000 without PR runs:
4247+233+242 / 3 = 241min
5
645'000 with PR runs:
7239+228+233 / 3 = 233min

 0dbcache=500 MB (dynamic batch size would be 17.8 MB)
 1  371bece67f: 2215.101±198.228 s  use fixed 64_MiB
 2  0a77b9eee6: 2698.961±108.559 s  use fixed 32_MiB
 3  1c88f34884: 2880.934±137.068 s  coins: introduce dynamic batch size calculator based on `dbcache` value
 4  6ca6f3b37b: 3513.171±761.674 s  Merge bitcoin/bitcoin#33241: Update libmultiprocess subtree to fix build issues
 5
 6dbcache=2500 MB (dynamic batch size would be 88.9 MB)
 7  0a77b9eee6: 2202.481±100.896 s  use fixed 32_MiB
 8  6ca6f3b37b: 2277.482±131.931 s  Merge bitcoin/bitcoin#33241: Update libmultiprocess subtree to fix build issues
 9  1c88f34884: 2313.098±202.818 s  coins: introduce dynamic batch size calculator based on `dbcache` value
10  371bece67f: 2360.670±73.818 s  use fixed 64_MiB

 0dbcache=500 MB (dynamic batch size would be 17.8 MB)
 1  371bece67f: 723.753±74.142 s  use fixed 64_MiB
 2  0a77b9eee6: 872.229±19.075 s  use fixed 32_MiB
 3  6ca6f3b37b: 924.901±34.220 s  Merge bitcoin/bitcoin#33241: Update libmultiprocess subtree to fix build issues
 4  1c88f34884: 927.929±20.000 s  coins: introduce dynamic batch size calculator based on `dbcache` value
 5
 6dbcache=1000 MB (dynamic batch size would be 35.6 MB)
 7  371bece67f: 763.976±18.272 s  use fixed 64_MiB
 8  0a77b9eee6: 780.121±21.817 s  use fixed 32_MiB
 9  6ca6f3b37b: 850.142±53.788 s  Merge bitcoin/bitcoin#33241: Update libmultiprocess subtree to fix build issues
10  1c88f34884: 865.503±48.942 s  coins: introduce dynamic batch size calculator based on `dbcache` value
11
12dbcache=1500 MB (dynamic batch size would be 53.3 MB)
13  0a77b9eee6: 741.196±42.555 s  use fixed 32_MiB
14  371bece67f: 753.405±23.264 s  use fixed 64_MiB
15  1c88f34884: 835.422±16.123 s  coins: introduce dynamic batch size calculator based on `dbcache` value
16  6ca6f3b37b: 841.494±27.479 s  Merge bitcoin/bitcoin#33241: Update libmultiprocess subtree to fix build issues
17
18dbcache=2000 MB (dynamic batch size would be 71.1 MB)
19  371bece67f: 726.012±25.313 s  use fixed 64_MiB
20  6ca6f3b37b: 799.339±29.058 s  Merge bitcoin/bitcoin#33241: Update libmultiprocess subtree to fix build issues
21  1c88f34884: 820.236±29.547 s  coins: introduce dynamic batch size calculator based on `dbcache` value
22  0a77b9eee6: 858.368±16.452 s  use fixed 32_MiB
23
24dbcache=2500 MB (dynamic batch size would be 88.9 MB)
25  371bece67f: 707.112±55.839 s  use fixed 64_MiB
26  1c88f34884: 749.380±35.179 s  coins: introduce dynamic batch size calculator based on `dbcache` value
27  6ca6f3b37b: 768.356±33.643 s  Merge bitcoin/bitcoin#33241: Update libmultiprocess subtree to fix build issues
28  0a77b9eee6: 781.631±25.348 s  use fixed 32_MiB
29
30dbcache=3000 MB (dynamic batch size would be 106.7 MB)
31  371bece67f: 704.683±36.839 s  use fixed 64_MiB
32  0a77b9eee6: 749.494±14.230 s  use fixed 32_MiB
33  6ca6f3b37b: 802.988±29.282 s  Merge bitcoin/bitcoin#33241: Update libmultiprocess subtree to fix build issues
34  1c88f34884: 812.413±14.338 s  coins: introduce dynamic batch size calculator based on `dbcache` value
35
36dbcache=3500 MB (dynamic batch size would be 124.4 MB)
37  371bece67f: 747.208±31.524 s  use fixed 64_MiB
38  0a77b9eee6: 756.109±9.717 s  use fixed 32_MiB
39  1c88f34884: 769.228±35.585 s  coins: introduce dynamic batch size calculator based on `dbcache` value
40  6ca6f3b37b: 783.391±43.066 s  Merge bitcoin/bitcoin#33241: Update libmultiprocess subtree to fix build issues
41
42dbcache=4000 MB (dynamic batch size would be 142.2 MB)
43  0a77b9eee6: 742.536±13.438 s  use fixed 32_MiB
44  1c88f34884: 752.711±21.669 s  coins: introduce dynamic batch size calculator based on `dbcache` value
45  6ca6f3b37b: 753.414±15.107 s  Merge bitcoin/bitcoin#33241: Update libmultiprocess subtree to fix build issues
46  371bece67f: 759.595±15.312 s  use fixed 64_MiB
47
48dbcache=4500 MB (dynamic batch size would be 160.0 MB)
49  371bece67f: 722.305±22.894 s  use fixed 64_MiB
50  0a77b9eee6: 738.573±28.200 s  use fixed 32_MiB
51  1c88f34884: 743.053±12.117 s  coins: introduce dynamic batch size calculator based on `dbcache` value
52  6ca6f3b37b: 747.821±16.638 s  Merge bitcoin/bitcoin#33241: Update libmultiprocess subtree to fix build issues
53
54dbcache=5000 MB (dynamic batch size would be 177.8 MB)
55  6ca6f3b37b: 707.499±23.626 s  Merge bitcoin/bitcoin#33241: Update libmultiprocess subtree to fix build issues
56  1c88f34884: 708.064±21.035 s  coins: introduce dynamic batch size calculator based on `dbcache` value
57  0a77b9eee6: 713.495±18.264 s  use fixed 32_MiB
58  371bece67f: 723.671±18.868 s  use fixed 64_MiB
59
60dbcache=5500 MB (dynamic batch size would be 195.6 MB)
61  371bece67f: 697.942±15.469 s  use fixed 64_MiB
62  0a77b9eee6: 705.869±15.561 s  use fixed 32_MiB
63  6ca6f3b37b: 797.158±15.429 s  Merge bitcoin/bitcoin#33241: Update libmultiprocess subtree to fix build issues
64  1c88f34884: 797.677±17.496 s  coins: introduce dynamic batch size calculator based on `dbcache` value