Seems fine, but All of those python3 -c '...' inside bash are fine, but seem a bit hacky.
At some point we can use vanilla Python and drop the jq dep. Something like:
<details><summary>completely untested draft diff</summary>
ci/test/00_setup_env_native_iwyu.sh | 2 +-
ci/test/03_test_script.sh | 44 +------------
ci/test/run-iwyu.py | 125 ++++++++++++++++++++++++++++++++++++
3 files changed, 127 insertions(+), 44 deletions(-)
diff --git a/ci/test/00_setup_env_native_iwyu.sh b/ci/test/00_setup_env_native_iwyu.sh
index 20240f6..9ed2a09 100755
--- a/ci/test/00_setup_env_native_iwyu.sh
+++ b/ci/test/00_setup_env_native_iwyu.sh
@@ -12,3 +12,3 @@ export IWYU_LLVM_V="23"
export APT_LLVM_V="${IWYU_LLVM_V}"
-export PACKAGES="clang-${IWYU_LLVM_V} clang-format-${IWYU_LLVM_V} libclang-${IWYU_LLVM_V}-dev llvm-${IWYU_LLVM_V}-dev jq libboost-dev libzmq3-dev systemtap-sdt-dev qt6-base-dev qt6-tools-dev qt6-l10n-tools libqrencode-dev libsqlite3-dev libcapnp-dev capnproto"
+export PACKAGES="clang-${IWYU_LLVM_V} clang-format-${IWYU_LLVM_V} libclang-${IWYU_LLVM_V}-dev llvm-${IWYU_LLVM_V}-dev libboost-dev libzmq3-dev systemtap-sdt-dev qt6-base-dev qt6-tools-dev qt6-l10n-tools libqrencode-dev libsqlite3-dev libcapnp-dev capnproto"
export NO_DEPENDS=1
diff --git a/ci/test/03_test_script.sh b/ci/test/03_test_script.sh
index 5793656..187834b 100755
--- a/ci/test/03_test_script.sh
+++ b/ci/test/03_test_script.sh
@@ -236,45 +236,3 @@ fi
if [[ "${RUN_IWYU}" == true ]]; then
- # TODO: Consider enforcing IWYU across the entire codebase.
- FILES_WITH_ENFORCED_IWYU='/src/((bench|common|consensus|crypto|index|init|kernel|primitives|rpc|script|univalue/(lib|test)|util|zmq)/.*|node/(blockstorage|interfaces|miner|mining_args|utxo_snapshot)|test/fuzz/(kitchen_sink|minisketch|parse_univalue)|clientversion|core_io|rest|signet|init)\.cpp'
- jq --arg patterns "$FILES_WITH_ENFORCED_IWYU" 'map(select(.file | test($patterns)))' "${BASE_BUILD_DIR}/compile_commands.json" > "${BASE_BUILD_DIR}/compile_commands_iwyu_errors.json"
- jq --arg patterns "$FILES_WITH_ENFORCED_IWYU" 'map(select(.file | test($patterns) | not))' "${BASE_BUILD_DIR}/compile_commands.json" > "${BASE_BUILD_DIR}/compile_commands_iwyu_warnings.json"
-
- cd "${BASE_ROOT_DIR}"
-
- run_iwyu() {
- mv "${BASE_BUILD_DIR}/$1" "${BASE_BUILD_DIR}/compile_commands.json"
- {
- python3 /include-what-you-use/mapgen/iwyu-mapgen-clang-intrin.py --lang imp "$("clang-${IWYU_LLVM_V}" -print-resource-dir)/include" > "${BASE_BUILD_DIR}/clang.intrinsics.imp"
- python3 /include-what-you-use/iwyu_tool.py \
- -p "${BASE_BUILD_DIR}" "${MAKEJOBS}" -- \
- -Xiwyu --cxx17ns \
- -Xiwyu --mapping_file="${BASE_ROOT_DIR}/contrib/devtools/iwyu/bitcoin.core.imp" \
- -Xiwyu --mapping_file="${BASE_BUILD_DIR}/clang.intrinsics.imp" \
- -Xiwyu --max_line_length=160 \
- -Xiwyu --check_also='*/common/types\.h' \
- -Xiwyu --check_also='*/consensus/*\.h' \
- -Xiwyu --check_also='*/interfaces/*\.h' \
- -Xiwyu --check_also='*/primitives/transaction_identifier\.h' \
- -Xiwyu --check_also='*/rpc/protocol\.h' \
- 2>&1 || true
- } | tee /tmp/iwyu_ci.out
- python3 "/include-what-you-use/fix_includes.py" --nosafe_headers < /tmp/iwyu_ci.out
- python3 -c '
-import runpy
-import subprocess
-
-subtrees = runpy.run_path("test/lint/lint_ignore_dirs.py")["SHARED_EXCLUDED_SUBTREES"]
-subprocess.run(["git", "restore", "--", *subtrees], check=True)
-'
- git diff -U1 | ./contrib/devtools/clang-format-diff.py -binary="clang-format-${IWYU_LLVM_V}" -p1 -i -v
- }
-
- run_iwyu "compile_commands_iwyu_errors.json"
- if ! ( git --no-pager diff --exit-code ); then
- echo "^^^ ⚠️ Failure generated from IWYU"
- false
- fi
-
- run_iwyu "compile_commands_iwyu_warnings.json"
- git --no-pager diff
+ "${BASE_ROOT_DIR}/ci/test/run-iwyu.py"
fi
diff --git a/ci/test/run-iwyu.py b/ci/test/run-iwyu.py
new file mode 100755
index 0000000..f62f789
--- /dev/null
+++ b/ci/test/run-iwyu.py
@@ -0,0 +1,125 @@
+#!/usr/bin/env python3
+# Copyright (c) The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or https://opensource.org/license/mit/.
+
+import json
+import os
+import re
+import runpy
+import shlex
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+
+FILES_WITH_ENFORCED_IWYU = re.compile(
+ r"/src/((bench|common|consensus|crypto|index|init|kernel|primitives|rpc|script|univalue/(lib|test)|util|zmq)/.*"
+ r"|node/(blockstorage|interfaces|miner|mining_args|utxo_snapshot)"
+ r"|test/fuzz/(kitchen_sink|minisketch|parse_univalue)"
+ r"|clientversion|core_io|rest|signet|init)\.cpp"
+)
+
+
+def run(cmd, **kwargs):
+ print("+ " + shlex.join(cmd), flush=True)
+ kwargs.setdefault("check", True)
+ try:
+ return subprocess.run(cmd, **kwargs)
+ except Exception as error:
+ sys.exit(str(error))
+
+
+def run_iwyu(root: Path, build_dir: Path, compile_commands: str, makejobs: str, llvm_version: str):
+ shutil.move(build_dir / compile_commands, build_dir / "compile_commands.json")
+ clang_resource_dir = run(
+ [f"clang-{llvm_version}", "-print-resource-dir"],
+ stdout=subprocess.PIPE,
+ text=True,
+ ).stdout.strip()
+ run(
+ [
+ "python3",
+ "/include-what-you-use/mapgen/iwyu-mapgen-clang-intrin.py",
+ "--lang",
+ "imp",
+ f"{clang_resource_dir}/include",
+ ],
+ stdout=(build_dir / "clang.intrinsics.imp").open("w"),
+ )
+
+ iwyu_output = run(
+ [
+ "python3",
+ "/include-what-you-use/iwyu_tool.py",
+ "-p",
+ str(build_dir),
+ makejobs,
+ "--",
+ "-Xiwyu",
+ "--cxx17ns",
+ "-Xiwyu",
+ f"--mapping_file={root / 'contrib/devtools/iwyu/bitcoin.core.imp'}",
+ "-Xiwyu",
+ f"--mapping_file={build_dir / 'clang.intrinsics.imp'}",
+ "-Xiwyu",
+ "--max_line_length=160",
+ "-Xiwyu",
+ r"--check_also=*/common/types\.h",
+ "-Xiwyu",
+ r"--check_also=*/consensus/*\.h",
+ "-Xiwyu",
+ r"--check_also=*/interfaces/*\.h",
+ "-Xiwyu",
+ r"--check_also=*/primitives/transaction_identifier\.h",
+ "-Xiwyu",
+ r"--check_also=*/rpc/protocol\.h",
+ ],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ text=True,
+ check=False,
+ )
+ Path("/tmp/iwyu_ci.out").write_text(iwyu_output.stdout)
+ print(iwyu_output.stdout, end="")
+ with Path("/tmp/iwyu_ci.out").open() as output:
+ run(
+ ["python3", "/include-what-you-use/fix_includes.py", "--nosafe_headers"],
+ stdin=output,
+ )
+
+ excluded_subtrees = runpy.run_path(str(root / "test/lint/lint_ignore_dirs.py"))["SHARED_EXCLUDED_SUBTREES"]
+ run(["git", "restore", "--", *excluded_subtrees])
+ diff = run(["git", "diff", "-U1"], stdout=subprocess.PIPE)
+ run(
+ ["./contrib/devtools/clang-format-diff.py", f"-binary=clang-format-{llvm_version}", "-p1", "-i", "-v"],
+ input=diff.stdout,
+ )
+
+
+def main():
+ root = Path(os.environ["BASE_ROOT_DIR"])
+ os.chdir(root)
+ build_dir = Path(os.environ["BASE_BUILD_DIR"])
+ all_compile_commands = json.loads((build_dir / "compile_commands.json").read_text())
+ (build_dir / "compile_commands_iwyu_errors.json").write_text(
+ json.dumps([entry for entry in all_compile_commands if FILES_WITH_ENFORCED_IWYU.search(entry["file"])])
+ )
+ (build_dir / "compile_commands_iwyu_warnings.json").write_text(
+ json.dumps([entry for entry in all_compile_commands if not FILES_WITH_ENFORCED_IWYU.search(entry["file"])])
+ )
+
+ makejobs = os.environ["MAKEJOBS"]
+ llvm_version = os.environ["IWYU_LLVM_V"]
+ run_iwyu(root, build_dir, "compile_commands_iwyu_errors.json", makejobs, llvm_version)
+ if run(["git", "diff", "--exit-code"], check=False).returncode:
+ print("^^^ ⚠️ Failure generated from IWYU")
+ raise SystemExit(1)
+
+ run_iwyu(root, build_dir, "compile_commands_iwyu_warnings.json", makejobs, llvm_version)
+ run(["git", "--no-pager", "diff"])
+
+
+if __name__ == "__main__":
+ main()
</details>
