unable to make a Guix build #150

Originally posted by @Sjors in https://github.com/bitcoin/bitcoin/issues/31741#issuecomment-2633426678

I'm unable to make a Guix build. I'm using a branch sjors/2025/02/ipc-guix which enables multiprocess in the simplest possible way, without modifying the depends system, by setting MULTIPROCESS=1 in the pre-download and depends make step.

It downloads and builds capnp as expected, but in the build says:

make: Entering directory '/bitcoin/depends'
Checksum missing or mismatched for native_libmultiprocess source. Forcing re-download.
...
Extracting native_capnp..
...
Caching native_capnp...
Fetching from
/gnu/store/7k364lx15sq567ivd79pp9dmnallcxz1-bash-minimal-5.1.16/bin/sh: line 1: curl: command not found
Fetching from https://bitcoincore.org/depends-sources
/gnu/store/7k364lx15sq567ivd79pp9dmnallcxz1-bash-minimal-5.1.16/bin/sh: line 1: curl: command not found
make: *** [funcs.mk:324: /bitcoin/depends/sources/download-stamps/.stamp_fetched-native_libmultiprocess-.hash] Error 127

I can add more debugging if you can't reproduce.

I can add more debugging if you can't reproduce.

Thanks, I will try to reproduce this, but if you can add debugging that would be helpful. Output from https://github.com/bitcoin/bitcoin/pull/30975#issuecomment-2625576039 would provide more information:

--- a/depends/Makefile
+++ b/depends/Makefile
@@ -251,8 +251,9 @@ endef
 
 define check_or_remove_sources
   mkdir -p $($(package)_source_dir); cd $($(package)_source_dir); \
-  test -f $($(package)_fetched) && ( $(build_SHA256SUM) -c $($(package)_fetched) >/dev/null 2>/dev/null || \
+  test -f $($(package)_fetched) && ( $(build_SHA256SUM) -c $($(package)_fetched) || \
     ( echo "Checksum missing or mismatched for $(package) source. Forcing re-download."; \
+      set -x; cat $($(package)_fetched); : $($(package)_sha256_hash); ls -ld --full-time $(CURDIR)/$($(package)_local_dir) $($(package)_fetched) $($(package)_source); \
       rm -f $($(package)_all_sources) $($(1)_fetched))) || true
 endef
 
--- a/depends/funcs.mk
+++ b/depends/funcs.mk
@@ -40,7 +40,7 @@ define fetch_file
 endef
 
 define fetch_local_dir_sha256
-    if ! [ -f $($(1)_source) ] || [ $($(1)_source) -ot $($(1)_local_dir) ]; then \
+    set -x; if ! [ -f $($(1)_source) ] || [ $($(1)_source) -ot $($(1)_local_dir) ]; then \
         mkdir -p $(dir $($(1)_source)) && \
         $(build_TAR) -c -f $($(1)_source) -C $($(1)_local_dir) . && \
         rm -f $($(1)_fetched); \

Mmm, this patch doesn't apply on 8d63c140ebc9c2a09aa874fdd048c7d45bb5521f.

Yes funcs.mk changed a bit. The only difference in funcs.mk is adding "set -x; " before "if"

diff --git a/depends/funcs.mk b/depends/funcs.mk
index 1907bab03a..373f296c9b 100644
--- a/depends/funcs.mk
+++ b/depends/funcs.mk
@@ -40,7 +40,7 @@ define fetch_file
 endef
 
 define fetch_local_dir_sha256
-    if ! [ -f $($(1)_source) ] || [ -n "$$(find $($(1)_local_dir) -newer $($(1)_source) | head -n1)" ]; then \
+    set -x; if ! [ -f $($(1)_source) ] || [ -n "$$(find $($(1)_local_dir) -newer $($(1)_source) | head -n1)" ]; then \
         mkdir -p $(dir $($(1)_source)) && \
         $(build_TAR) -c -f $($(1)_source) -C $($(1)_local_dir) . && \
         rm -f $($(1)_fetched); \

Which gives me the following logs:

make: Entering directory '/home/sjors/bitcoin/depends'
+ [ -f /home/sjors/bitcoin/depends/sources/src-ipc-libmultiprocess.tar ]
+ find ../src/ipc/libmultiprocess -newer /home/sjors/bitcoin/depends/sources/src-ipc-libmultiprocess.tar
+ head -n1
+ [ -n  ]
+ [ -f /home/sjors/bitcoin/depends/sources/download-stamps/.stamp_fetched-native_libmultiprocess-.hash ]
+ head -n1
+ find /home/sjors/bitcoin/depends/sources/src-ipc-libmultiprocess.tar -newer /home/sjors/bitcoin/depends/sources/download-stamps/.stamp_fetched-native_libmultiprocess-.hash
+ [ -n  ]
+ cut -d  -f1 /home/sjors/bitcoin/depends/sources/download-stamps/.stamp_fetched-native_libmultiprocess-.hash
make[1]: Entering directory '/home/sjors/bitcoin/depends'
+ [ -f /home/sjors/bitcoin/depends/sources/src-ipc-libmultiprocess.tar ]
+ find ../src/ipc/libmultiprocess -newer /home/sjors/bitcoin/depends/sources/src-ipc-libmultiprocess.tar
+ head -n1
+ [ -n  ]
+ [ -f /home/sjors/bitcoin/depends/sources/download-stamps/.stamp_fetched-native_libmultiprocess-.hash ]
+ head -n1
+ find /home/sjors/bitcoin/depends/sources/src-ipc-libmultiprocess.tar -newer /home/sjors/bitcoin/depends/sources/download-stamps/.stamp_fetched-native_libmultiprocess-.hash
+ [ -n  ]
+ cut -d  -f1 /home/sjors/bitcoin/depends/sources/download-stamps/.stamp_fetched-native_libmultiprocess-.hash
boost_1_81_0.tar.gz: OK
libevent-2.1.12-stable.tar.gz: OK
qtbase-everywhere-opensource-src-5.15.16.tar.xz: OK
qttranslations-everywhere-opensource-src-5.15.16.tar.xz: OK
qttools-everywhere-opensource-src-5.15.16.tar.xz: OK
qrencode-4.1.1.tar.gz: OK
db-4.8.30.NC.tar.gz: OK
sqlite-autoconf-3460100.tar.gz: OK
zeromq-4.3.5.tar.gz: OK
capnproto-cxx-1.1.0.tar.gz: OK
sha256sum: /home/sjors/bitcoin/depends/sources/download-stamps/.stamp_fetched-native_libmultiprocess-.hash: no properly formatted checksum lines found
Checksum missing or mismatched for native_libmultiprocess source. Forcing re-download.
+ cat /home/sjors/bitcoin/depends/sources/download-stamps/.stamp_fetched-native_libmultiprocess-.hash
+ :
+ ls -ld --full-time /home/sjors/bitcoin/depends/../src/ipc/libmultiprocess /home/sjors/bitcoin/depends/sources/download-stamps/.stamp_fetched-native_libmultiprocess-.hash /home/sjors/bitcoin/depends/sources/src-ipc-libmultiprocess.tar
drwxrwxr-x 9 sjors sjors   4096 2025-02-04 09:44:29.396999115 +0000 /home/sjors/bitcoin/depends/../src/ipc/libmultiprocess
-rw-r--r-- 1 sjors sjors      0 2025-02-04 10:00:26.154937995 +0000 /home/sjors/bitcoin/depends/sources/download-stamps/.stamp_fetched-native_libmultiprocess-.hash
-rw-rw-r-- 1 sjors sjors 296960 2025-02-04 10:00:03.689024413 +0000 /home/sjors/bitcoin/depends/sources/src-ipc-libmultiprocess.tar
+ rm -f /home/sjors/bitcoin/depends/sources/download-stamps/.stamp_fetched-native_libmultiprocess-.hash
capnproto-cxx-1.1.0.tar.gz: OK
Fetching from
curl: (3) URL rejected: No host part in the URL
Fetching from https://bitcoincore.org/depends-sources
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
  0  308k    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
curl: (22) The requested URL returned error: 403
make[1]: *** [funcs.mk:324: /home/sjors/bitcoin/depends/sources/download-stamps/.stamp_fetched-native_libmultiprocess-.hash] Error 22
make[1]: Leaving directory '/home/sjors/bitcoin/depends'
make: *** [Makefile:281: download-osx] Error 2
make: Leaving directory '/home/sjors/bitcoin/depends'

After a git clean -dfx:

make: Entering directory '/home/sjors/bitcoin/depends'
+ [ -f /home/sjors/bitcoin/depends/sources/src-ipc-libmultiprocess.tar ]
+ mkdir -p /home/sjors/bitcoin/depends/sources/
+ tar -c -f /home/sjors/bitcoin/depends/sources/src-ipc-libmultiprocess.tar -C ../src/ipc/libmultiprocess .
+ rm -f /home/sjors/bitcoin/depends/sources/download-stamps/.stamp_fetched-native_libmultiprocess-.hash
+ [ -f /home/sjors/bitcoin/depends/sources/download-stamps/.stamp_fetched-native_libmultiprocess-.hash ]
+ mkdir -p /home/sjors/bitcoin/depends/sources/download-stamps/
+ sha256sum /home/sjors/bitcoin/depends/sources/src-ipc-libmultiprocess.tar
+ cut -d  -f1 /home/sjors/bitcoin/depends/sources/download-stamps/.stamp_fetched-native_libmultiprocess-.hash
make[1]: Entering directory '/home/sjors/bitcoin/depends'
+ [ -f /home/sjors/bitcoin/depends/sources/src-ipc-libmultiprocess.tar ]
+ find ../src/ipc/libmultiprocess -newer /home/sjors/bitcoin/depends/sources/src-ipc-libmultiprocess.tar
+ head -n1
+ [ -n  ]
+ [ -f /home/sjors/bitcoin/depends/sources/download-stamps/.stamp_fetched-native_libmultiprocess-.hash ]
+ head -n1
+ find /home/sjors/bitcoin/depends/sources/src-ipc-libmultiprocess.tar -newer /home/sjors/bitcoin/depends/sources/download-stamps/.stamp_fetched-native_libmultiprocess-.hash
+ [ -n  ]
+ cut -d  -f1 /home/sjors/bitcoin/depends/sources/download-stamps/.stamp_fetched-native_libmultiprocess-.hash
/home/sjors/bitcoin/depends/sources/src-ipc-libmultiprocess.tar: OK
Fetching boost_1_81_0.tar.gz from https://archives.boost.io/release/1.81.0/source/
...
/home/sjors/bitcoin/depends/work/download/capnp-1.1.0/capnproto-cxx-1.1.0.tar.gz.temp: OK
make[1]: Leaving directory '/home/sjors/bitcoin/depends'
make: Leaving directory '/home/sjors/bitcoin/depends'
INFO: Building 8d63c140ebc9 for platform triple arm64-apple-darwin:
...
make: Entering directory '/bitcoin/depends'
+ '[' -f /bitcoin/depends/sources/src-ipc-libmultiprocess.tar ']'
++ find ../src/ipc/libmultiprocess -newer /bitcoin/depends/sources/src-ipc-libmultiprocess.tar
++ head -n1
+ '[' -n '' ']'
+ '[' -f /bitcoin/depends/sources/download-stamps/.stamp_fetched-native_libmultiprocess-.hash ']'
++ find /bitcoin/depends/sources/src-ipc-libmultiprocess.tar -newer /bitcoin/depends/sources/download-stamps/.stamp_fetched-native_libmultiprocess-.hash
++ head -n1
+ '[' -n '' ']'
+ cut '-d ' -f1 /bitcoin/depends/sources/download-stamps/.stamp_fetched-native_libmultiprocess-.hash
boost_1_81_0.tar.gz: OK
libevent-2.1.12-stable.tar.gz: OK
...
Caching native_capnp...
Fetching from
/gnu/store/7k364lx15sq567ivd79pp9dmnallcxz1-bash-minimal-5.1.16/bin/sh: line 1: curl: command not found
Fetching from https://bitcoincore.org/depends-sources
/gnu/store/7k364lx15sq567ivd79pp9dmnallcxz1-bash-minimal-5.1.16/bin/sh: line 1: curl: command not found
make: *** [funcs.mk:324: /bitcoin/depends/sources/download-stamps/.stamp_fetched-native_libmultiprocess-.hash] Error 127
make: Leaving directory '/bitcoin/depends'

Thanks so in first log which shows a non-clean build https://github.com/chaincodelabs/libmultiprocess/issues/150#issuecomment-2634221309 it looks like creation of tarball and checksum files are skipped because they are up to date. But the at some point after that there are errors:

sha256sum: /home/sjors/bitcoin/depends/sources/download-stamps/.stamp_fetched-native_libmultiprocess-.hash: no properly formatted checksum lines found Checksum missing or mismatched for native_libmultiprocess source. Forcing re-download.

Happening because .stamp_fetched-native_libmultiprocess-.hash seems to have size 0.

Second log https://github.com/chaincodelabs/libmultiprocess/issues/150#issuecomment-2634233791 on clean build seems to show tarball and checksum being created in way you would expect and no "Checksum missing" error (or maybe that part of the output is just omitted) and then is an error attempting to build .stamp_fetched-native_libmultiprocess-.hash which should have been generated earlier and not supposed to be getting built.

If you see "Checksum missing or mismatched for native_libmultiprocess source. Forcing re-download." error in the clean build it would be helpful to see output around it. Otherwise I probably have to debug. Am in the middle of trying to debug https://github.com/bitcoin/bitcoin/issues/31772 so haven't tried this yet.

I forgot to copy that bit:

capnproto-cxx-1.1.0.tar.gz: OK
sha256sum: /home/sjors/bitcoin/depends/sources/src-ipc-libmultiprocess.tar: No such file or directory
/home/sjors/bitcoin/depends/sources/src-ipc-libmultiprocess.tar: FAILED open or read
sha256sum: WARNING: 1 listed file could not be read
Checksum missing or mismatched for native_libmultiprocess source. Forcing re-download.
+ cat /bitcoin/depends/sources/download-stamps/.stamp_fetched-native_libmultiprocess-.hash
625aad21a6dac1005501646b8077776c782c7df35f4f00b64e5d67aea36d2326  /home/sjors/bitcoin/depends/sources/src-ipc-libmultiprocess.tar
+ : 625aad21a6dac1005501646b8077776c782c7df35f4f00b64e5d67aea36d2326
+ ls -ld --full-time /bitcoin/depends/../src/ipc/libmultiprocess /bitcoin/depends/sources/download-stamps/.stamp_fetched-native_libmultiprocess-.hash /bitcoin/depends/sources/src-ipc-libmultiprocess.tar
drwxrwxr-x 9 sjors users   4096 2025-02-04 09:44:29.396999115 +0000 /bitcoin/depends/../src/ipc/libmultiprocess
-rw-rw-r-- 1 sjors users    130 2025-02-04 14:55:19.385601822 +0000 /bitcoin/depends/sources/download-stamps/.stamp_fetched-native_libmultiprocess-.hash
-rw-rw-r-- 1 sjors users 296960 2025-02-04 14:55:19.383601814 +0000 /bitcoin/depends/sources/src-ipc-libmultiprocess.tar
+ rm -f /bitcoin/depends/sources/download-stamps/.stamp_fetched-native_libmultiprocess-.hash
capnproto-cxx-1.1.0.tar.gz: OK

Thanks that is interesting. It seems to be looking in the wrong location for the tarball. There is a "no such file or directory" error about:

/home/sjors/bitcoin/depends/sources/src-ipc-libmultiprocess.tar

And directory listing shows file is instead located in:

/bitcoin/depends/sources/src-ipc-libmultiprocess.tar

So I think path in checksum file needs to be corrected. It looks like other hash files use relative paths and this one is created with an absolute path.

(Also I started trying to repoduce this myself, but the build is taking a lot of time to download things so I haven't seen an error yet.)

I think following should fix the problem. It switched the checksum paths to be relative instead of absolute. (I haven't been able to fully test this yet because the guix run is still building/downloading stuff on my machine.). Will squash into #31741 if this works

--- a/depends/funcs.mk
+++ b/depends/funcs.mk
@@ -47,7 +47,8 @@ define fetch_local_dir_sha256
     fi && \
     if ! [ -f $($(1)_fetched) ] || [ -n "$$(find $($(1)_source) -newer $($(1)_fetched) | head -n1)" ]; then \
         mkdir -p $(dir $($(1)_fetched)) && \
-        $(build_SHA256SUM) $($(1)_source) > $($(1)_fetched); \
+        cd $($(1)_source_dir) && \
+        $(build_SHA256SUM) $($(1)_all_sources) > $($(1)_fetched); \
     fi && \
     cut -d" " -f1 $($(1)_fetched)
 endef
@@ -82,7 +83,8 @@ define int_get_build_properties
 $(1)_build_subdir?=.
 $(1)_download_file?=$($(1)_file_name)
 $(1)_source_dir:=$(SOURCES_PATH)
-$(1)_source:=$$($(1)_source_dir)/$(if $($(1)_local_dir),$(subst $(null) $(null),-,$(strip $(subst ., ,$(subst /, ,$($(1)_local_dir))))).tar,$($(1)_file_name))
+$(if $($(1)_file_name),,$(eval $(1)_file_name:=$(if $($(1)_local_dir),$(subst $(null) $(null),-,$(strip $(subst ., ,$(subst /, ,$($(1)_local_dir))))).tar,)))
+$(1)_source:=$$($(1)_source_dir)/$($(1)_file_name)
 $(1)_download_dir:=$(base_download_dir)/$(1)-$($(1)_version)
 $(1)_prefixbin:=$($($(1)_type)_prefix)/bin/
 $(1)_all_sources=$($(1)_file_name) $($(1)_extra_sources)

That did the trick for completing the depends build.

Though the bitcoin core build itself still fails:

/gnu/store/7k364lx15sq567ivd79pp9dmnallcxz1-bash-minimal-5.1.16/bin/sh: line 1: /bitcoin/depends/arm64-apple-darwin/bin/capnp: cannot execute binary file: Exec format error
make[2]: *** [src/ipc/libmultiprocess/CMakeFiles/multiprocess.dir/build.make:74: src/ipc/libmultiprocess/include/mp/proxy.capnp.c++] Error 126
make[1]: *** [CMakeFiles/Makefile2:1477: src/ipc/libmultiprocess/CMakeFiles/multiprocess.dir/all] Error 2
make[1]: *** Waiting for unfinished jobs....

For that I needed https://github.com/bitcoin/bitcoin/pull/30975/commits/a69e49310ffd0ebcdc2e0ca5f63c44b391bc3084, like in https://github.com/bitcoin/bitcoin/pull/30975#issuecomment-2634149344.

I'm now running a guix build for all 5 platforms that I care about for the purpose of initial Stratum v2 binaries: aarch64-apple-darwin x86_64-apple-darwin x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu

Will let you know if any fail.

The guix fix above is now included in the https://github.com/bitcoin/bitcoin/pull/31741#issuecomment-2635727935 update. So hopefully this issue is fixed, but can leave it open until that is confirmed

I was able to run the guix build after rebasing on your latest changes.

Thanks for testing, will close this issue