From dc4ec6d3dbb5018cdc059eebb55994fab5d23679 Mon Sep 17 00:00:00 2001 From: Cory Fields Date: Mon, 4 Apr 2016 15:48:18 -0400 Subject: [PATCH 1/4] depends: create a hostid and buildid and add option for salts These add very simple sanity checks to ensure that the build/host toolchains have not changed since the last run. If they have, all ids will change and packages will be rebuilt. For more complicated usage (like parsing dpkg), HOST_ID_SALT/BUILD_ID_SALT may be used to introduce arbitrary data to the ids. --- depends/Makefile | 18 +++++++++++++++++- depends/README.md | 2 ++ depends/funcs.mk | 2 +- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/depends/Makefile b/depends/Makefile index ef5a20e6c..7ae14524e 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -15,6 +15,8 @@ BASEDIR = $(CURDIR) HASH_LENGTH:=11 DOWNLOAD_CONNECT_TIMEOUT:=10 DOWNLOAD_RETRIES:=3 +HOST_ID_SALT ?= salt +BUILD_ID_SALT ?= salt host:=$(BUILD) ifneq ($(HOST),) @@ -73,6 +75,20 @@ include builders/$(build_os).mk include builders/default.mk include packages/packages.mk +build_id_string:=$(BUILD_ID_SALT) +build_id_string+=$(shell $(build_CC) --version 2>/dev/null) +build_id_string+=$(shell $(build_AR) --version 2>/dev/null) +build_id_string+=$(shell $(build_CXX) --version 2>/dev/null) +build_id_string+=$(shell $(build_RANLIB) --version 2>/dev/null) +build_id_string+=$(shell $(build_STRIP) --version 2>/dev/null) + +$(host_arch)_$(host_os)_id_string:=$(HOST_ID_SALT) +$(host_arch)_$(host_os)_id_string+=$(shell $(host_CC) --version 2>/dev/null) +$(host_arch)_$(host_os)_id_string+=$(shell $(host_AR) --version 2>/dev/null) +$(host_arch)_$(host_os)_id_string+=$(shell $(host_CXX) --version 2>/dev/null) +$(host_arch)_$(host_os)_id_string+=$(shell $(host_RANLIB) --version 2>/dev/null) +$(host_arch)_$(host_os)_id_string+=$(shell $(host_STRIP) --version 2>/dev/null) + qt_packages_$(NO_QT) = $(qt_packages) $(qt_$(host_os)_packages) qt_native_packages_$(NO_QT) = $(qt_native_packages) wallet_packages_$(NO_WALLET) = $(wallet_packages) @@ -90,7 +106,7 @@ include funcs.mk toolchain_path=$($($(host_arch)_$(host_os)_native_toolchain)_prefixbin) final_build_id_long+=$(shell $(build_SHA256SUM) config.site.in) -final_build_id+=$(shell echo -n $(final_build_id_long) | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH)) +final_build_id+=$(shell echo -n "$(final_build_id_long)" | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH)) $(host_prefix)/.stamp_$(final_build_id): $(native_packages) $(packages) $(AT)rm -rf $(@D) $(AT)mkdir -p $(@D) diff --git a/depends/README.md b/depends/README.md index d85e652c6..271bbd80b 100644 --- a/depends/README.md +++ b/depends/README.md @@ -38,6 +38,8 @@ The following can be set when running make: make FOO=bar NO_WALLET: Don't download/build/cache libs needed to enable the wallet NO_UPNP: Don't download/build/cache packages needed for enabling upnp DEBUG: disable some optimizations and enable more runtime checking + HOST_ID_SALT: Optional salt to use when generating host package ids + BUILD_ID_SALT: Optional salt to use when generating build package ids If some packages are not built, for example `make NO_WALLET=1`, the appropriate options will be passed to bitcoin's configure. In this case, `--disable-wallet`. diff --git a/depends/funcs.mk b/depends/funcs.mk index 050a9b132..45bb96dd4 100644 --- a/depends/funcs.mk +++ b/depends/funcs.mk @@ -39,7 +39,7 @@ define int_get_build_id $(eval $(1)_dependencies += $($(1)_$(host_arch)_$(host_os)_dependencies) $($(1)_$(host_os)_dependencies)) $(eval $(1)_all_dependencies:=$(call int_get_all_dependencies,$(1),$($($(1)_type)_native_toolchain) $($(1)_dependencies))) $(foreach dep,$($(1)_all_dependencies),$(eval $(1)_build_id_deps+=$(dep)-$($(dep)_version)-$($(dep)_recipe_hash))) -$(eval $(1)_build_id_long:=$(1)-$($(1)_version)-$($(1)_recipe_hash)-$(release_type) $($(1)_build_id_deps)) +$(eval $(1)_build_id_long:=$(1)-$($(1)_version)-$($(1)_recipe_hash)-$(release_type) $($(1)_build_id_deps) $($($(1)_type)_id_string)) $(eval $(1)_build_id:=$(shell echo -n "$($(1)_build_id_long)" | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH))) final_build_id_long+=$($(package)_build_id_long) From fe740f14690f70fbb84287595049afc719a5bd3b Mon Sep 17 00:00:00 2001 From: Cory Fields Date: Thu, 3 Mar 2016 01:01:30 -0500 Subject: [PATCH 2/4] depends: fix fallback downloads In some cases, failed downloads wouldn't trigger a fallback download attempt. Namely, checksum mismatches. --- depends/funcs.mk | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/depends/funcs.mk b/depends/funcs.mk index 45bb96dd4..15e404e42 100644 --- a/depends/funcs.mk +++ b/depends/funcs.mk @@ -19,15 +19,19 @@ define int_get_all_dependencies $(sort $(foreach dep,$(2),$(2) $(call int_get_all_dependencies,$(1),$($(dep)_dependencies)))) endef -define fetch_file -(test -f $$($(1)_source_dir)/$(4) || \ - ( mkdir -p $$($(1)_download_dir) && echo Fetching $(1)... && \ - ( $(build_DOWNLOAD) "$$($(1)_download_dir)/$(4).temp" "$(2)/$(3)" || \ - $(build_DOWNLOAD) "$$($(1)_download_dir)/$(4).temp" "$(FALLBACK_DOWNLOAD_PATH)/$(3)" ) && \ +define fetch_file_inner + ( mkdir -p $$($(1)_download_dir) && echo Fetching $(3) from $(2) && \ + $(build_DOWNLOAD) "$$($(1)_download_dir)/$(4).temp" "$(2)/$(3)" && \ echo "$(5) $$($(1)_download_dir)/$(4).temp" > $$($(1)_download_dir)/.$(4).hash && \ $(build_SHA256SUM) -c $$($(1)_download_dir)/.$(4).hash && \ mv $$($(1)_download_dir)/$(4).temp $$($(1)_source_dir)/$(4) && \ - rm -rf $$($(1)_download_dir) )) + rm -rf $$($(1)_download_dir) ) +endef + +define fetch_file + ( test -f $$($(1)_source_dir)/$(4) || \ + ( $(call fetch_file_inner,$(1),$(2),$(3),$(4),$(5)) || \ + $(call fetch_file_inner,$(1),$(FALLBACK_DOWNLOAD_PATH),$(3),$(4),$(5)))) endef define int_get_build_recipe_hash From bb717f43756cf2754b6ed6fc54d52b535ed25fb2 Mon Sep 17 00:00:00 2001 From: Cory Fields Date: Mon, 4 Apr 2016 15:44:02 -0400 Subject: [PATCH 3/4] depends: fix "unexpected operator" error during "make download" --- depends/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/depends/Makefile b/depends/Makefile index 7ae14524e..3ddfc85a4 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -147,9 +147,9 @@ endef define check_or_remove_sources mkdir -p $($(package)_source_dir); cd $($(package)_source_dir); \ - $(build_SHA256SUM) -c $($(package)_fetched) >/dev/null 2>/dev/null || \ - ( if test -f $($(package)_all_sources); then echo "Checksum missing or mismatched for $(package) source. Forcing re-download."; fi; \ - rm -f $($(package)_all_sources) $($(1)_fetched)) + test -f $($(package)_fetched) && ( $(build_SHA256SUM) -c $($(package)_fetched) >/dev/null 2>/dev/null || \ + ( echo "Checksum missing or mismatched for $(package) source. Forcing re-download."; \ + rm -f $($(package)_all_sources) $($(1)_fetched))) || true endef check-packages: From 11d9f6b8b82df04e9cf0ba7d8434eaa3fe394ee5 Mon Sep 17 00:00:00 2001 From: Cory Fields Date: Mon, 4 Apr 2016 18:20:20 -0400 Subject: [PATCH 4/4] depends: qt/cctools: fix checksum checksum tests Checksums were being verified after download, but not again before extraction --- depends/packages/native_cctools.mk | 4 ++++ depends/packages/qt.mk | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/depends/packages/native_cctools.mk b/depends/packages/native_cctools.mk index 1c1bcf199..b5603a8d4 100644 --- a/depends/packages/native_cctools.mk +++ b/depends/packages/native_cctools.mk @@ -17,6 +17,10 @@ $(call fetch_file,$(package),$($(package)_clang_download_path),$($(package)_clan endef define $(package)_extract_cmds + mkdir -p $($(package)_extract_dir) && \ + echo "$($(package)_sha256_hash) $($(package)_source)" > $($(package)_extract_dir)/.$($(package)_file_name).hash && \ + echo "$($(package)_clang_sha256_hash) $($(package)_source_dir)/$($(package)_clang_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \ + $(build_SHA256SUM) -c $($(package)_extract_dir)/.$($(package)_file_name).hash && \ mkdir -p toolchain/bin toolchain/lib/clang/3.5/include && \ tar --strip-components=1 -C toolchain -xf $($(package)_source_dir)/$($(package)_clang_file_name) && \ echo "#!/bin/sh" > toolchain/bin/$(host)-dsymutil && \ diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index 901b761fd..77df77b73 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -108,8 +108,8 @@ endef define $(package)_extract_cmds mkdir -p $($(package)_extract_dir) && \ echo "$($(package)_sha256_hash) $($(package)_source)" > $($(package)_extract_dir)/.$($(package)_file_name).hash && \ - echo "$($(package)_qttranslations_sha256_hash) $($(package)_source_dir)/$($(package)_qttranslations_file_name)" > $($(package)_extract_dir)/.$($(package)_file_name).hash && \ - echo "$($(package)_qttools_sha256_hash) $($(package)_source_dir)/$($(package)_qttools_file_name)" > $($(package)_extract_dir)/.$($(package)_file_name).hash && \ + echo "$($(package)_qttranslations_sha256_hash) $($(package)_source_dir)/$($(package)_qttranslations_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \ + echo "$($(package)_qttools_sha256_hash) $($(package)_source_dir)/$($(package)_qttools_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \ $(build_SHA256SUM) -c $($(package)_extract_dir)/.$($(package)_file_name).hash && \ mkdir qtbase && \ tar --strip-components=1 -xf $($(package)_source) -C qtbase && \