Refine Cargo / C++ build integration (#2080)

* Refine build process

* Update proto

* Additional refinements

* Fix lints

* Add vscode default

* Update gitignores

* Update comments

* Cleanup invalid cfg flags

* Add all flags from depends config.site

* Cleanup rust workaround args

* Fix locales  on CI

* Fix cross compilers

* Better naming

* Refactor locales properly to fit into make.sh

.cargo/config.toml

@@ -0,0 +1,5 @@
+[build]
+# We set this so that Makefile based
+# builds and ad-hoc cargo invocations all
+# result in the consistent compilation
+target-dir = "./build/lib/target"

.dockerignore

@@ -5,6 +5,7 @@
 # Editors.
 *.sw*
 .vscode
+!.vscode/settings.default.json
 .clangd
 
 # Misc

.gitignore

@@ -5,6 +5,7 @@
 # Editors.
 *.sw*
 .vscode
+!.vscode/settings.default.json
 .clangd
 
 # Misc

.vscode/settings.default.json

@@ -0,0 +1,8 @@
+{
+    // Recommended default config. Place into settings.json
+    // or copy over as `.vscode/settings.json` to take effect.
+    "rust-analyzer.assist.expressionFillDefault": "default",
+    "rust-analyzer.cargo.buildScripts.overrideCommand": ["../make.sh", "rust-analyzer-check" ],
+    "rust-analyzer.check.overrideCommand":  ["../make.sh", "rust-analyzer-check" ],
+    "rust-analyzer.workspace.symbol.search.kind": "all_symbols"
+}

Makefile.am

@@ -52,11 +52,6 @@ DIST_SHARE = \
 BIN_CHECKS=$(top_srcdir)/contrib/devtools/symbol-check.py \
            $(top_srcdir)/contrib/devtools/security-check.py
 
-COVERAGE_INFO = baseline.info \
-  test_defi_filtered.info total_coverage.info \
-  baseline_filtered.info functional_test.info functional_test_filtered.info \
-  test_defi_coverage.info test_defi.info
-
 dist-hook:
 	-$(GIT) archive --format=tar HEAD -- src/clientversion.cpp | $(AMTAR) -C $(top_distdir) -xf -
 
@@ -69,51 +64,6 @@ $(DEFI_CLI_BIN): FORCE
 $(DEFI_TX_BIN): FORCE
 	$(MAKE) -C src $(@F)
 
-if USE_LCOV
-LCOV_FILTER_PATTERN=-p "/usr/include/" -p "/usr/lib/" -p "src/leveldb/" -p "src/bench/" -p "src/univalue" -p "src/crypto/ctaes" -p "src/secp256k1"
-
-baseline.info:
-	$(LCOV) -c -i -d $(abs_builddir)/src -o $@
-
-baseline_filtered.info: baseline.info
-	$(abs_builddir)/contrib/filter-lcov.py $(LCOV_FILTER_PATTERN) $< $@
-	$(LCOV) -a $@ $(LCOV_OPTS) -o $@
-
-test_defi.info: baseline_filtered.info
-	$(MAKE) -C src/ check
-	$(LCOV) -c $(LCOV_OPTS) -d $(abs_builddir)/src -t test_defi -o $@
-	$(LCOV) -z $(LCOV_OPTS) -d $(abs_builddir)/src
-
-test_defi_filtered.info: test_defi.info
-	$(abs_builddir)/contrib/filter-lcov.py $(LCOV_FILTER_PATTERN) $< $@
-	$(LCOV) -a $@ $(LCOV_OPTS) -o $@
-
-functional_test.info: test_defi_filtered.info
-	@TIMEOUT=15 test/functional/test_runner.py $(EXTENDED_FUNCTIONAL_TESTS)
-	$(LCOV) -c $(LCOV_OPTS) -d $(abs_builddir)/src --t functional-tests -o $@
-	$(LCOV) -z $(LCOV_OPTS) -d $(abs_builddir)/src
-
-functional_test_filtered.info: functional_test.info
-	$(abs_builddir)/contrib/filter-lcov.py $(LCOV_FILTER_PATTERN) $< $@
-	$(LCOV) -a $@ $(LCOV_OPTS) -o $@
-
-test_defi_coverage.info: baseline_filtered.info test_defi_filtered.info
-	$(LCOV) -a $(LCOV_OPTS) baseline_filtered.info -a test_defi_filtered.info -o $@
-
-total_coverage.info: test_defi_filtered.info functional_test_filtered.info
-	$(LCOV) -a $(LCOV_OPTS) baseline_filtered.info -a test_defi_filtered.info -a functional_test_filtered.info -o $@ | $(GREP) "\%" | $(AWK) '{ print substr($$3,2,50) "/" $$5 }' > coverage_percent.txt
-
-test_defi.coverage/.dirstamp:  test_defi_coverage.info
-	$(GENHTML) -s $(LCOV_OPTS) $< -o $(@D)
-	@touch $@
-
-total.coverage/.dirstamp: total_coverage.info
-	$(GENHTML) -s $(LCOV_OPTS) $< -o $(@D)
-	@touch $@
-
-cov: test_defi.coverage/.dirstamp total.coverage/.dirstamp
-
-endif
 
 dist_noinst_SCRIPTS = autogen.sh
 
@@ -125,61 +75,12 @@ EXTRA_DIST += \
 
 EXTRA_DIST += \
     test/util/defi-util-test.py \
-    test/util/data/defi-util-test.json \
-    test/util/data/blanktxv1.hex \
-    test/util/data/blanktxv1.json \
-    test/util/data/blanktxv2.hex \
-    test/util/data/blanktxv2.json \
-    test/util/data/tt-delin1-out.hex \
-    test/util/data/tt-delin1-out.json \
-    test/util/data/tt-delout1-out.hex \
-    test/util/data/tt-delout1-out.json \
-    test/util/data/tt-locktime317000-out.hex \
-    test/util/data/tt-locktime317000-out.json \
-    test/util/data/tx394b54bb.hex \
-    test/util/data/txcreate1.hex \
-    test/util/data/txcreate1.json \
-    test/util/data/txcreate2.hex \
-    test/util/data/txcreate2.json \
-    test/util/data/txcreatedata1.hex \
-    test/util/data/txcreatedata1.json \
-    test/util/data/txcreatedata2.hex \
-    test/util/data/txcreatedata2.json \
-    test/util/data/txcreatedata_seq0.hex \
-    test/util/data/txcreatedata_seq0.json \
-    test/util/data/txcreatedata_seq1.hex \
-    test/util/data/txcreatedata_seq1.json \
-    test/util/data/txcreatemultisig1.hex \
-    test/util/data/txcreatemultisig1.json \
-    test/util/data/txcreatemultisig2.hex \
-    test/util/data/txcreatemultisig2.json \
-    test/util/data/txcreatemultisig3.hex \
-    test/util/data/txcreatemultisig3.json \
-    test/util/data/txcreatemultisig4.hex \
-    test/util/data/txcreatemultisig4.json \
-    test/util/data/txcreatemultisig5.json \
-    test/util/data/txcreateoutpubkey1.hex \
-    test/util/data/txcreateoutpubkey1.json \
-    test/util/data/txcreateoutpubkey2.hex \
-    test/util/data/txcreateoutpubkey2.json \
-    test/util/data/txcreateoutpubkey3.hex \
-    test/util/data/txcreateoutpubkey3.json \
-    test/util/data/txcreatescript1.hex \
-    test/util/data/txcreatescript1.json \
-    test/util/data/txcreatescript2.hex \
-    test/util/data/txcreatescript2.json \
-    test/util/data/txcreatescript3.hex \
-    test/util/data/txcreatescript3.json \
-    test/util/data/txcreatescript4.hex \
-    test/util/data/txcreatescript4.json \
-    test/util/data/txcreatesignv1.hex \
-    test/util/data/txcreatesignv1.json \
-    test/util/data/txcreatesignv2.hex \
-    test/util/rpcauth-test.py
+    test/util/rpcauth-test.py \
+    $(wildcard test/util/data/*)
 
 CLEANFILES = 
 
-.INTERMEDIATE: $(COVERAGE_INFO)
+.INTERMEDIATE: 
 
 DISTCHECK_CONFIGURE_FLAGS = 
 

configure.ac

@@ -83,15 +83,11 @@ dnl Check/return PATH for base programs.
 AC_PATH_TOOL(AR, ar)
 AC_PATH_TOOL(RANLIB, ranlib)
 AC_PATH_TOOL(STRIP, strip)
-AC_PATH_TOOL(GCOV, gcov)
-AC_PATH_PROG(LCOV, lcov)
 dnl Python 3.5 is specified in .python-version and should be used if available, see doc/dependencies.md
 AC_PATH_PROGS([PYTHON], [python3.5 python3.6 python3.7 python3.8 python3 python])
-AC_PATH_PROG(GENHTML, genhtml)
 AC_PATH_PROG([GIT], [git])
 AC_PATH_PROG(CCACHE,ccache)
-AC_PATH_PROG(XGETTEXT,xgettext)
-AC_PATH_PROG(HEXDUMP,hexdump)
+AC_PATH_PROG(HEXDUMP, hexdump)
 AC_PATH_TOOL(READELF, readelf)
 AC_PATH_TOOL(CPPFILT, c++filt)
 AC_PATH_TOOL(OBJCOPY, objcopy)
@@ -144,7 +140,7 @@ AC_ARG_ENABLE(bench,
     [use_bench=yes])
 
 AC_ARG_ENABLE([extended-functional-tests],
-    AS_HELP_STRING([--enable-extended-functional-tests],[enable expensive functional tests when using lcov (default no)]),
+    AS_HELP_STRING([--enable-extended-functional-tests],[enable expensive functional tests]),
     [use_extended_functional_tests=$enableval],
     [use_extended_functional_tests=no])
 
@@ -172,18 +168,6 @@ AC_ARG_ENABLE([ccache],
   [use_ccache=$enableval],
   [use_ccache=auto])
 
-AC_ARG_ENABLE([lcov],
-  [AS_HELP_STRING([--enable-lcov],
-  [enable lcov testing (default is no)])],
-  [use_lcov=$enableval],
-  [use_lcov=no])
-
-AC_ARG_ENABLE([lcov-branch-coverage],
-  [AS_HELP_STRING([--enable-lcov-branch-coverage],
-  [enable lcov testing branch coverage (default is no)])],
-  [use_lcov_branch=yes],
-  [use_lcov_branch=no])
-
 AC_ARG_ENABLE([glibc-back-compat],
   [AS_HELP_STRING([--enable-glibc-back-compat],
   [enable backwards compatibility with glibc])],
@@ -610,32 +594,6 @@ if test x$use_extended_functional_tests != xno; then
   AC_SUBST(EXTENDED_FUNCTIONAL_TESTS, --extended)
 fi
 
-if test x$use_lcov = xyes; then
-  if test x$LCOV = x; then
-    AC_MSG_ERROR("lcov testing requested but lcov not found")
-  fi
-  if test x$GCOV = x; then
-    AC_MSG_ERROR("lcov testing requested but gcov not found")
-  fi
-  if test x$PYTHON = x; then
-    AC_MSG_ERROR("lcov testing requested but python not found")
-  fi
-  if test x$GENHTML = x; then
-    AC_MSG_ERROR("lcov testing requested but genhtml not found")
-  fi
-  LCOV="$LCOV --gcov-tool=$GCOV"
-  AX_CHECK_LINK_FLAG([[--coverage]], [LDFLAGS="$LDFLAGS --coverage"],
-    [AC_MSG_ERROR("lcov testing requested but --coverage linker flag does not work")])
-  AX_CHECK_COMPILE_FLAG([--coverage],[CXXFLAGS="$CXXFLAGS --coverage"],
-    [AC_MSG_ERROR("lcov testing requested but --coverage flag does not work")])
-  AC_DEFINE(USE_COVERAGE, 1, [Define this symbol if coverage is enabled])
-  CXXFLAGS="$CXXFLAGS -Og"
-fi
-
-if test x$use_lcov_branch != xno; then
-  AC_SUBST(LCOV_OPTS, "$LCOV_OPTS --rc lcov_branch_coverage=1")
-fi
-
 dnl Check for endianness
 AC_C_BIGENDIAN
 
@@ -1335,7 +1293,6 @@ AM_CONDITIONAL([ENABLE_WALLET],[test x$enable_wallet = xyes])
 AM_CONDITIONAL([ENABLE_TESTS],[test x$BUILD_TEST = xyes])
 AM_CONDITIONAL([ENABLE_FUZZ],[test x$enable_fuzz = xyes])
 AM_CONDITIONAL([ENABLE_BENCH],[test x$use_bench = xno])
-AM_CONDITIONAL([USE_LCOV],[test x$use_lcov = xyes])
 AM_CONDITIONAL([GLIBC_BACK_COMPAT],[test x$use_glibc_compat = xyes])
 AM_CONDITIONAL([HARDEN],[test x$use_hardening = xyes])
 AM_CONDITIONAL([ENABLE_SSE42],[test x$enable_sse42 = xyes])
@@ -1412,8 +1369,7 @@ AC_SUBST(HAVE_STRONG_GETAUXVAL)
 AC_SUBST(PROTOC)
 AC_SUBST(PROTOC_INCLUDE_DIR)
 
-dnl If the host and build triplets are the same, we don't 
-dnl keep this empty
+dnl If the host and build triplets are the same, we keep this empty
 if test x$host = x$build; then
   RUST_TARGET=
 else
@@ -1423,6 +1379,7 @@ else
   fi
 fi
 AC_SUBST(RUST_TARGET)
+AM_CONDITIONAL([HAVE_RUST_TARGET], [test x$RUST_TARGET != x])
 
 AC_CONFIG_FILES([Makefile src/Makefile lib/Makefile test/config.ini])
 AC_CONFIG_FILES([contrib/devtools/split-debug.sh],[chmod +x contrib/devtools/split-debug.sh])
@@ -1485,32 +1442,32 @@ esac
 
 echo
 echo "Options used to compile and link:"
-echo "  with wallet   = $enable_wallet"
-echo "  with zmq      = $use_zmq"
-echo "  with test     = $use_tests"
+echo "  with wallet         = $enable_wallet"
+echo "  with zmq            = $use_zmq"
+echo "  with test           = $use_tests"
 if test x$use_tests != xno; then
-    echo "    with prop   = $enable_property_tests"
-    echo "    with fuzz   = $enable_fuzz"
+    echo "    with prop         = $enable_property_tests"
+    echo "    with fuzz         = $enable_fuzz"
 fi
-echo "  with bench    = $use_bench"
-echo "  with upnp     = $use_upnp"
-echo "  use asm       = $use_asm"
-echo "  sanitizers    = $use_sanitizers"
-echo "  debug enabled = $enable_debug"
-echo "  gprof enabled = $enable_gprof"
-echo "  werror        = $enable_werror"
+echo "  with bench          = $use_bench"
+echo "  with upnp           = $use_upnp"
+echo "  use asm             = $use_asm"
+echo "  sanitizers          = $use_sanitizers"
+echo "  debug enabled       = $enable_debug"
+echo "  gprof enabled       = $enable_gprof"
+echo "  werror              = $enable_werror"
 echo
-echo "  target os     = $TARGET_OS"
-echo "  build os      = $BUILD_OS"
+echo "  target os           = $TARGET_OS"
+echo "  build os            = $BUILD_OS"
 echo
-echo "  CC            = $CC"
-echo "  CFLAGS        = $CFLAGS"
-echo "  CPPFLAGS      = $DEBUG_CPPFLAGS $HARDENED_CPPFLAGS $CPPFLAGS"
-echo "  CXX           = $CXX"
-echo "  CXXFLAGS      = $DEBUG_CXXFLAGS $HARDENED_CXXFLAGS $WARN_CXXFLAGS $NOWARN_CXXFLAGS $ERROR_CXXFLAGS $GPROF_CXXFLAGS $CXXFLAGS"
-echo "  LDFLAGS       = $PTHREAD_CFLAGS $HARDENED_LDFLAGS $GPROF_LDFLAGS $LDFLAGS"
-echo "  ARFLAGS       = $ARFLAGS"
-echo "  CARGO         = $CARGO"
-echo "  ENABLE_DEBUG  = $ENABLE_DEBUG"
-echo "  RUST_TARGET   = $RUST_TARGET"
+echo "  CC                  = $CC"
+echo "  CFLAGS              = $CFLAGS"
+echo "  CPPFLAGS            = $DEBUG_CPPFLAGS $HARDENED_CPPFLAGS $CPPFLAGS"
+echo "  CXX                 = $CXX"
+echo "  CXXFLAGS            = $DEBUG_CXXFLAGS $HARDENED_CXXFLAGS $WARN_CXXFLAGS $NOWARN_CXXFLAGS $ERROR_CXXFLAGS $GPROF_CXXFLAGS $CXXFLAGS"
+echo "  LDFLAGS             = $PTHREAD_CFLAGS $HARDENED_LDFLAGS $GPROF_LDFLAGS $LDFLAGS"
+echo "  ARFLAGS             = $ARFLAGS"
+echo "  CARGO               = $CARGO"
+echo "  RUST_TARGET         = $RUST_TARGET"
+echo "  ENABLE_DEBUG        = $ENABLE_DEBUG"
 echo

depends/packages/protobuf.mk

@@ -1,28 +1,23 @@
 package=protobuf
-$(package)_version=22.2
-$(package)_download_path=https://github.com/protocolbuffers/protobuf/releases/download/v22.2/
+$(package)_version=23.3
+$(package)_download_path=https://github.com/protocolbuffers/protobuf/releases/download/v$($(package)_version)/
 
 # NOTE: protoc gets invoked on the BUILD OS during compile. So, we don't
 # care about HOST OS, unlike all other dependencies. 
 # That is: protoc is run on the BUILD OS, not targeted to run on the 
 # HOST OS.
 
-# TODO: Fill up hashes later
 ifeq ($(build_os)-$(build_arch),linux-x86_64)
   $(package)_file_name=protoc-$($(package)_version)-linux-x86_64.zip
-  $(package)_sha256_hash=15f281b36897e0ffbbe3a02f687ff9108c7a0f98bb653fb433e4bd62e698abe7
+  $(package)_sha256_hash=8f5abeb19c0403a7bf6e48f4fa1bb8b97724d8701f6823a327922df8cc1da4f5
 endif
 ifeq ($(build_os)-$(build_arch),linux-aarch64)
   $(package)_file_name=protoc-$($(package)_version)-linux-aarch_64.zip
-  $(package)_sha256_hash=aa2efbb2d481b7ad3c2428e0aa4dd6d813e4538e6c0a1cd4d921ac998187e07e
+  $(package)_sha256_hash=4e5154e51744c288ca1362f9cca942188003fc7b860431a984a30cb1e73aed9e
 endif
-ifeq ($(build_os)-$(build_arch),darwin-x86_64)
-  $(package)_file_name=protoc-$($(package)_version)-osx-x86_64.zip
-  $(package)_sha256_hash=8bb75680c376190d960ef1d073618c1103960f70dc4fafa7bde872029562aec1
-endif
-ifeq ($(build_os)-$(build_arch),darwin-arm)
-  $(package)_file_name=protoc-$($(package)_version)-osx-aarch_64.zip
-  $(package)_sha256_hash=a196fd39acd312688b58e81266fd88e9f7799967c5439738c10345a29049261d
+ifeq ($(build_os),darwin)
+  $(package)_file_name=protoc-$($(package)_version)-osx-universal_binary.zip
+  $(package)_sha256_hash=2783d50e35cbb546c0d75fbc2a46f76e2c620fb66c9f6bfc5ff70f5dda234100
 endif
 
 ifeq ($($(package)_file_name),)