From 5f62ed90f6d6629477617872cd808fdbd8efe169 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Haf?= <74937347+Retropex@users.noreply.github.com> Date: Fri, 4 Aug 2023 18:40:19 +0200 Subject: [PATCH] set DEFAULT_PERMIT_BAREMULTISIG to false MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Release note for changing -permitbaremultisig config default to false Remove unused MessageStartChars parameters from BlockManager methods tracepoints: Disables `-Wgnu-zero-variadic-macro-arguments` to compile without warnings Fixes #26916 by disabling the warning `-Wgnu-zero-variadic-macro-arguments` when clang is used as the compiler. lint: drop DIR_IWYU global lint: remove lint-logs.py lint: remove /* Continued */ markers from codebase refactor: fix unterminated LogPrintf()s tidy: Integrate bicoin-tidy clang-tidy plugin Enable `bitcoin-unterminated-logprintf`. Co-authored-by: Cory Fields doc: remove Fedora libdb4-*-devel install docs These are no-longer installable on any recent Fedora (33+). Remove the install instructions. Fix the typo in the Ubuntu/Debian instructions. refactor: Wrap DestroyDB in dbwrapper helper Wrap leveldb::DestroyDB in a helper function without exposing leveldb-specifics. Also, add missing optional include. The context of this commit is an effort to decouple the dbwrapper header file from leveldb includes. To this end, the includes are moved to the dbwrapper implementation file. This is done as part of the kernel project to reduce the number of required includes for users of the kernel. refactor: Split dbwrapper CDBBatch::Write implementation Keep the generic serialization in the header, while moving leveldb-specifics to the implementation file. The context of this commit is an effort to decouple the dbwrapper header file from leveldb includes. To this end, the includes are moved to the dbwrapper implementation file. This is done as part of the kernel project to reduce the number of required includes for users of the kernel. refactor: Split dbwrapper CDBatch::Erase implementation Keep the generic serialization in the header, while moving leveldb-specifics to the implementation file. The context of this commit is an effort to decouple the dbwrapper header file from leveldb includes. To this end, the includes are moved to the dbwrapper implementation file. This is done as part of the kernel project to reduce the number of required includes for users of the kernel. refactor: Pimpl leveldb::batch for CDBBatch Hide the leveldb::WriteBatch member variable with a pimpl in order not to expose it directly in the header. Also move CDBBatch::Clear to the dbwrapper implementation to use the new impl_batch. The context of this commit is an effort to decouple the dbwrapper header file from leveldb includes. To this end, the includes are moved to the dbwrapper implementation file. This is done as part of the kernel project to reduce the number of required includes for users of the kernel. refactor: Split dbwrapper CDBIterator::Seek implementation Keep the generic serialization in the header, while moving leveldb-specifics to the implementation file. The context of this commit is an effort to decouple the dbwrapper header file from leveldb includes. To this end, the includes are moved to the dbwrapper implementation file. This is done as part of the kernel project to reduce the number of required includes for users of the kernel. refactor: Split dbwrapper CDBIterator::GetKey implementation Keep the generic serialization in the header, while moving leveldb-specifics to the implementation file. The context of this commit is an effort to decouple the dbwrapper header file from leveldb includes. To this end, the includes are moved to the dbwrapper implementation file. This is done as part of the kernel project to reduce the number of required includes for users of the kernel. refactor: Split dbwrapper CDBIterator::GetValue implementation Keep the generic serialization in the header, while moving leveldb-specifics to the implementation file. The context of this commit is an effort to decouple the dbwrapper header file from leveldb includes. To this end, the includes are moved to the dbwrapper implementation file. This is done as part of the kernel project to reduce the number of required includes for users of the kernel. refactor: Pimpl leveldb::Iterator for CDBIterator Hide the leveldb::Iterator member variable with a pimpl in order not to expose it directly in the header. Also, move CDBWrapper::NewIterator to the dbwrapper implementation to use the pimpl for CDBIterator initialziation. The context of this commit is an effort to decouple the dbwrapper header file from leveldb includes. To this end, the includes are moved to the dbwrapper implementation file. This is done as part of the kernel project to reduce the number of required includes for users of the kernel. refactor: Split dbwrapper CDBWrapper::Read implementation Keep the generic serialization in the header, while moving leveldb-specifics to the implementation file. The context of this commit is an effort to decouple the dbwrapper header file from leveldb includes. To this end, the includes are moved to the dbwrapper implementation file. This is done as part of the kernel project to reduce the number of required includes for users of the kernel. refactor: Fix logging.h includes These were uncovered as missing by the next commit. refactor: Split dbwrapper CDBWrapper::Exists implementation Keep the generic serialization in the header, while moving leveldb-specifics to the implementation file. The context of this commit is an effort to decouple the dbwrapper header file from leveldb includes. To this end, the includes are moved to the dbwrapper implementation file. This is done as part of the kernel project to reduce the number of required includes for users of the kernel. refactor: Move HandleError to dbwrapper implementation Make it a static function in dbwrapper.cpp, since it is not used elsewhere and when left in the header, would expose a leveldb type. The context of this commit is an effort to decouple the dbwrapper header file from leveldb includes. To this end, the includes are moved to the dbwrapper implementation file. This is done as part of the kernel project to reduce the number of required includes for users of the kernel. refactor: Split dbwrapper CDBWrapper::EstimateSize implementation Keep the generic serialization in the header, while moving leveldb-specifics to the implementation file. Since CharCast is no longer needed in the header, move it to the implementation file. The context of this commit is an effort to decouple the dbwrapper header file from leveldb includes. To this end, the includes are moved to the dbwrapper implementation file. This is done as part of the kernel project to reduce the number of required includes for users of the kernel. refactor: Move CDBWrapper leveldb members to their own context struct The context of this commit is an effort to decouple the dbwrapper header file from leveldb includes. To this end, the includes are moved to the dbwrapper implementation file. This is done as part of the kernel project to reduce the number of required includes for users of the kernel. build: Remove leveldb from BITCOIN_INCLUDES Since leveldb is no longer in our header tree, move its include flags to whereever dbwrapper.cpp is built. refactor: Correct dbwrapper key naming The ss- prefix should connotate a DataStream variable. Now that these variables are byte spans, drop the prefix. ci: Use qemu-user through container engine doc: document PeerManager::Options members net processing: clamp -maxorphantx to uint32_t bounds net processing: clamp -blockreconstructionextratxn to uint32_t bounds Also changes max_extra_txs into a uint32_t to avoid platform-specific behaviour doc: Add release note crypto: remove outdated variant of ChaCha20Poly1305 AEAD Remove the variant of ChaCha20Poly1305 AEAD that was previously added in anticipation of BIP324 using it. BIP324 was updated to instead use rekeying wrappers around otherwise unmodified versions of the ChaCha20 stream cipher and the ChaCha20Poly1305 AEAD as specified in RFC8439. crypto: add the ChaCha20Poly1305 AEAD as specified in RFC8439 This adds an implementation of the ChaCha20Poly1305 AEAD exactly matching the version specified in RFC8439 section 2.8, including tests and official test vectors. crypto: add FSChaCha20, a rekeying wrapper around ChaCha20 This adds the FSChaCha20 stream cipher as specified in BIP324, a wrapper around the ChaCha20 stream cipher (specified in RFC8439 section 2.4) which automatically rekeys every N messages, and manages the nonces used for encryption. Co-authored-by: dhruv <856960+dhruv@users.noreply.github.com> crypto: add FSChaCha20Poly1305, rekeying wrapper around ChaCha20Poly1305 This adds the FSChaCha20Poly1305 AEAD as specified in BIP324, a wrapper around the ChaCha20Poly1305 AEAD (as specified in RFC8439 section 2.8) which automatically rekeys every N messages, and automatically increments the nonce every message. bench: add benchmark for FSChaCha20Poly1305 Add a benchmark for FSChaCha20Poly1305 encryption, so the overhead of key generation and authentication can be observed for various message sizes. crypto: support split plaintext in ChaCha20Poly1305 Encrypt/Decrypt Add BIP324Cipher, encapsulating key agreement, derivation, and stream/AEAD ciphers Co-authored-by: dhruv <856960+dhruv@users.noreply.github.com> tests: add decryption test to bip324_tests doc: use llvm-config for bitcoin-tidy example An LLVM installation will have `llvm-config` available to query for info. Ask it for the `--cmakedir`, and use that in our bitcoin-tidy example, rather than listing multiple different (potential) paths per distro/OS etc. bitcoin-tidy: fix macOS build LLVM uses these options for building as well, so there's precedent. Also fix the shared library extension which was incorrectly being set to dylib test: locked_wallet, skip default fee estimation Same as we do with the nodes default wallets. No test case on this file is meant to exercise fee estimation. fuzz: coins_view: correct an incorrect assertion It is incorrect to assert that `cache.HaveCoin()` will always be `true` if `backend.HaveCoin()` is. The coin could well have been marked as spent in the cache but not yet flushed, in which case `cache.HaveCoin()` would return `false`. Note this was never hit because `exists_using_have_coin_in_backend` is currently never `true` (it's the default implementation of `CCoinsView`. However this might change if we were to add a target where the backend is a `CCoinsViewDB`. fuzz: coins_view: remove an incorrect assertion Again, this was not hit because the default implementation of `CCoinsView` return `false` for `GetCoin`. crypto: BIP324 ciphersuite follow-up follow-up to #28008. * move `dummy_tag` variable in FSChaCha20Poly1305 crypto_tests outside of the loop to be reused every time * use easy to read `cipher.last()` in `AEADChaCha20Poly1305::Decrypt()` * comment for initiator in `BIP324Cipher::Initialize()` * systematically damage ciphertext with bit positions in bip324_tests * use 4095 max bytes for aad in bip324 fuzz test refactor: add missing headers for BIP324 ciphersuite ci: Drop no longer needed `macos_sdk_cache` It has been cached in the Docker image since https://github.com/bitcoin/bitcoin/pull/27028. ci: Drop BASE_SCRATCH_DIR from LIBCXX_DIR Using a hard-coded path avoids non-determinism issues and improves CI UX. ci: Only create folders when needed Now that container volumes are used, the folders are no longer mounted. They are only needed when running without a container engine (docker, podman). ci: Use hard-coded root path for CI containers ci: Run "macOS native x86_64" job on GitHub Actions Also, the "macOS native arm64" task has been removed from Cirrus CI. Remove unused boost signals2 from torcontrol iwyu on torcontrol Replace LocaleIndependentAtoi with ToIntegral No need for saturating behavior when the int is composed of 3 digits. remove unused limits.h include in compat.h Sort includes in compat.h Can be reviewed with: --color-moved=blocks --color-moved-ws=ignore-all-space --ignore-all-space test: check backup from `migratewallet` can be successfully restored ci: Fix macOS-cross SDK rsync This should fix the macOS-cross build on Cirrus CI containers. Locally this was already working, because the SDK was cached in /ci_container_base/ in the image, which is also the folder used for a later CI run. However, on Cirrus CI, when using an image *and* a custom BASE_ROOT_DIR, the SDK will not be found in /ci_base_install/, nor in BASE_ROOT_DIR. Fix this by normalizing *all* folders to /ci_container_base/. ci: Avoid error on macOS native This avoids "mkdir: /ci_container_base: Read-only file system" mempool_entry: add mempool entry sequence number validation: when adding txs due to a block reorg, allow immediate relay net_processing: drop m_recently_announced_invs bloom filter Rather than using a bloom filter to track announced invs, simply allow a peer to request any tx that entered the mempool prior to the last INV message we sent them. This also obsoletes the UNCONDITIONAL_RELAY_DELAY. net_processing: don't add txids to m_tx_inventory_known_filter We no longer have m_recently_announced_invs, so there is no need to add txids to m_tx_inventory_known_filter to dedupe that filter. test: Check tx from disconnected block is immediately requestable Check that peers can immediately request txs from blocks that have been reorged out and are now in our mempool. net_processing: Clean up INVENTORY_BROADCAST_MAX constants mempool_entry: improve struct packing ci: Move tidy to persistent worker refactor: Remove PERSISTENT_WORKER_* yaml templates * PERSISTENT_WORKER_TEMPLATE_ENV is not needed at all, because RESTART_CI_DOCKER_BEFORE_RUN is already set on the persistent worker. * PERSISTENT_WORKER_TEMPLATE can be replaced by pinning the previous_releases task to a type of worker. This should make the CI performance more consistent. Move MANDATORY_SCRIPT_VERIFY_FLAGS from script/standard.h to policy/policy.h Remove ScriptHash from CScriptID constructor Replaces the constructor in CScriptID that converts a ScriptHash with a function ToScriptID that does the same. This prepares for a move of CScriptID to avoid a circular dependency. Move CScriptID to script.{h/cpp} CScriptID should be next to CScript just as CKeyID is next to CPubKey Move Taproot{SpendData/Builder} to signingprovider.{h/cpp} TaprootSpendData and TaprootBuilder are used in signing in SigningProvider contexts, so they should live near that. Move CTxDestination to its own file CTxDestination is really our internal representation of an address and doesn't really have anything to do with standard script types, so move them to their own file. MOVEONLY: Move datacarrier defaults to policy.h Clean up things that include script/standard.h Remove standard.h from files that don't use anything in it, and include it in files that do. Clean up script/standard.{h/cpp} includes Rename script/standard.{cpp/h} to script/solver.{cpp/h} Since script/standard only contains things that are used by the Solver and its callers, rename the files to script/solver. Rework receive buffer pushback Co-authored-by: Anthony Towns test: Fix intermittent issue in mining_getblocktemplate_longpoll.py Bugfix: RPC: Remove quotes from non-string oneline descriptions RPC/rpcdoccheck: Error if a oneline_description has a quote for a non-string RPC/Mining: Document template_request better for getblocktemplate ci: Ensure that only a single workflow processes `github.ref` at a time ci: Refactor: Remove CI_USE_APT_INSTALL rpc: remove one more quote from non-string oneline description This fixes a silent conflict betwen #28123 and #27460 crypto: refactor ChaCha20 classes to use Span interface random: simplify FastRandomContext::randbytes using fillrand crypto: require key on ChaCha20 initialization fuzz: support std::byte in Consume{Fixed,Variable}LengthByteVector tests: miscellaneous hex / std::byte improvements crypto: make ChaCha20::SetKey wipe buffer doc: Fix bitcoin-unterminated-logprintf tidy comments * Move module description from test to LogPrintfCheck * Add test doc * Remove unused comment, see https://github.com/bitcoin/bitcoin/pull/26296/files#r1279351539 refactor: Enforce C-str fmt strings in WalletLogPrintf() refactor: Enable all clang-tidy plugin bitcoin tests This makes it easier to add new ones without having to modify this file every time. bitcoin-tidy: Apply bitcoin-unterminated-logprintf to spkm as well ci: Add missing ${CI_RETRY_EXE} before curl ci: Add missing amd64 to win64-cross task Also, do the same for android, which also fails. assumeutxo cleanup: Move IsInitialBlockDownload & NotifyHeaderTip to ChainstateManager This change makes IsInitialBlockDownload and NotifyHeaderTip functions no longer tied to individual Chainstate objects. It makes them work with the ChainstateManager object instead so code is simpler and it is no longer possible to call them incorrectly with an inactive Chainstate. This change also makes m_cached_finished_ibd caching easier to reason about, because now there is only one cached value instead of two (for background and snapshot chainstates) so the cached IBD state now no longer gets reset when a snapshot is loaded. There should be no change in behavior because these functions were always called on the active ChainState objects. These changes were discussed previously https://github.com/bitcoin/bitcoin/pull/27746#discussion_r1246868905 and https://github.com/bitcoin/bitcoin/pull/27746#discussion_r1237552792 as possible followups for that PR. ci: Disable cache save for pull requests in GitHub Actions Otherwise, multiple pull requests fill GitHub Actions cache quota shortly. tx fees, policy: doc: update and delete unnecessary comment test: ensure acceptstalefeeestimates is supported only on regtest chain test: fix 'unknown named parameter' test in `wallet_basic` Fixes loop when testing an unknown named parameter. Remove unused includes from txmempool.h ... and move them to where they are really needed. This was found by IWYU: txmempool.h should remove these lines: - #include // lines 29-29 - class CBlockIndex; // lines 43-43 - class Chainstate; // lines 45-45 Also, move the stdlib section to the right place. Can be reviewed with: --color-moved=dimmed-zebra move-only: Create src/kernel/mempool_removal_reason.h This is needed for a future commit. Can be reviewed with: --color-moved=dimmed-zebra Remove unused includes from blockfilter.h This removes unused includes, primitives/block found manually, and the others by iwyu: blockfilter.h should remove these lines: - #include // lines 16-16 - #include // lines 18-18 Remove unused includes from wallet.cpp This removes unused includes, such as undo.h or txmempool.h from wallet.cpp. Also, add missing ones, according to IWYU. test: Support riscv64 in get_previous_releases.py refactor: Add missing includes Refactor: Remove confusing static_cast guix: pre time-machine bump changes (Windows) Split out of #27897. This is some refactoring to the Windows Guix build that facilitates bumping our Guix time-machine. Namely, avoiding `package-with-extra-configure-variable`, which is non-functional in the newer time-machine, see https://issues.guix.gnu.org/64436. At the same time, consolidate our Windows GCC build into mingw-w64-base-gcc. Rename `gcc-10-remap-guix-store.patch` to avoid changing it whenever GCC changes. We move the old `building-on` inside `explicit-cross-configure`, so that non-windows builds continue to work. Note that `explicit-cross-configure` will be going away entirely (see #27897). [test framework] make it easier to fast-forward setmocktime Have each TestNode keep track of the last timestamp it called setmocktime with, and add a bumpmocktime() function to bump by a number of seconds. Makes it easy to fast forward n seconds without keeping track of what the last timestamp was. [functional test] transaction orphan handling ci: Switch remaining tasks to self-hosted This allows to drop unused templates, such as cirrus_ephemeral_worker_template_env, or container_depends_template. Also, ccache_cache, previous_releases_cache, and base_depends_built_cache can be dropped, because the caching is done in container volumes on the self-hosted runners. ci: Remove distro-name from task name The exact distro name should not be important. Also, it is easy to find out, if needed. Thus, remove it to avoid bloat and maintenance overhead having to keep it in sync. doc, policy: Clarify comment on STANDARD_SCRIPT_VERIFY_FLAGS Make post-p2sh consensus rules mandatory for tx relay Update help text for spend and rawtransaction rpcs fixing typo guix: consolidate glibc 2.27 package Refactor our glibc 2.27 to be a single 'package', and avoid the use of `package-with-extra-configure-variable`. This also lets us drop the `enable_werror` workaround, and just use --disable-werror directly. Employ the same workaround as the Guix glibc, to avoid a "permission denied" failure during build: ```bash make subdir=sunrpc -C sunrpc ..=../ subdir_install make[2]: Entering directory '/tmp/guix-build-glibc-cross-x86_64-linux-gnu-2.27.drv-0/source/sunrpc' .././scripts/mkinstalldirs /gnu/store/ga8jciqrd5lh52m572x3mk4q1smf5agq-glibc-cross-x86_64-linux-gnu-2.27/include/rpc mkdir -p -- /gnu/store/ga8jciqrd5lh52m572x3mk4q1smf5agq-glibc-cross-x86_64-linux-gnu-2.27/include/rpc /gnu/store/kvpvk5wh70wdbjnr83hh85rg22ysxm9h-coreutils-8.32/bin/install -c -m 644 rpc/netdb.h /gnu/store/ga8jciqrd5lh52m572x3mk4q1smf5agq-glibc-cross-x86_64-linux-gnu-2.27/include/rpc/netdb.h .././scripts/mkinstalldirs /gnu/store/ga8jciqrd5lh52m572x3mk4q1smf5agq-glibc-cross-x86_64-linux-gnu-2.27/include/nfs mkdir -p -- /gnu/store/ga8jciqrd5lh52m572x3mk4q1smf5agq-glibc-cross-x86_64-linux-gnu-2.27/include/nfs /gnu/store/kvpvk5wh70wdbjnr83hh85rg22ysxm9h-coreutils-8.32/bin/install -c -m 644 ../sysdeps/unix/sysv/linux/nfs/nfs.h /gnu/store/ga8jciqrd5lh52m572x3mk4q1smf5agq-glibc-cross-x86_64-linux-gnu-2.27/include/nfs/nfs.h /gnu/store/kvpvk5wh70wdbjnr83hh85rg22ysxm9h-coreutils-8.32/bin/install -c -m 644 /tmp/guix-build-glibc-cross-x86_64-linux-gnu-2.27.drv-0/build/gnu/lib-names-64.h /gnu/store/ga8jciqrd5lh52m572x3mk4q1smf5agq-glibc-cross-x86_64-linux-gnu-2.27/include/gnu/lib-names-64.h /gnu/store/kvpvk5wh70wdbjnr83hh85rg22ysxm9h-coreutils-8.32/bin/install -c -m 644 etc.rpc /etc/rpc /gnu/store/kvpvk5wh70wdbjnr83hh85rg22ysxm9h-coreutils-8.32/bin/install: cannot create regular file '/etc/rpc': Permission denied make[2]: *** [Makefile:197: /etc/rpc] Error 1 ``` guix: consolidate Linux GCC package Refactor our Linux GCC to be a single 'package', and avoid the use of `package-with-extra-configure-variable`. ci: Limit scope of some env vars No need to have a larger scope than needed. Can be reviewed with --color-moved=dimmed-zebra ci: Start with clean env This should help to avoid non-determinism. ci: Remove no longer applicable section This fails with: "Error: determining starting point for build: no FROM statement found" test: Fix intermittent issue in mempool_reorg ci: Use concurrency for pull requests only Otherwise, any previously pending workflow will be canceled on the following push. test: wallet_backup.py, fix intermittent failure in "restore using dumped wallet" The failure arises because the test expects 'init_wallet()' (the test framework function) creating a wallet with no keys. However, the function also imports the deterministic private key used to receive the coinbase coins. This causes a race within the "restore using dumped wallet" case, where we intend to have a new wallet (with no existing keys) to test the 'importwallet()' RPC result. The reason behind the intermittent failures might be other peers delivering the chain right after node2 startup (sync of the validation queue included) and prior to the 'node2.getbalance()' check. test: previous releases: speed up fetching sources with shallow clone For the sake of building previous releases, fetching the whole history of the repository for each version seems to be overkill as it takes much more time, bandwidth and disk space than necessary. Create a shallow clone instead with history truncated to the one commit of the version tag, which is directly checked out in the same command. This has the nice side-effect that we can remove the extra `git checkout` step after as it's not needed anymore. Note that it might look confusing to pass a _tag_ to a parameter named `--branch`, but the git-clone manpage explicitly states that this is supported. ci: Add missing docker.io prefix to CI_IMAGE_NAME_TAG rpc: Add MaybeArg() and Arg() default helper refactor: merge transport serializer and deserializer into Transport class This allows state that is shared between both directions to be encapsulated into a single object. Specifically the v2 transport protocol introduced by BIP324 has sending state (the encryption keys) that depends on received messages (the DH key exchange). Having a single object for both means it can hide logic from callers related to that key exchange and other interactions. net: add V1Transport lock protecting receive state Rather than relying on the caller to prevent concurrent calls to the various receive-side functions of Transport, introduce a private m_cs_recv inside the implementation to protect the lock state. Of course, this does not remove the need for callers to synchronize calls entirely, as it is a stateful object, and e.g. the order in which Receive(), Complete(), and GetMessage() are called matters. It seems impossible to use a Transport object in a meaningful way in a multi-threaded way without some form of external synchronization, but it still feels safer to make the transport object itself responsible for protecting its internal state. refactor: rename Transport class receive functions Now that the Transport class deals with both the sending and receiving side of things, make the receive side have function names that clearly indicate they're about receiving. * Transport::Read() -> Transport::ReceivedBytes() * Transport::Complete() -> Transport::ReceivedMessageComplete() * Transport::GetMessage() -> Transport::GetReceivedMessage() * Transport::SetVersion() -> Transport::SetReceiveVersion() Further, also update the comments on these functions to (among others) remove the "deserialization" terminology. That term is better reserved for just the serialization/deserialization between objects and bytes (see serialize.h), and not the conversion from/to wire bytes as performed by the Transport. net: abstract sending side of transport serialization further This makes the sending side of P2P transports mirror the receiver side: caller provides message (consisting of type and payload) to be sent, and then asks what bytes must be sent. Once the message has been fully sent, a new message can be provided. This removes the assumption that P2P serialization of messages follows a strict structure of header (a function of type and payload), followed by (unmodified) payload, and instead lets transports decide the structure themselves. It also removes the assumption that a message must always be sent at once, or that no bytes are even sent on the wire when there is no message. This opens the door for supporting traffic shaping mechanisms in the future. net: make V1Transport implicitly use current chainparams The rest of net.cpp already uses Params() to determine chainparams in many places (and even V1Transport itself does so in some places). Since the only chainparams dependency is through the message start characters, just store those directly in the transport. fuzz: add bidirectional fragmented transport test This adds a simulation test, with two V1Transport objects, which send messages to each other, with sending and receiving fragmented into multiple pieces that may be interleaved. It primarily verifies that the sending and receiving side are compatible with each other, plus a few sanity checks. net: measure send buffer fullness based on memory usage This more accurately captures the intent of limiting send buffer size, as many small messages can have a larger overhead that is not counted with the current approach. It also means removing the dependency on the header size (which will become a function of the transport choice) from the send buffer calculations. net: move message conversion to wire bytes from PushMessage to SocketSendData This furthers transport abstraction by removing the assumption that a message can always immediately be converted to wire bytes. This assumption does not hold for the v2 transport proposed by BIP324, as no messages can be sent before the handshake completes. This is done by only keeping (complete) CSerializedNetMsg objects in vSendMsg, rather than the resulting bytes (for header and payload) that need to be sent. In SocketSendData, these objects are handed to the transport as permitted by it, and sending out the bytes the transport tells us to send. This also removes the nSendOffset member variable in CNode, as keeping track of how much has been sent is now a responsability of the transport. This is not a pure refactor, and has the following effects even for the current v1 transport: * Checksum calculation now happens in SocketSendData rather than PushMessage. For non-optimistic-send messages, that means this computation now happens in the network thread rather than the message handler thread (generally a good thing, as the message handler thread is more of a computational bottleneck). * Checksum calculation now happens while holding the cs_vSend lock. This is technically unnecessary for the v1 transport, as messages are encoded independent from one another, but is untenable for the v2 transport anyway. * Statistics updates about per-message sent bytes now happen when those bytes are actually handed to the OS, rather than at PushMessage time. refactor: make Transport::ReceivedBytes just return success/fail fuzz: coinselection, add `CreateCoins` Move coins creation for a specific function. It allows us to use it in other parts of the code. fuzz: coinselection, add coverage for `EligibleForSpending` fuzz: coinselection, add coverage for `AddInputs` fuzz: coinselection, add coverage for `GetShuffledInputVector`/`GetInputSet` fuzz: coinselection, add coverage for `Merge` fuzz: coinselection, improve `ComputeAndSetWaste` Instead of using `cost_of_change` for `min_viable_change` and `change_cost`, and 0 for `change_fee`, use values from `coin_params`. The previous values don't generate any effects that is relevant for that context. fuzz: coinselection, compare `GetSelectedValue` with target The valid results should have a target below the sum of the selected inputs amounts. Also, it increases the minimum value for target to make it more realistic. fuzz: coinselection, BnB should never produce change fuzz: coinselection, fix `m_cost_of_change` `m_cost_of_change` must not be generated randomly independent from m_change_fee. This commit changes it to set it up according to `wallet/spend`. doc: Improve documentation of rpcallowip rpchelp Closes #21070 v21.0 introduced a behaviour changed noted in #21070 where using a config value `rpcallowip=::0` no longer also permitted ipv4 ip addresses. The rpc_bind.py functional test covers this new behaviour already by checking that the list of bind addresses exactly matches what is expected so this commit only updates the documentation. rpc: add test-only sendmsgtopeer rpc This rpc can be used when we want a node to send a message, but cannot use a python P2P object, for example for testing of low-level net transport behavior. test: add basic tests for sendmsgtopeer to rpc_net.py test: add functional test for deadlock situation guix: backport glibc patch to fix powerpc build Do this prior to bumping the time-machine, to avoid the following build failure: ```bash /tmp/guix-build-glibc-cross-powerpc64le-linux-gnu-2.27.drv-0/build/string/memset-power8.o.dt -MT /tmp/guix-build-glibc-cross-powerpc64le-linux-gnu-2.27.drv-0/build/string/memset-power8.o ../sysdeps/powerpc/powerpc64/power4/memcmp.S: Assembler messages: ../sysdeps/powerpc/powerpc64/power4/memcmp.S:87: Error: unrecognized opcode: `ldbrx' ../sysdeps/powerpc/powerpc64/power4/memcmp.S:88: Error: unrecognized opcode: `ldbrx' ../sysdeps/powerpc/powerpc64/power4/memcmp.S:112: Error: unrecognized opcode: `ldbrx' ``` See: https://sourceware.org/git/?p=glibc.git;a=commit;h=9250e6610fdb0f3a6f238d2813e319a41fb7a810. https://github.com/gcc-mirror/gcc/commit/e154242724b084380e3221df7c08fcdbd8460674. guix: update python-oscrypto to 1.3.0 This is required for bumping the time-machine, for compatibility with OpenSSL: oscrypto: openssl backend, 1.2.1, /tmp/guix-build-python-oscrypto-1.2.1.drv-0/source/oscrypto Traceback (most recent call last): File "/tmp/guix-build-python-oscrypto-1.2.1.drv-0/source/oscrypto/_openssl/_libcrypto_ctypes.py", line 304, in libcrypto.EVP_PKEY_size.argtypes = [ File "/gnu/store/9dkl9fnidcdpw19ncw5pk0p7dljx7ijb-python-3.10.7/lib/python3.10/ctypes/__init__.py", line 387, in __getattr__ func = self.__getitem__(name) File "/gnu/store/9dkl9fnidcdpw19ncw5pk0p7dljx7ijb-python-3.10.7/lib/python3.10/ctypes/__init__.py", line 392, in __getitem__ func = self._FuncPtr((name_or_ordinal, self)) AttributeError: /gnu/store/2hr7w64zhr6jjznidyc2xi40d5ynhj9c-openssl-3.0.8/lib/libcrypto.so.3: undefined symbol: EVP_PKEY_size. Did you mean: 'EVP_PKEY_free'? guix: update time-machine to 160f78a4d92205df986ed9efcce7d3aac188cb24 In our time-machine environment this changes the following: GCC 10.3.0 -> 10.4.0 Binutils 2.37 -> 2.38 Linux Libre Headers 5.15.37 -> 5.15.127 git 2.36.0 -> 2.41.0 mingw-w64 8.0.0 -> 11.0.1 NSIS 3.05 -> 3.09 xorriso 1.5.2 -> 1.5.6.pl02 Python 3.9 -> 3.10.7 Python-asn1crypto 1.4.0 -> 1.5.1 GCC 12.3.0 becomes available. LLVM 15.0.7 becomes available. guix: use cross-* keyword arguments Using the new time-machine results in warnings about consistently using keyword arguments: ```bash guix environment: warning: 'cross-kernel-headers' must be used with keyword arguments guix environment: warning: 'cross-libc' must be used with keyword arguments ``` guix: drop NSIS patch now that we use 3.09 See https://sourceforge.net/p/nsis/bugs/1283/. guix: drop Windows broken-longjmp.patch This is no-longer required, now that we are building using GCC 10.4.0. depends: use LLVM/Clang 15.0.6 for macOS cross-compile There is no x86_64 binaries for 15.0.7. guix: use clang-toolchain-15 for macOS compilation ci: Run "Win64 native" job on GitHub Actions refactor: Use HashWriter over legacy CHashWriter refactor: Use HashWriter over legacy CHashWriter (via SerializeHash) config: default acceptnonstdtxn=0 on all chains Previously, the default for acceptnonstdtxn defaulted to 0 on all chains except testnet. Change this to be consistent across all chains, and remove the parameter from chainparams entirely. doc: Release notes for testnet defaulting to -acceptnonstdtxn=0 script: replace deprecated pkg_resources with importlib.metadata in our python linter: ``` ./test/lint/lint-python.py:12: DeprecationWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html import pkg_resources ``` The importlib.metadata library was added in Python 3.8, which is currently our minimum-supported Python version. For more details about importlib.metadata, see https://docs.python.org/3/library/importlib.metadata.html ci, windows: Do not run extended functional tests for pull requests This change is intended to speed up the CI feedback for pull requests. test: Support powerpc64le in get_previous_releases.py guix: remove GCC 10 workaround from NSIS Fixed upstream in 3.06, see https://github.com/kichik/nsis/commit/229b6136c41ba5caba25936f4927476d20aa283f. https://sourceforge.net/p/nsis/bugs/1248/ doc: Remove sudo from command that is already run as root ci: Remove /ro_base bind mount Just set the bind mount to BASE_READ_ONLY_DIR, which allows to drop one line of code and makes the code easier to understand. ci: Remove unused TEST_RUNNER_ENV="LC_ALL=C" from s390x task gui: make '-min' minimize wallet loading dialog When '-min' is enabled, no loading dialog should be presented on screen during startup. doc: Fill in the required skills in the good_first_issue template [log] include wtxid in tx {relay,validation,orphanage} logging [log] add category TXPACKAGES for orphanage and package relay [log] add more logs related to orphan handling - Whenever a tx is erased. Allows somebody to see which transactions have been erased due to expiry/overflow, not just how many. - Whenever a tx is added to a peer's workset. - AcceptToMemoryPool when a tx is accepted, mirroring the one logged for a tx received from a peer. This allows someone to see all of the transactions that are accepted to mempool just by looking for ATMP logs. - MEMPOOLREJ when a tx is rejected, mirroring the one logged for a tx received from a peer. This allows someone to see all of the transaction rejections by looking at MEMPOOLREJ logs. [doc] move comment about AlreadyHaveTx DoS score to the right place This comment isn't in the right place, as detection of a tx in recent_rejects would cause the function to exit much earlier. Move the comment to the right place and tweak the first sentence for accuracy. ci: Avoid oversubscription in functional tests on Windows ci: Avoid saving the same Ccache cache This occurred when a job was being rerun. log: Print error message when coindb is in inconsistent state qt: Translation updates from Transifex The diff is generated by executing the `update-translations.py` script. qt: Bump Transifex slug for 26.x qt: Update translation source file The diff is generated by executing `make -C src translate`. removed StrFormatInternalBug quote delimitation ci: Bump `actions/checkout` version See: https://github.com/actions/checkout/releases/tag/v4.0.0 test: p2p: check that `getaddr` msgs are only responded once per connection test: remove fixed timeouts from feature_config_args They cannot be scaled by the timeout_factor option and can therefore cause timeouts in slow environments. They are also not necessary for the test, since they measure time frome startup until a debug message is encountered, which is not restricted to 1 minute by any internal logic within bitcoind. ci: Asan with -ftrivial-auto-var-init=pattern Squashed 'src/secp256k1/' changes from c545fdc374..199d27cea3 199d27cea3 Merge bitcoin-core/secp256k1#1415: release: Prepare for 0.4.0 16339804c9 release: Prepare for 0.4.0 d9a85065a9 changelog: Catch up in preparation of release 0b4640aedd Merge bitcoin-core/secp256k1#1413: ci: Add `release` job 8659a01714 ci: Add `release` job f9b38894ba ci: Update `actions/checkout` version 727bec5bc2 Merge bitcoin-core/secp256k1#1414: ci/gha: Add ARM64 QEMU jobs for clang and clang-snapshot 2635068abf ci/gha: Let MSan continue checking after errors in all jobs e78c7b68eb ci/Dockerfile: Reduce size of Docker image further 2f0d3bbffb ci/Dockerfile: Warn if `ulimit -n` is too high when running Docker 4b8a647ad3 ci/gha: Add ARM64 QEMU jobs for clang and clang-snapshot 6ebe7d2bb3 ci/Dockerfile: Always use versioned clang packages 65c79fe2d0 Merge bitcoin-core/secp256k1#1412: ci: Switch macOS from Ventura to Monterey and add Valgrind c223d7e33d ci: Switch macOS from Ventura to Monterey and add Valgrind ea26b71c3a Merge bitcoin-core/secp256k1#1411: ci: Make repetitive command the default one cce0456304 ci: Make repetitive command the default one 317a4c48f0 ci: Move `git config ...` to `run-in-docker-action` 4d7fe60905 Merge bitcoin-core/secp256k1#1409: ci: Move remained task from Cirrus to GitHub Actions 676ed8f9cf ci: Move "C++ (public headers)" from Cirrus to GitHub Actions 61fc3a2dc8 ci: Move "C++ -fpermissive..." from Cirrus to GitHub Actions d51fb0a533 ci: Move "MSan" from Cirrus to GitHub Actions c22ac27529 ci: Move sanitizers task from Cirrus to GitHub Actions 26a989924b Merge bitcoin-core/secp256k1#1410: ci: Use concurrency for pull requests only ee1be62d84 ci: Use concurrency for pull requests only 6ee14550c8 Merge bitcoin-core/secp256k1#1406: ci, gha: Move more non-x86_64 tasks from Cirrus CI to GitHub Actions fc3dea29ea ci: Move "ppc64le: Linux..." from Cirrus to GitHub Actions 7782dc8276 ci: Move "ARM64: Linux..." from Cirrus to GitHub Actions 0a16de671c ci: Move "ARM32: Linux..." from Cirrus to GitHub Actions ea33914e00 ci: Move "s390x (big-endian): Linux..." from Cirrus to GitHub Actions 880be8af99 ci: Move "i686: Linux (Debian stable)" from Cirrus to GiHub Actions 2e6cf9bae5 Merge bitcoin-core/secp256k1#1396: ci, gha: Add "x86_64: Linux (Debian stable)" GitHub Actions job 5373693e45 Merge bitcoin-core/secp256k1#1405: ci: Drop no longer needed workaround ef9fe959de ci: Drop no longer needed workaround e10878f58e ci, gha: Drop `driver-opts.network` input for `setup-buildx-action` 4ad4914bd1 ci, gha: Add `retry_builder` Docker image builder 6617a620d9 ci: Remove "x86_64: Linux (Debian stable)" task from Cirrus CI 03c9e6508c ci, gha: Add "x86_64: Linux (Debian stable)" GitHub Actions job ad3e65d9fe ci: Remove GCC build files and sage to reduce size of Docker image 6b9507adf6 Merge bitcoin-core/secp256k1#1398: ci, gha: Add Windows jobs based on Linux image 87d35f30c0 ci: Rename `cirrus.sh` to more general `ci.sh` d6281dd008 ci: Remove Windows tasks from Cirrus CI 2b6f9cd546 ci, gha: Add Windows jobs based on Linux image 48b1d939b5 Merge bitcoin-core/secp256k1#1403: ci, gha: Ensure only a single workflow processes `github.ref` at a time 0ba2b94551 Merge bitcoin-core/secp256k1#1373: Add invariant checking for scalars 060e32cb60 Merge bitcoin-core/secp256k1#1401: ci, gha: Run all MSVC tests on Windows natively de657c2044 Merge bitcoin-core/secp256k1#1062: Removes `_fe_equal_var`, and unwanted `_fe_normalize_weak` calls (in tests) bcffeb14bc Merge bitcoin-core/secp256k1#1404: ci: Remove "arm64: macOS Ventura" task from Cirrus CI c2f6435802 ci: Add comment about switching macOS to M1 on GHA later 4a24fae0bc ci: Remove "arm64: macOS Ventura" task from Cirrus CI b0886fd35c ci, gha: Ensure only a single workflow processes `github.ref` at a time 3d05c86d63 Merge bitcoin-core/secp256k1#1394: ci, gha: Run "x86_64: macOS Ventura" job on GitHub Actions d78bec7001 ci: Remove Windows MSVC tasks from Cirrus CI 3545dc2b9b ci, gha: Run all MSVC tests on Windows natively 5d8fa825e2 Merge bitcoin-core/secp256k1#1274: test: Silent noisy clang warnings about Valgrind code on macOS x86_64 8e54a346d2 ci, gha: Run "x86_64: macOS Ventura" job on GitHub Actions b327abfcea Merge bitcoin-core/secp256k1#1402: ci: Use Homebrew's gcc in native macOS task d62db57427 ci: Use Homebrew's gcc in native macOS task 54058d16fe field: remove `secp256k1_fe_equal_var` bb4efd6404 tests: remove unwanted `secp256k1_fe_normalize_weak` call eedd781085 Merge bitcoin-core/secp256k1#1348: tighten group magnitude limits, save normalize_weak calls in group add methods (revival of #1032) b2f6712dd3 Merge bitcoin-core/secp256k1#1400: ctimetests: Use new SECP256K1_CHECKMEM macros also for ellswift 9c91ea41b1 ci: Enable ellswift module where it's missing db32a24761 ctimetests: Use new SECP256K1_CHECKMEM macros also for ellswift ce765a5b8e Merge bitcoin-core/secp256k1#1399: ci, gha: Run "SageMath prover" job on GitHub Actions 8408dfdc4c Revert "ci: Run sage prover on CI" c8d9914fb1 ci, gha: Run "SageMath prover" job on GitHub Actions 8d2960c8e2 Merge bitcoin-core/secp256k1#1397: ci: Remove "Windows (VS 2022)" task from Cirrus CI f1774e5ec4 ci, gha: Make MSVC job presentation more explicit 5ee039bb58 ci: Remove "Windows (VS 2022)" task from Cirrus CI 96294c00fb Merge bitcoin-core/secp256k1#1389: ci: Run "Windows (VS 2022)" job on GitHub Actions a2f7ccdecc ci: Run "Windows (VS 2022)" job on GitHub Actions 374e2b54e2 Merge bitcoin-core/secp256k1#1290: cmake: Set `ENVIRONMENT` property for examples on Windows 1b13415df9 Merge bitcoin-core/secp256k1#1391: refactor: take use of `secp256k1_scalar_{zero,one}` constants (part 2) a1bd4971d6 refactor: take use of `secp256k1_scalar_{zero,one}` constants (part 2) b7c685e74a Save _normalize_weak calls in group add methods c83afa66e0 Tighten group magnitude limits 26392da2fb Merge bitcoin-core/secp256k1#1386: ci: print $ELLSWIFT in cirrus.sh d23da6d557 use secp256k1_scalar_verify checks 4692478853 ci: print $ELLSWIFT in cirrus.sh c7d0454932 add verification for scalars c734c64278 Merge bitcoin-core/secp256k1#1384: build: enable ellswift module via SECP_CONFIG_DEFINES ad152151b0 update max scalar in scalar_cmov_test and fix schnorrsig_verify exhaustive test 78ca880788 build: enable ellswift module via SECP_CONFIG_DEFINES 0e00fc7d10 Merge bitcoin-core/secp256k1#1383: util: remove unused checked_realloc b097a466c1 util: remove unused checked_realloc 2bd5f3e618 Merge bitcoin-core/secp256k1#1382: refactor: Drop unused cast 4f8c5bd761 refactor: Drop unused cast 173e8d061a Implement current magnitude assumptions 49afd2f5d8 Take use of _fe_verify_magnitude in field_impl.h 4e9661fc42 Add _fe_verify_magnitude (no-op unless VERIFY is enabled) 690b0fc05a add missing group element invariant checks 175db31149 ci: Drop no longer needed `PATH` variable update on Windows 116d2ab3df cmake: Set `ENVIRONMENT` property for examples on Windows cef373997c cmake, refactor: Use helper function instead of interface library 747ada3587 test: Silent noisy clang warnings about Valgrind code on macOS x86_64 git-subtree-dir: src/secp256k1 git-subtree-split: 199d27cea32203b224b208627533c2e813cd3b21 scripted-diff: Use blocks_path where possible -BEGIN VERIFY SCRIPT- sed -i 's|].chain_path, .blocks.|].blocks_path|g' $(git grep -l chain_path) -END VERIFY SCRIPT- index: Drop legacy -txindex check move-only: Move CBlockTreeDB to node/blockstorage The block index (CBlockTreeDB) is required to write and read blocks, so move it to blockstorage. This allows to drop the txdb.h include from `node/blockstorage.h`. Can be reviewed with: --color-moved=dimmed-zebra --color-moved-ws=ignore-all-space Fixup style of moved code Can be reviewed with --word-diff-regex=. scripted-diff: Rename CBlockTreeDB -> BlockTreeDB -BEGIN VERIFY SCRIPT- sed -i 's|CBlockTreeDB|BlockTreeDB|g' $( git grep -l CBlockTreeDB ) -END VERIFY SCRIPT- build: use -muse-unaligned-vector-move for Windows We currently work around a longstanding GCC issue with aligned vector instructions, in our release builds, by patching the behaviour we want into GCC (see discussion in #24736). A new option now exists in the binutils assembler, `-muse-unaligned-vector-move`, which should also achieve the behaviour we want (at least for our code). This was added in the 2.38 release, see https://sourceware.org/git/?p=binutils-gdb.git;a=commit;h=c8480b58e1968f209b6365af7422678f348222c2. ```bash x86: Add -muse-unaligned-vector-move to assembler Unaligned load/store instructions on aligned memory or register are as fast as aligned load/store instructions on modern Intel processors. Add a command-line option, -muse-unaligned-vector-move, to x86 assembler to encode encode aligned vector load/store instructions as unaligned vector load/store instructions. ``` Even if we introduce this option into our build system, we'll have to maintain our GCC patching, as we want all code that ends up in the binary, to avoid these instructions. However, there may be some value in adding the option, as it could be an improvement for someone building (bitcoind.exe) with an unpatched compiler. test: Combine sync_send_with_ping and sync_with_ping miniscript: make GetStackSize independent of P2WSH context It was taking into account the P2WSH script push in the number of stack elements. miniscript: introduce a helper to get the maximum witness size Similarly to how we compute the maximum stack size. Also note how it would be quite expensive to recompute it recursively by accounting for different ECDSA signature sizes. So we just assume high-R everywhere. It's only a trivial difference anyways. descriptor: introduce a method to get the satisfaction size In the wallet code, we are currently estimating the size of a signed input by doing a dry run of the signing logic. This is unnecessary as all outputs we are able to sign for can be represented by a descriptor, and we can derive the size of a satisfaction ("signature") from the descriptor itself directly. In addition, this approach does not scale: getting the size of a satisfaction through a dry run of the signing logic is only possible for the most basic scripts. This commit introduces the computation of the size of satisfaction per descriptor. It's a bit intricate for 2 main reasons: - We want to conserve the behaviour of the current dry-run logic used by the wallet that sometimes assumes ECDSA signatures will be low-r, sometimes not (when we don't create them). - We need to account for the witness discount. A single descriptor may sometimes benefit of it, sometimes not (for instance `pk()` if used as top-level versus if used inside `wsh()`). script/signingprovider: introduce a MultiSigningProvider It is sometimes useful to interface with multiple signing providers at once. For instance when inferring a descriptor with solving information being provided from multiple sources (see next commit). Instead of inneficiently copying the information from one provider into the other, introduce a new signing provider that takes a list of pointers to existing providers. wallet: use descriptor satisfaction size to estimate inputs size Instead of using the dummysigner to compute a placeholder satisfaction, infer a descriptor on the scriptPubKey of the coin being spent and use the estimation of the satisfaction size given by the descriptor directly. Note this (almost, see next paragraph) exactly conserves the previous behaviour. For instance CalculateMaximumSignedInputSize was previously assuming the input to be spent in a transaction that spends at least one Segwit coin, since it was always accounting for the serialization of the number of witness elements. In this commit we use a placeholder for the size of the serialization of the witness stack size (1 byte). Since the logic in this commit is already tricky enough to review, and that it is only a very tiny approximation not observable through the existing tests, it is addressed in the next commit. wallet: accurately account for the size of the witness stack When estimating the maximum size of an input, we were assuming the number of elements on the witness stack could be encode in a single byte. This is a valid approximation for all the descriptors we support (including P2WSH Miniscript ones), but may not hold anymore once we support Miniscript within Taproot descriptors (since the max standard witness stack size of 100 gets lifted). It's a low-hanging fruit to account for it correctly, so just do it now. fuzz: introduce and use `ConsumePrivateKey` helper doc: s/--no-substitute/--no-substitutes in guix/INSTALL Replace READWRITEAS macro with AsBase wrapping function Co-authored-by: Pieter Wuille Rename CSerAction* to Action* This allows new code, added in the next commit, to conform to the coding guideline: No C-prefix for class names. Support for serialization parameters The moved part can be reviewed with the git options --ignore-all-space --color-moved=dimmed-zebra --color-moved-ws=ignore-all-space (Modified by Marco Falke) Co-authored-by: Pieter Wuille Use serialization parameters for CAddress serialization This also cleans up the addrman (de)serialization code paths to only allow `Disk` serialization. Some unit tests previously forced a `Network` serialization, which does not make sense, because Bitcoin Core in production will always `Disk` serialize. This cleanup idea was suggested by Pieter Wuille and implemented by Anthony Towns. Co-authored-by: Pieter Wuille Co-authored-by: Anthony Towns test: add tests that exercise WithParams() Co-authored-by: Vasil Dimov Remove unused legacy CHashVerifier depends: libtapi 1300.0.6.5 depends: cctools 986 & ld64 711 test: remove unused variables in `p2p_invalid_block` fuzz: add ConstructPubKeyBytes function Today, this code only has one spot where it needs well-formed pubkeys, but future PRs will want to reuse this code. Add a function which creates a well-formed byte array that can be turned into a pubkey. It is not required that the pubkey is valid, just that it can be recognized as a compressed or uncompressed pubkey. Note: while the main intent of this commit is to wrap the existing logic into a function, it also switches to `PickValueFromArray` so that we are only choosing one of 0x04, 0x06, or 0x07. The previous code, `ConsumeIntegralInRange` would have also picked 0x05, which is not definied in the context of compressed vs uncompressed keys. See https://bitcoin.stackexchange.com/questions/57855/c-secp256k1-what-do-prefixes-0x06-and-0x07-in-an-uncompressed-public-key-signif for more details. net: add have_next_message argument to Transport::GetBytesToSend() Before this commit, there are only two possibly outcomes for the "more" prediction in Transport::GetBytesToSend(): * true: the transport itself has more to send, so the answer is certainly yes. * false: the transport has nothing further to send, but if vSendMsg has more message(s) left, that still will result in more wire bytes after the next SetMessageToSend(). For the BIP324 v2 transport, there will arguably be a third state: * definitely not: the transport has nothing further to send, but even if vSendMsg has more messages left, they can't be sent (right now). This happens before the handshake is complete. To implement this, we move the entire decision logic to the Transport, by adding a boolean to GetBytesToSend(), called have_next_message, which informs the transport whether more messages are available. The return values are still true and false, but they mean "definitely yes" and "definitely no", rather than "yes" and "maybe". net: remove unused Transport::SetReceiveVersion crypto: Spanify EllSwiftPubKey constructor net: add V2Transport class with subset of BIP324 functionality This introduces a V2Transport with a basic subset of BIP324 functionality: * no ability to send garbage (but receiving is supported) * no ability to send decoy packets (but receiving them is supported) * no support for short message id encoding (neither encoding or decoding) * no waiting until 12 non-V1 bytes have been received * (and thus) no detection of V1 connections on the responder side (on the sender side, detecting V1 is not supported either, but that needs to be dealt with at a higher layer, by reconnecting) net: make V2Transport auto-detect incoming V1 and fall back to it net: add short message encoding/decoding support to V2Transport net: make V2Transport send uniformly random number garbage bytes net: make V2Transport preallocate receive buffer space test: add unit tests for V2Transport net: detect wrong-network V1 talking to V2Transport Remove version/hashing options from CBlockLocator/CDiskBlockIndex refactor: Use DataStream now that version/type are unused refactor: remove clientversion include from dbwrapper.h consensus/validation.h: remove needless GetTransactionOutputWeight helper Introduced in 9b7ec393b82ca9d7ada77d06e0835df0386a8b85. This copied the format of the other Get.*Weight helpers but it's useless for a CTxOut. test: refactor: remove unnecessary blocks_checked counter Since we already store all the blocks in `events`, keeping an additional counter is redundant. test: refactor: rename inbound to is_inbound Makes it easier to recognize this variable represents a flag. test: refactor: deduplicate handle_utxocache_* logic Carve out the comparison logic into a helper function to avoid code duplication. test: store utxocache events By storing the events instead of doing the comparison inside the handle_utxocache_* functions, we simplify the overall logic and potentially making debugging easier, by allowing pdb to access the events. Mostly a refactor, but changes logging behaviour slightly by not raising and not calling self.log.exception("Assertion failed") test: log sanity check assertion failures test: refactor: remove unnecessary nonlocal Since we're only mutating, and not reassigning, we don't need to declare `events` as `nonlocal`. test: refactor: usdt_mempool: store all events Even though we expect these functions to only produce one event, we still keep a counter to check if that's true. By simply storing all the events, we can remove the counters and make debugging easier, by allowing pdb to access the events. net: merge V2Transport constructors, move key gen This removes the ability for BIP324Cipher to generate its own key, moving that responsibility to the caller (mostly, V2Transport). This allows us to write the random-key V2Transport constructor by delegating to the explicit-key one. net: do not use send buffer to store/cache garbage Before this commit the V2Transport::m_send_buffer is used to store the garbage: * During MAYBE_V1 state, it's there despite not being sent. * During AWAITING_KEY state, while it is being sent. * At the end of the AWAITING_KEY state it cannot be wiped as it's still needed to compute the garbage authentication packet. Change this by introducing a separate m_send_garbage field, taking over the first and last role listed above. This means the garbage is only in the send buffer when it's actually being sent, removing a few special cases related to this. doc: fix typos and mistakes in BIP324 code comments index: coinstats reorg, fail when block cannot be reversed During a reorg, continuing execution when a block cannot be reversed leaves the coinstats index in an inconsistent state, which was surely overlooked when 'CustomRewind' was implemented. index: add [nodiscard] attribute to functions writing to the db rpc: Deprecate rpcserialversion=0 doc: Clarify that -fstack-reuse=all bugs exist on all versions of GCC wallet rpc: return final tx hex from walletprocesspsbt if complete test: remove unnecessary finalizepsbt rpc calls doc: add release note for PR #28414 doc, refactor: Changing -torcontrol help to specify that a default port is used Right now when we get the help for -torcontrol it says that there is a default ip and port we dont specify if there is a specified ip that we would also use port 9051 as default [policy] check for duplicate txids in package Duplicates of normal transactions would be found by looking for conflicting inputs, but this doesn't catch identical empty transactions. These wouldn't be valid but exiting early is good and AcceptPackage's result sanity checks assume non-duplicate transactions. [CCoinsViewMemPool] track non-base coins and allow Reset Temporary coins should not be available in separate subpackage submissions. Any mempool coins that are cached in m_view should be removed whenever mempool contents change, as they may be spent or no longer exist. [validation] add AcceptSubPackage to delegate Accept* calls and clean up m_view (1) Call AcceptSingleTransaction when there is only 1 transaction in the subpackage. This avoids calling PackageMempoolChecks() which enforces rules that don't need to be applied for a single transaction, i.e. disabling CPFP carve out. There is a slight change in the error type returned, as shown in the txpackage_tests change. When a transaction is the last one left in the package and its fee is too low, this returns a PCKG_TX instead of PCKG_POLICY. This interface is clearer; "package-fee-too-low" for 1 transaction would be a bit misleading. (2) Clean up m_view and m_viewmempool so that coins created in this sub-package evaluation are not available for other sub-package evaluations. The contents of the mempool may change, so coins that are available now might not be later. [validation] make PackageMempoolAcceptResult members mutable After the PackageMempoolAcceptResult is returned from AcceptMultipleTransactions, leave room for results to change due to LimitMempool() eviction. [refactor] back-fill results in AcceptPackage Instead of populating the last PackageMempoolAcceptResult with stuff from results_final and individual_results_nonfinal, fill results_final and create a PackageMempoolAcceptResult using that one. A future commit will add LimitMempoolSize() which may change the status of each of these transactions from "already in mempool" or "submitted to mempool" to "no longer in mempool". We will change those transactions' results here. A future commit also gets rid of the last AcceptSubPackage outside of the loop. It makes more sense to use results_final as the place where all results end up. [validation] return correct result when already-in-mempool tx gets evicted Bug fix: a transaction may be in the mempool when package evaluation begins (so it is added to results_final with MEMPOOL_ENTRY or DIFFERENT_WITNESS), but get evicted due to another transaction submission. [validation] don't LimitMempoolSize in any subpackage submissions Don't do any mempool evictions until package validation is done, preventing the mempool minimum feerate from changing. Whether we submit transactions separately or as a package depends on whether they meet the mempool minimum feerate threshold, so it's best that the value not change while we are evaluating a package. This avoids a situation where we have a CPFP package in which the parents meet the mempool minimum feerate and are submitted by themselves, but they are evicted before we have submitted the child. [test framework] add ability to spend only confirmed utxos Useful to ensure that the topologies of packages/transactions are as expected, preventing bugs caused by having unexpected mempool ancestors. [refactor] split setup in mempool_limit test We want to be able to re-use fill_mempool so that none of the tests affect each other. Change the logs from info to debug because they are otherwise repeated many times in the test output. [test] mempool coins disappearing mid-package evaluation Test for scenario(s) outlined in PR 28251. Test what happens when a package transaction spends a mempool coin which is fetched and then disappears mid-package evaluation due to eviction or replacement. dbwrapper: Use DataStream for batch operations Remove unused GetType() from CBufferedFile and CAutoFile GetType() is only called in tests, so it is unused and can be removed. scripted-diff: Rename CBufferedFile to BufferedFile While touching all constructors in the previous commit, the class name can be adjusted to comply with the style guide. -BEGIN VERIFY SCRIPT- sed -i 's/CBufferedFile/BufferedFile/g' $( git grep -l CBufferedFile ) -END VERIFY SCRIPT- ci: Add test-each-commit task ci: Limit test-each-commit to --max-count=6 [refactor] Allow std::array in serialize.h This is already possible for C-style arrays, so allow it for C++11 std::array as well. [refactor] Define MessageStartChars as std::array kernel: Move MessageStartChars to its own file The protocol.h file contains many non-consensus related definitions and should thus not be part of the libbitcoinkernel. This commit makes protocol.h no longer a required include for users of the libbitcoinkernel. This commit is part of the libbitcoinkernel project, namely its stage 1 step 3: Decouple most non-consensus headers from libbitcoinkernel. Co-Authored-By: Cory Fields [refactor] Add missing includes for next commit [refactor] Add CChainParams member to CConnman This is done in preparation to the next commit, but has the nice effect of removing one further data structure relying on the global `Params()`. [refactor] Remove netaddress.h from kernel headers Move functions requiring the netaddress.h include out of libbitcoinkernel source files. The netaddress.h file contains many non-consensus related definitions and should thus not be part of the libbitcoinkernel. This commit makes netaddress.h no longer a required include for users of the libbitcoinkernel. This commit is part of the libbitcoinkernel project, namely its stage 1 step 3: Decouple most non-consensus headers from libbitcoinkernel. [refactor] Remove compat.h from kernel headers This commit makes compat.h no longer a required include for users of the libbitcoinkernel. Including compat.h imports a bunch of platform-specific definitions. This commit is part of the libbitcoinkernel project, namely its stage 1 step 3: Decouple most non-consensus headers from libbitcoinkernel. ci: clang-17 for fuzz and tsan [fuzz] Use afl++ shared-memory fuzzing Using shared-memory is faster than reading from stdin, see https://github.com/AFLplusplus/AFLplusplus/blob/7d2122e0596132f9344a5d0896020ebc79cd33db/instrumentation/README.persistent_mode.md Revert "Merge bitcoin/bitcoin#28279: ci: Add test-each-commit task" This reverts commit 744e0e36703e26d06bc5cd1ef36a1c8568e71d05, reversing changes made to 8209e86eeb4ceb6dd0e06c45fb3c799bb42834ab. unit test: add coverage for BlockManager ci: enable chattr +i capability inside containers test: cover read-only blockstore Co-authored-by: Andrew Chow Match tx names to index in miniminer overlap test Follow-up from #27021: In the prior commit, the vector started counting at 0, but the transaction names started with 1. This commit matches the names to the transactions’ vector indices for better readability. Co-authored-by: theStack Fix calculation of ancestor set feerates in test Follow-up from #27021. Also included is an ASCII art visualization of the test’s transaction topology by theStack. Co-authored-by: theStack Remove unused imports Follow-up from #27021 Make MiniMinerMempoolEntry fields private Follow-up from #27021: accessing of fields in MiniMinerMempoolEntry was done inconsistently. Even though we had a getter, we would directly write to the fields when we needed to update them. This commits sets the fields to private and introduces a method for updating the ancestor information in transactions using the same method name as used for Mempool Entries. [node] interface to get bump fees coinselection: Move GetSelectionWaste into SelectionResult GetSelectionWaste will need to access more context within a selection result, and so should be a private member function rather than a static function. It's only use outside of SelectionResult was for tests which have now been updated to just make a SelectionResult. Co-authored-by: Murch Bump unconfirmed parent txs to target feerate When a transaction uses an unconfirmed input, preceding this commit it would not consider the feerate of the parent transaction. Given a parent transaction with a lower ancestor feerate, this resulted in the new transaction's ancestor feerate undershooting the target feerate. This commit changes how we calculate the effective value of unconfirmed UTXOs. The effective value of unconfirmed UTXOs is decreased by the fee necessary to bump its ancestry to the target feerate. This also impacts the calculation of the waste metric: since the estimate for the current fee is increased by the bump fees, unconfirmed UTXOs current fees appear less favorable compared to their unchanged long term fees. This has one caveat: if multiple UTXOs have overlapping ancestries, each of their individual estimates will account for bumping all ancestors. Amend bumpfee for inputs with overlapping ancestry At the end of coin selection reduce the fees by the difference between the individual bump fee estimates and the collective bump fee estimate. Do not use std::vector = {} to release memory [fuzz] Don't use afl++ deferred forkserver mode Deferring the forkserver initialization doesn't make sense for some of our targets since they involve state that can't be forked (e.g. threads). We therefore remove the use of __AFL_INIT entirely. We also increase the __AFL_LOOP count to 100000. Our fuzz targets are meant to all be deterministic and stateless therefore this should be fine. ci: use LLVM 17.0.0 in MSAN jobs See https://libcxx.llvm.org/Hardening.html as well as https://discourse.llvm.org/t/rfc-removing-the-legacy-debug-mode-from-libc/71026. serialize: move ser_action functions out of global namespace serialize: specify type for ParamsWrapper not ref serialize: add SER_PARAMS_OPFUNC scripted-diff: use SER_PARAMS_OPFUNC -BEGIN VERIFY SCRIPT- sed -i 's/WithParams(\(CAddress::V[12]_[A-Z]*\) *, */\1(/g' $(git grep -l 'WithParams' src/) sed -i 's/WithParams(\(CNetAddr::V[12]\) *, */\1(/g' $(git grep -l 'WithParams' src/) sed -i 's@\(CNetAddr::V1.CService{}.*\) //@\1 //@' src/test/util/net.cpp -END VERIFY SCRIPT- tests: fix incorrect assumption in v2transport_test build: use _LIBCPP_ENABLE_DEBUG_MODE over ENABLE_ASSERTIONS `_LIBCPP_ENABLE_ASSERTIONS` is deprecated, and will be removed. [See (from libc++ __config in main)](https://github.com/llvm/llvm-project/blob/b57df9fe9a1a230f277d671bfa0884bbda9fc1c5/libcxx/include/__config#L205-L209): > TODO(hardening): remove this in LLVM 19. > This is for backward compatibility -- make enabling `_LIBCPP_ENABLE_ASSERTIONS` (which predates hardening modes) > equivalent to setting the safe mode. > ifdef _LIBCPP_ENABLE_ASSERTIONS > warning "_LIBCPP_ENABLE_ASSERTIONS is deprecated, please use _LIBCPP_ENABLE_SAFE_MODE instead." From LLVM 17, `_LIBCPP_ENABLE_DEBUG_MODE` can be used instead. See https://libcxx.llvm.org/Hardening.html. Related to #28476. fuzz: Add missing PROVIDE_FUZZ_MAIN_FUNCTION guard to __AFL_FUZZ_INIT ci: Reintroduce fixed "test-each-commit" job The new job checks at most 6 commits of a pull request, excluding the top one. Make WitnessUnknown members private Make sure that nothing else can change WitnessUnknown's data members by making them private. Also change the program to use a vector rather than C-style array. Allow CNoDestination to represent a raw script Even if a script is not a standard destination type, it can still be useful to have a CTxDestination that stores the script. Add PubKeyDestination for P2PK scripts P2PK scripts are not PKHash destinations, they should have their own type. This also results in no longer showing a p2pkh address for p2pk outputs. However for backwards compatibility, ListCoinst will still do this conversion. wallet: Use CTxDestination in CRecipient rather than scriptPubKey wallet: disallow migration of invalid or not-watched scripts The legacy wallet allowed to import any raw script, without checking if it was valid or not. Appending it to the watch-only set. This causes a crash in the migration process because we are only expecting to find valid scripts inside the legacy spkm. These stored scripts internally map to `ISMINE_NO` (same as if they weren't stored at all..). So we need to check for these special case, and take into account that the legacy spkm could be storing invalid not watched scripts. Which, in code words, means IsMineInner() returning IsMineResult::INVALID for them. test: wallet, verify migration doesn't crash for an invalid script The migration process must skip any invalid script inside the legacy spkm and all the addressbook records linked to them. These scripts are not being watched by the current wallet, nor should be watched by the migrated one. IsMine() returns ISMINE_NO for them. fuzz: Drop unused params from serialize helpers With the ser-type and ser-version going away, it seems unlikely that there is need for them in the future, so just remove them. fuzz: Rework addr fuzzing * Replace ConsumeDeserializationParams with V1, because V2 is unconditionally checked as well. * Also fuzz CAddress::Format::Disk in the address_deserialize fuzz target. build: add -zip option to macdeployqtplus This zips the app bundle in /dist. build: produce a .zip for macOS distribution Instead of a .dmg. Co-authored-by: fanquake macdeploy: remove DMG generation from deploy script build: remove dmg dependencies Remove MemPoolAccept::m_limits, only have local copies for carveouts Add regression test for m_limit mutation rpc: Introduce getaddrmaninfo for count of addresses stored in new/tried table test: add functional test for getaddrmaninfo ci: Use nproc over MAKEJOBS in 01_base_install test: refactor: deduplicate segwitv0 ECDSA signing for tx inputs Follow-up for #28025. interfaces, wallet: Expose migrate wallet gui: Optionally return passphrase after unlocking AskPassphraseDialog has an optional parameter for the caller to get the passphrase. Make this available for Unlocking. gui: Add File > Migrate Wallet rpc, refactor: clean-up `addnode` 1. Use const where possible; 2. Rename variables to make them clearer; 3. There is no need to check whether `command` is null since it's a non-optional field. test: `addnode` with an invalid command should throw an error doc: make it clear that `node` in `addnode` refers to the node's address Move IsPeerAddrLocalGood() declaration from header to implementation Move CaptureMessageToFile() declaration from header to implementation Move GetLocal() declaration from header to implementation Add and use CNetAddr::HasCJDNSPrefix() helper GetLocal() type-safety, naming, const, and formatting cleanups Co-authored-by: Jon Atack Add CNetAddr::IsPrivacyNet() and CNode::IsConnectedThroughPrivacyNet() Co-authored-by: Vasil Dimov Use higher-level CNetAddr and CNode helpers in net.cpp rather than low-level comparisons with Network enum values. Convert GetLocal() to std::optional and remove out-param Co-authored-by: stickies-v Inline short, often-called, rarely-changed basic CNetAddr getters and make them nodiscard. Member functions containing a few lines of code are usually inlined, either implicitly by defining them in the declaration as done here, or declared inline. References https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#Rf-inline https://google.github.io/styleguide/cppguide#Inline_Functions https://www.ibm.com/docs/en/i/7.1?topic=only-inline-member-functions-c test: added coverage to estimatefee Added a assert for an rpc error when we try to estimate fee for the max conf_target Replace MAX_PACKAGE_SIZE with MAX_PACKAGE_WEIGHT to avoid vbyte confusion While allowing submitted packages to be slightly larger than what may be allowed in the mempool to allow simpler reasoning about contextual-less checks vs chain limits. Bugfix: Pass correct virtual size to CheckPackageLimits Handle over-sized (in virtual bytes) packages with no in-mempool ancestors Add functional test to catch too large vsize packages Refactor: Remove m_is_test_chain Refactor: Replace 'isMockableChain' with inline 'ChainType' check for 'submitpackage' ci: Install Homebrew's `pkg-config` package Some versions of macOS images lack the 'pkg-config' package, which is required for the build process. qt: Silence `-Wcast-function-type` warning Required for https://github.com/bitcoin/bitcoin/pull/25972. Picked from https://trac.nginx.org/nginx/ticket/1865. gui: Disable and uncheck blank when private keys are disabled Unify the GUI's create wallet with the RPC createwallet so that the blank flag is not set when private keys are disabled. GUI: TransactionRecord: Use "any from me" as the criteria for deciding whether a transaction is a send or receive This changes behaviour (IMO for the better) in the case where some but not all inputs are from us, and the net amount is positive. GUI: TransactionRecord: Refactor to turn send-to-self into send+receive pairs GUI: Remove SendToSelf TransactionRecord type Bugfix: Ignore ischange flag when we're not the sender If we didn't send it, it can't possibly be change, even if that's the key's purpose GUI: TransactionRecord: When time/index/etc match, sort send before receive rpc: `descriptorprocesspsbt` return hex encoded tx If processed psbt is complete return hex encoded network transaction in the output. test: check `descriptorprocesspsbt` return hex encoded tx Test that if the processed psbt is complete the hex encoded tx is returned and remove unneccessary rpc call to finalize the psbt. doc: add release note [refactor] batch-add transactions to DisconnectedBlockTransactions No behavior change. In a future commit, we can optimize by reserving vtx.size(). [bench] DisconnectedBlockTransactions add std::list to memusage rewrite DisconnectedBlockTransactions as a list + map And encapsulate underlying data structures to avoid misuse. It's better to use stdlib instead of boost when we can achieve the same thing. Behavior change: the number returned by DynamicMemoryUsage for the same transactions is higher (due to change in usage or more accurate accounting), which effectively decreases the maximum amount of transactions kept for resubmission in a reorg. Co-authored-by: Cory Fields MOVEONLY: DisconnectedBlockTransactions to its own file This struct is only used in validation + tests and has very little to do with txmempool. make DisconnectedBlockTransactions responsible for its own memory management Co-authored-by: Cory Fields doc: Be vague instead of wrong about MALLOC_ARENA_MAX Before this commit, we claim that glibc's malloc implementation uses 2 arenas by default. But that's true only on 32-bit systems, and even there, it uses *up* to 2 arenas. This commit fixes the wrong statement. The new statement is intentionally vague to reduce our maintenance burden. For details, see: https://www.gnu.org/software/libc/manual/html_node/Memory-Allocation-Tunables.html#index-glibc_002emalloc_002earena_005fmax Noticed in: https://github.com/bitcoin/bitcoin/pull/27642#issuecomment-1728103427 refactor: Drop unused fclose() from BufferedFile This was only explicitly used in the tests, where it can be replaced by wrapping the original raw file pointer into a CAutoFile on creation and then calling CAutoFile::fclose(). Also, it was used in LoadExternalBlockFile(), where it can also be replaced by the (implicit call to the) CAutoFile destructor after wrapping the original raw file pointer in a CAutoFile. Make BufferedFile to be a CAutoFile wrapper This refactor allows to forward some calls to the underlying CAutoFile, instead of re-implementing the logic in the buffered file. Return CAutoFile from BlockManager::Open*File() This is a refactor. rpc: bumpfee, improve doc for 'reduce_output' arg The current argument name and description are dangerous as it don't describe the case where the user selects the recipient output as the change address. This one could end up been increased by the inputs minus outputs remainder. Which, when bumpfee adds new inputs to the transaction, leads the process to send more coins to the recipient. Which is not what the user would expect from a 'reduce_output' param naming. Co-authored-by: Murch Add package evaluation fuzzer tests: Use manual bumps instead of bumpfee for resendwallettransactions Bumpfee will try to increase the entire package to the target feerate, which causes repeated bumpfees to quickly shoot up in fees, causing intermittent failures when the fee is too large. We don't care about this property, just that the child is continuously replaced until we observe it's position in mapWallet is before its parent. Instead of using bumpfee, we can create raw transactions which have only pay the additional incremental relay fee, thus avoiding this problem. net: Drop v2 garbage authentication packet See also https://github.com/bitcoin/bips/pull/1498 The benefit is a simpler implementation: - The protocol state machine does not need separate states for garbage authentication and version phases. - The special case of "ignoring the ignore bit" is removed. - The freedom to choose the contents of the garbage authentication packet is removed. This simplifies testing. net: Simplify v2 recv logic by decoupling AAD from state machine blockstorage: Mark FindBlockPos as nodiscard A false return value indicates a fatal error (disk space being too low), so make sure we always consume this error code. This commit is part of an ongoing process for making the handling of fatal errors more transparent and easier to understand. blockstorage: Return on fatal block file flush error By returning an error code if `FlushBlockFile` fails, the caller now has to explicitly handle block file flushing errors. Before this change such errors were non-explicitly ignored without a clear rationale. Prior to this patch `FlushBlockFile` may have failed silently in `Chainstate::FlushStateToDisk`. Improve this with a log line. Also add a TODO comment to flesh out whether returning early in the case of an error is appropriate or not. Returning early might be appropriate to prohibit `WriteBlockIndexDB` from writing a block index entry that does not refer to a fully flushed block. Besides `Chainstate::FlushStateToDisk`, `FlushBlockFile` is also called by `FindBlockPos`. Don't change the abort behavior there, since we don't want to fail the function if the flushing of already written blocks fails. Instead, just document it. blockstorage: Return on fatal undo file flush error By returning an error code if either `FlushUndoFile` or `FlushBlockFile` fail, the caller now has to explicitly handle block undo file flushing errors. Before this change such errors were non-explicitly ignored without a clear rationale. Besides the call to `FlushUndoFile` in `FlushBlockFile`, ignore its return code at its call site in `WriteUndoDataForBlock`. There, a failed flush of the undo data should not be indicative of a failed write. Add [[nodiscard]] annotations to `FlushUndoFile` such that its return value is not just ignored in the future. test: add unit test coverage for Python ECDSA implementation --- .cirrus.yml | 315 +- .github/ISSUE_TEMPLATE/good_first_issue.yml | 6 +- .../install-homebrew-valgrind/action.yml | 33 + .../actions/run-in-docker-action/action.yml | 49 + .github/workflows/ci.yml | 832 +++ .gitignore | 2 +- .tx/config | 2 +- Makefile.am | 17 +- ci/README.md | 10 +- ci/ci.sh | 132 + ci/test/00_setup_env.sh | 22 +- ci/test/00_setup_env_android.sh | 4 +- ci/test/00_setup_env_arm.sh | 10 +- ci/test/00_setup_env_i686_centos.sh | 4 +- ci/test/00_setup_env_i686_multiprocess.sh | 2 +- ci/test/00_setup_env_mac.sh | 8 +- ...ve_arm64.sh => 00_setup_env_mac_native.sh} | 6 +- ci/test/00_setup_env_native_asan.sh | 7 +- ci/test/00_setup_env_native_fuzz.sh | 8 +- ci/test/00_setup_env_native_fuzz_with_msan.sh | 6 +- .../00_setup_env_native_fuzz_with_valgrind.sh | 4 +- ci/test/00_setup_env_native_msan.sh | 6 +- ...up_env_native_nowallet_libbitcoinkernel.sh | 4 +- ci/test/00_setup_env_native_qt5.sh | 4 +- ci/test/00_setup_env_native_tidy.sh | 4 +- ci/test/00_setup_env_native_tsan.sh | 8 +- ci/test/00_setup_env_native_valgrind.sh | 4 +- ci/test/00_setup_env_s390x.sh | 14 +- ci/test/00_setup_env_win64.sh | 4 +- ci/test/01_base_install.sh | 67 +- ci/test/04_install.sh | 37 +- ci/test/06_script_b.sh | 52 +- ci/test/wrap-qemu.sh | 18 - ci/test/wrapped-cl.bat | 1 - ci/test_imagefile | 4 +- configure.ac | 119 +- contrib/devtools/bitcoin-tidy/CMakeLists.txt | 56 + contrib/devtools/bitcoin-tidy/README | 11 + .../devtools/bitcoin-tidy/bitcoin-tidy.cpp | 22 + .../bitcoin-tidy/example_logprintf.cpp | 108 + contrib/devtools/bitcoin-tidy/logprintf.cpp | 60 + contrib/devtools/bitcoin-tidy/logprintf.h | 29 + contrib/guix/INSTALL.md | 4 +- contrib/guix/libexec/build.sh | 5 +- contrib/guix/libexec/codesign.sh | 7 +- contrib/guix/libexec/prelude.bash | 2 +- contrib/guix/manifest.scm | 193 +- contrib/guix/patches/gcc-broken-longjmp.patch | 70 - ...store.patch => gcc-remap-guix-store.patch} | 0 .../patches/glibc-2.27-powerpc-ldbrx.patch | 245 + .../nsis-disable-installer-reloc.patch | 30 - .../guix/patches/nsis-gcc-10-memmove.patch | 23 - contrib/macdeploy/README.md | 19 +- contrib/macdeploy/background.tiff | Bin 18464 -> 0 bytes contrib/macdeploy/macdeployqtplus | 105 +- depends/README.md | 2 +- depends/hosts/linux.mk | 2 +- depends/packages/native_cctools.mk | 13 +- depends/packages/native_clang.mk | 8 +- depends/packages/native_ds_store.mk | 15 - depends/packages/native_libtapi.mk | 9 +- depends/packages/native_mac_alias.mk | 15 - depends/packages/packages.mk | 2 +- .../native_cctools/no_fixup_chains.patch | 23 - .../patches/native_libtapi/disable_zlib.patch | 17 + doc/build-osx.md | 12 +- doc/build-unix.md | 8 +- doc/policy/packages.md | 11 +- doc/reduce-memory.md | 2 +- doc/release-notes-27213.md | 8 + doc/release-notes-28217.md | 4 + doc/release-notes-28354.md | 6 + doc/release-notes-28414.md | 6 + doc/release-notes-28448.md | 6 + doc/release-process.md | 2 +- src/.clang-tidy | 1 + src/Makefile.am | 23 +- src/Makefile.bench.include | 2 +- src/Makefile.qt_locale.include | 19 + src/Makefile.test.include | 4 +- src/addrdb.cpp | 29 +- src/addrdb.h | 7 +- src/addresstype.cpp | 169 + src/addresstype.h | 152 + src/addrman.cpp | 35 +- src/addrman_impl.h | 3 +- src/bench/block_assemble.cpp | 1 + src/bench/chacha20.cpp | 43 +- src/bench/chacha_poly_aead.cpp | 126 - src/bench/coin_selection.cpp | 2 +- src/bench/descriptors.cpp | 1 - src/bench/disconnected_transactions.cpp | 130 + src/bench/duplicate_inputs.cpp | 1 + src/bench/load_external.cpp | 3 +- src/bench/mempool_eviction.cpp | 3 +- src/bench/mempool_stress.cpp | 4 +- src/bench/rpc_mempool.cpp | 2 +- src/bench/streams_findbyte.cpp | 16 +- src/bench/verify_script.cpp | 2 +- src/bip324.cpp | 109 + src/bip324.h | 96 + src/bitcoin-chainstate.cpp | 3 +- src/bitcoin-util.cpp | 1 + src/blockencodings.cpp | 1 + src/blockfilter.cpp | 2 + src/blockfilter.h | 13 +- src/chain.h | 12 +- src/common/bloom.cpp | 2 +- src/compat/compat.h | 25 +- src/compressor.cpp | 2 +- src/core_write.cpp | 2 +- src/crypto/chacha20.cpp | 157 +- src/crypto/chacha20.h | 118 +- src/crypto/chacha20poly1305.cpp | 140 + src/crypto/chacha20poly1305.h | 148 + src/crypto/chacha_poly_aead.cpp | 132 - src/crypto/chacha_poly_aead.h | 146 - src/crypto/muhash.cpp | 3 +- src/dbwrapper.cpp | 234 +- src/dbwrapper.h | 164 +- src/hash.h | 39 +- src/headerssync.cpp | 5 +- src/index/base.cpp | 4 +- src/index/blockfilterindex.cpp | 5 +- src/index/blockfilterindex.h | 2 + src/index/coinstatsindex.cpp | 6 +- src/index/coinstatsindex.h | 2 +- src/index/disktxpos.h | 3 +- src/index/txindex.cpp | 5 +- src/init.cpp | 28 +- src/interfaces/chain.h | 37 + src/interfaces/wallet.h | 23 +- src/kernel/chainparams.cpp | 11 +- src/kernel/chainparams.h | 22 +- src/kernel/disconnected_transactions.h | 137 + src/kernel/mempool_entry.h | 5 +- src/kernel/mempool_options.h | 3 +- src/kernel/mempool_persist.cpp | 4 +- src/kernel/mempool_removal_reason.cpp | 21 + src/kernel/mempool_removal_reason.h | 24 + src/kernel/messagestartchars.h | 13 + src/key_io.cpp | 19 +- src/key_io.h | 2 +- src/logging.cpp | 3 + src/logging.h | 1 + src/memusage.h | 16 + src/net.cpp | 1002 +++- src/net.h | 461 +- src/net_processing.cpp | 218 +- src/net_processing.h | 16 +- src/netaddress.cpp | 23 +- src/netaddress.h | 44 +- src/netgroup.cpp | 2 +- src/node/blockstorage.cpp | 201 +- src/node/blockstorage.h | 57 +- src/node/chainstate.cpp | 4 +- src/node/interfaces.cpp | 31 +- src/node/mempool_args.cpp | 3 +- src/node/miner.cpp | 1 + src/node/miner.h | 3 +- src/node/mini_miner.cpp | 7 +- src/node/mini_miner.h | 9 +- src/node/peerman_args.cpp | 7 +- src/node/utxo_snapshot.cpp | 4 +- src/outputtype.cpp | 1 - src/outputtype.h | 2 +- src/policy/fees.cpp | 2 - src/policy/fees.h | 5 +- src/policy/packages.cpp | 15 +- src/policy/packages.h | 20 +- src/policy/policy.cpp | 2 +- src/policy/policy.h | 40 +- src/primitives/block.h | 17 +- src/protocol.cpp | 2 +- src/protocol.h | 47 +- src/psbt.cpp | 1 + src/pubkey.cpp | 6 + src/pubkey.h | 6 +- src/qt/askpassphrasedialog.cpp | 3 + src/qt/bitcoin.cpp | 12 +- src/qt/bitcoin_locale.qrc | 19 + src/qt/bitcoingui.cpp | 16 +- src/qt/bitcoingui.h | 4 +- src/qt/bitcoinstrings.cpp | 35 +- src/qt/createwalletdialog.cpp | 17 +- src/qt/guiutil.cpp | 2 +- src/qt/locale/bitcoin_am.ts | 118 +- src/qt/locale/bitcoin_ar.ts | 4420 ++++++++------- src/qt/locale/bitcoin_az.ts | 88 +- src/qt/locale/bitcoin_az@latin.ts | 1659 ++++++ src/qt/locale/bitcoin_be.ts | 102 +- src/qt/locale/bitcoin_bg.ts | 287 +- src/qt/locale/bitcoin_bn.ts | 143 +- src/qt/locale/bitcoin_br.ts | 179 + src/qt/locale/bitcoin_bs.ts | 382 +- src/qt/locale/bitcoin_ca.ts | 4105 +++++++------- src/qt/locale/bitcoin_cmn.ts | 3697 +++++++++++++ src/qt/locale/bitcoin_cs.ts | 4796 ++++++++-------- src/qt/locale/bitcoin_da.ts | 4339 ++++++++------- src/qt/locale/bitcoin_de.ts | 4178 +------------- src/qt/locale/bitcoin_de_AT.ts | 4715 ++++++++++++++++ src/qt/locale/bitcoin_de_CH.ts | 4718 ++++++++++++++++ src/qt/locale/bitcoin_el.ts | 4162 +++++++------- src/qt/locale/bitcoin_en.ts | 219 +- src/qt/locale/bitcoin_en.xlf | 3324 ++++++----- src/qt/locale/bitcoin_eo.ts | 154 +- src/qt/locale/bitcoin_es.ts | 4790 ++++++++-------- src/qt/locale/bitcoin_es_CL.ts | 2529 ++++++++- src/qt/locale/bitcoin_es_CO.ts | 1952 ++++--- src/qt/locale/bitcoin_es_DO.ts | 3748 +++++++++++-- src/qt/locale/bitcoin_es_MX.ts | 2607 +++------ src/qt/locale/bitcoin_es_SV.ts | 4151 ++++++++++++++ src/qt/locale/bitcoin_es_VE.ts | 3706 +++++++++++-- src/qt/locale/bitcoin_et.ts | 130 +- src/qt/locale/bitcoin_eu.ts | 155 +- src/qt/locale/bitcoin_fa.ts | 4052 +++++++------- src/qt/locale/bitcoin_fi.ts | 4060 +++++++------- src/qt/locale/bitcoin_fil.ts | 3059 +++++------ src/qt/locale/bitcoin_fr.ts | 4728 ++++++++-------- src/qt/locale/bitcoin_fr_CM.ts | 4830 ++++++++++++++++ src/qt/locale/bitcoin_fr_LU.ts | 4830 ++++++++++++++++ src/qt/locale/bitcoin_ga.ts | 3431 ++++++------ src/qt/locale/bitcoin_ga_IE.ts | 3552 ++++++++++++ src/qt/locale/bitcoin_gl.ts | 491 +- src/qt/locale/bitcoin_gl_ES.ts | 10 +- src/qt/locale/bitcoin_gu.ts | 115 + src/qt/locale/bitcoin_ha.ts | 129 +- src/qt/locale/bitcoin_hak.ts | 3737 +++++++++++++ src/qt/locale/bitcoin_he.ts | 3501 ++++++------ src/qt/locale/bitcoin_hi.ts | 2408 ++++++++ src/qt/locale/bitcoin_hr.ts | 4415 ++++++++------- src/qt/locale/bitcoin_hu.ts | 4808 ++++++++-------- src/qt/locale/bitcoin_id.ts | 4496 +++------------ src/qt/locale/bitcoin_is.ts | 14 +- src/qt/locale/bitcoin_it.ts | 4738 ++++++++-------- src/qt/locale/bitcoin_ja.ts | 4812 ++++++++-------- src/qt/locale/bitcoin_ka.ts | 900 ++- src/qt/locale/bitcoin_kk.ts | 52 +- src/qt/locale/bitcoin_km.ts | 1021 +++- src/qt/locale/bitcoin_kn.ts | 245 + src/qt/locale/bitcoin_ko.ts | 4309 ++++++++------- src/qt/locale/bitcoin_ku.ts | 528 +- src/qt/locale/bitcoin_ku_IQ.ts | 130 +- src/qt/locale/bitcoin_la.ts | 126 +- src/qt/locale/bitcoin_lt.ts | 326 +- src/qt/locale/bitcoin_lv.ts | 70 +- src/qt/locale/bitcoin_mg.ts | 444 ++ src/qt/locale/bitcoin_ml.ts | 189 +- src/qt/locale/bitcoin_mn.ts | 24 +- src/qt/locale/bitcoin_mr.ts | 435 ++ src/qt/locale/bitcoin_mr_IN.ts | 74 +- src/qt/locale/bitcoin_ms.ts | 18 +- src/qt/locale/bitcoin_nb.ts | 4032 +++++++------- src/qt/locale/bitcoin_ne.ts | 481 +- src/qt/locale/bitcoin_nl.ts | 4631 ++++++++-------- src/qt/locale/bitcoin_pa.ts | 29 +- src/qt/locale/bitcoin_pam.ts | 78 +- src/qt/locale/bitcoin_pl.ts | 4461 +++++++-------- src/qt/locale/bitcoin_pt.ts | 4598 ++++++++-------- src/qt/locale/bitcoin_pt@qtfiletype.ts | 250 + src/qt/locale/bitcoin_pt_BR.ts | 4423 ++++++++------- src/qt/locale/bitcoin_ro.ts | 3137 +++++------ src/qt/locale/bitcoin_ru.ts | 4832 ++++++++-------- src/qt/locale/bitcoin_si.ts | 113 +- src/qt/locale/bitcoin_sk.ts | 4435 ++++++++------- src/qt/locale/bitcoin_sl.ts | 4647 ++++++++-------- src/qt/locale/bitcoin_so.ts | 443 ++ src/qt/locale/bitcoin_sq.ts | 18 +- src/qt/locale/bitcoin_sr.ts | 3984 +++++++------- src/qt/locale/bitcoin_sr@ijekavianlatin.ts | 4073 ++++++++++++++ src/qt/locale/bitcoin_sr@latin.ts | 3679 ++++++++++++- src/qt/locale/bitcoin_sv.ts | 3749 +++++++------ src/qt/locale/bitcoin_sw.ts | 427 +- src/qt/locale/bitcoin_szl.ts | 126 +- src/qt/locale/bitcoin_ta.ts | 3014 +++++----- src/qt/locale/bitcoin_te.ts | 424 +- src/qt/locale/bitcoin_th.ts | 2938 ++-------- src/qt/locale/bitcoin_tk.ts | 46 +- src/qt/locale/bitcoin_tl.ts | 60 +- src/qt/locale/bitcoin_tr.ts | 3754 ++++++------- src/qt/locale/bitcoin_ug.ts | 28 + src/qt/locale/bitcoin_uk.ts | 4876 +++++++++-------- src/qt/locale/bitcoin_ur.ts | 34 +- src/qt/locale/bitcoin_uz.ts | 2499 ++++++++- src/qt/locale/bitcoin_uz@Cyrl.ts | 854 ++- src/qt/locale/bitcoin_uz@Latn.ts | 975 +++- src/qt/locale/bitcoin_vi.ts | 553 +- src/qt/locale/bitcoin_yue.ts | 3628 ++++++++++++ src/qt/locale/bitcoin_zh-Hans.ts | 4814 ++++++++-------- src/qt/locale/bitcoin_zh-Hant.ts | 3737 +++++++++++++ src/qt/locale/bitcoin_zh.ts | 941 +++- src/qt/locale/bitcoin_zh_CN.ts | 4731 +++------------- src/qt/locale/bitcoin_zh_HK.ts | 3380 +++++++++++- src/qt/locale/bitcoin_zh_TW.ts | 4455 ++++++++------- src/qt/locale/bitcoin_zu.ts | 62 +- src/qt/test/apptests.cpp | 1 + src/qt/test/wallettests.cpp | 1 + src/qt/transactiondesc.cpp | 1 + src/qt/transactionrecord.cpp | 160 +- src/qt/transactionrecord.h | 1 - src/qt/transactiontablemodel.cpp | 9 +- src/qt/transactionview.cpp | 1 - src/qt/walletcontroller.cpp | 73 +- src/qt/walletcontroller.h | 26 +- src/qt/walletmodel.h | 1 - src/qt/winshutdownmonitor.cpp | 2 +- src/random.cpp | 31 +- src/random.h | 6 +- src/randomenv.cpp | 1 + src/rpc/blockchain.cpp | 4 +- src/rpc/client.cpp | 5 +- src/rpc/mempool.cpp | 10 +- src/rpc/mining.cpp | 17 +- src/rpc/net.cpp | 121 +- src/rpc/node.cpp | 1 + src/rpc/output_script.cpp | 6 +- src/rpc/rawtransaction.cpp | 17 +- src/rpc/util.cpp | 66 +- src/rpc/util.h | 60 +- src/script/descriptor.cpp | 171 +- src/script/descriptor.h | 12 + src/script/miniscript.cpp | 1 - src/script/miniscript.h | 88 +- src/script/script.cpp | 3 + src/script/script.h | 13 +- src/script/sign.cpp | 3 +- src/script/sign.h | 2 +- src/script/signingprovider.cpp | 353 +- src/script/signingprovider.h | 149 +- src/script/solver.cpp | 224 + src/script/solver.h | 66 + src/script/standard.cpp | 653 --- src/script/standard.h | 330 -- src/secp256k1/CHANGELOG.md | 12 +- src/secp256k1/CMakeLists.txt | 8 +- src/secp256k1/ci/cirrus.sh | 17 +- src/secp256k1/ci/linux-debian.Dockerfile | 72 +- src/secp256k1/examples/CMakeLists.txt | 40 +- src/secp256k1/src/bench_ecmult.c | 3 +- src/secp256k1/src/checkmem.h | 7 + src/secp256k1/src/ctime_tests.c | 20 +- src/secp256k1/src/field.h | 9 +- src/secp256k1/src/field_impl.h | 42 +- src/secp256k1/src/group.h | 8 + src/secp256k1/src/group_impl.h | 165 +- src/secp256k1/src/hash_impl.h | 2 +- .../modules/extrakeys/tests_exhaustive_impl.h | 2 +- .../src/modules/schnorrsig/main_impl.h | 2 +- .../schnorrsig/tests_exhaustive_impl.h | 10 +- src/secp256k1/src/scalar.h | 3 + src/secp256k1/src/scalar_4x64_impl.h | 79 +- src/secp256k1/src/scalar_8x32_impl.h | 80 +- src/secp256k1/src/scalar_impl.h | 19 + src/secp256k1/src/scalar_low_impl.h | 70 +- src/secp256k1/src/tests.c | 157 +- src/secp256k1/src/tests_exhaustive.c | 20 +- src/secp256k1/src/util.h | 8 - src/serialize.h | 302 +- src/signet.cpp | 1 - src/streams.h | 46 +- src/test/addrman_tests.cpp | 38 +- src/test/bip324_tests.cpp | 300 + src/test/blockfilter_index_tests.cpp | 2 +- src/test/blockfilter_tests.cpp | 2 + src/test/blockmanager_tests.cpp | 79 +- src/test/coins_tests.cpp | 2 +- src/test/compress_tests.cpp | 2 +- src/test/crypto_tests.cpp | 412 +- src/test/denialofservice_tests.cpp | 17 +- src/test/descriptor_tests.cpp | 14 +- src/test/fuzz/addrman.cpp | 16 +- src/test/fuzz/bip324.cpp | 130 + src/test/fuzz/buffered_file.cpp | 12 +- src/test/fuzz/coins_view.cpp | 6 +- src/test/fuzz/connman.cpp | 1 + src/test/fuzz/crypto_chacha20.cpp | 72 +- .../fuzz/crypto_chacha20_poly1305_aead.cpp | 72 - src/test/fuzz/crypto_diff_fuzz_chacha20.cpp | 29 +- src/test/fuzz/crypto_poly1305.cpp | 13 +- src/test/fuzz/descriptor_parse.cpp | 15 +- src/test/fuzz/deserialize.cpp | 165 +- src/test/fuzz/fuzz.cpp | 20 +- src/test/fuzz/integer.cpp | 2 +- src/test/fuzz/key.cpp | 20 +- src/test/fuzz/load_external_block_file.cpp | 5 +- src/test/fuzz/message.cpp | 4 +- src/test/fuzz/miniscript.cpp | 2 +- src/test/fuzz/net.cpp | 2 +- src/test/fuzz/p2p_transport_serialization.cpp | 337 +- src/test/fuzz/package_eval.cpp | 294 + src/test/fuzz/process_message.cpp | 4 +- src/test/fuzz/process_messages.cpp | 7 +- src/test/fuzz/rpc.cpp | 10 +- src/test/fuzz/script.cpp | 11 +- src/test/fuzz/script_sign.cpp | 7 +- src/test/fuzz/util.cpp | 54 +- src/test/fuzz/util.h | 47 +- src/test/fuzz/util/mempool.cpp | 3 +- src/test/fuzz/util/net.cpp | 21 + src/test/interfaces_tests.cpp | 2 +- src/test/logging_tests.cpp | 8 +- src/test/miner_tests.cpp | 2 +- src/test/miniminer_tests.cpp | 400 +- src/test/miniscript_tests.cpp | 71 +- src/test/net_tests.cpp | 630 ++- src/test/netbase_tests.cpp | 16 +- src/test/orphanage_tests.cpp | 1 - src/test/script_p2sh_tests.cpp | 2 +- src/test/script_standard_tests.cpp | 11 +- src/test/script_tests.cpp | 1 + src/test/serialize_tests.cpp | 143 + src/test/sigopcount_tests.cpp | 4 +- src/test/streams_tests.cpp | 39 +- src/test/transaction_tests.cpp | 2 +- src/test/txindex_tests.cpp | 2 +- src/test/txpackage_tests.cpp | 30 +- src/test/txvalidation_tests.cpp | 1 - src/test/txvalidationcache_tests.cpp | 2 +- src/test/util/blockfilter.cpp | 2 + src/test/util/chainstate.h | 2 +- src/test/util/mining.cpp | 1 - src/test/util/net.cpp | 37 +- src/test/util/net.h | 3 +- src/test/util/setup_common.cpp | 10 +- src/test/util/txmempool.cpp | 2 +- src/test/util/txmempool.h | 2 + src/test/util/validation.cpp | 4 +- src/test/util/validation.h | 2 +- src/test/util_tests.cpp | 25 + src/test/validation_block_tests.cpp | 1 - .../validation_chainstatemanager_tests.cpp | 15 +- src/torcontrol.cpp | 22 +- src/torcontrol.h | 12 +- src/txdb.cpp | 130 +- src/txdb.h | 30 - src/txmempool.cpp | 60 +- src/txmempool.h | 155 +- src/txorphanage.cpp | 17 +- src/util/check.cpp | 2 +- src/util/message.cpp | 3 +- src/util/time.h | 2 - src/util/trace.h | 35 +- src/util/vector.h | 18 + src/validation.cpp | 392 +- src/validation.h | 42 +- src/version.h | 2 +- src/wallet/coincontrol.h | 1 - src/wallet/coinselection.cpp | 31 +- src/wallet/coinselection.h | 62 +- src/wallet/db.cpp | 2 +- src/wallet/feebumper.cpp | 40 +- src/wallet/feebumper.h | 4 +- src/wallet/interfaces.cpp | 15 +- src/wallet/rpc/addresses.cpp | 6 +- src/wallet/rpc/backup.cpp | 8 +- src/wallet/rpc/coins.cpp | 7 +- src/wallet/rpc/spend.cpp | 47 +- src/wallet/rpc/wallet.cpp | 2 +- src/wallet/scriptpubkeyman.cpp | 19 +- src/wallet/scriptpubkeyman.h | 16 +- src/wallet/spend.cpp | 192 +- src/wallet/spend.h | 6 +- src/wallet/sqlite.cpp | 4 +- src/wallet/test/coinselector_tests.cpp | 270 +- src/wallet/test/fuzz/coinselection.cpp | 107 +- src/wallet/test/ismine_tests.cpp | 3 +- src/wallet/test/scriptpubkeyman_tests.cpp | 2 +- src/wallet/test/spend_tests.cpp | 54 +- src/wallet/test/util.cpp | 1 + src/wallet/test/util.h | 2 +- src/wallet/test/wallet_tests.cpp | 5 +- src/wallet/test/walletload_tests.cpp | 3 + src/wallet/wallet.cpp | 170 +- src/wallet/wallet.h | 68 +- src/wallet/walletdb.cpp | 1 + src/wallet/walletdb.h | 1 - test/functional/feature_blocksdir.py | 4 +- test/functional/feature_cltv.py | 10 +- test/functional/feature_config_args.py | 13 +- test/functional/feature_csv_activation.py | 16 +- test/functional/feature_dersig.py | 4 +- test/functional/feature_nulldummy.py | 2 +- test/functional/feature_pruning.py | 4 +- test/functional/feature_reindex_readonly.py | 64 + test/functional/feature_segwit.py | 17 +- .../feature_txindex_compatibility.py | 11 - test/functional/interface_usdt_mempool.py | 44 +- test/functional/interface_usdt_net.py | 11 +- test/functional/interface_usdt_utxocache.py | 68 +- test/functional/interface_usdt_validation.py | 5 +- test/functional/mempool_dust.py | 3 +- test/functional/mempool_limit.py | 257 +- test/functional/mempool_reorg.py | 91 +- test/functional/mempool_sigoplimit.py | 46 +- test/functional/mining_basic.py | 2 +- .../mining_getblocktemplate_longpoll.py | 9 +- test/functional/p2p_addr_relay.py | 12 +- test/functional/p2p_addrfetch.py | 2 +- test/functional/p2p_blocksonly.py | 2 +- .../p2p_compactblocks_blocksonly.py | 4 +- test/functional/p2p_filter.py | 2 +- test/functional/p2p_ibd_stalling.py | 2 +- test/functional/p2p_ibd_txrelay.py | 2 +- test/functional/p2p_invalid_block.py | 2 - test/functional/p2p_net_deadlock.py | 37 + test/functional/p2p_orphan_handling.py | 416 ++ test/functional/p2p_permissions.py | 6 +- test/functional/p2p_segwit.py | 44 +- test/functional/rpc_deriveaddresses.py | 5 +- test/functional/rpc_estimatefee.py | 3 + test/functional/rpc_misc.py | 2 +- test/functional/rpc_net.py | 63 +- test/functional/rpc_packages.py | 8 +- test/functional/rpc_psbt.py | 80 +- test/functional/test_framework/key.py | 33 +- test/functional/test_framework/p2p.py | 18 +- test/functional/test_framework/script.py | 9 + test/functional/test_framework/test_node.py | 18 + test/functional/test_framework/wallet.py | 25 +- test/functional/test_runner.py | 4 + test/functional/wallet_backup.py | 12 +- test/functional/wallet_basic.py | 6 +- test/functional/wallet_bumpfee.py | 35 +- test/functional/wallet_fundrawtransaction.py | 13 +- test/functional/wallet_hd.py | 4 +- test/functional/wallet_migration.py | 41 +- .../wallet_multisig_descriptor_psbt.py | 3 +- test/functional/wallet_pruning.py | 2 +- .../wallet_resendwallettransactions.py | 26 +- test/functional/wallet_send.py | 5 +- test/functional/wallet_signer.py | 3 +- test/functional/wallet_spend_unconfirmed.py | 508 ++ test/get_previous_releases.py | 9 +- test/lint/lint-circular-dependencies.py | 1 - test/lint/lint-format-strings.py | 2 +- test/lint/lint-include-guards.py | 3 +- test/lint/lint-includes.py | 3 +- test/lint/lint-logs.py | 34 - test/lint/lint-python.py | 10 +- test/lint/run-lint-format-strings.py | 8 +- 540 files changed, 169365 insertions(+), 97718 deletions(-) create mode 100644 .github/actions/install-homebrew-valgrind/action.yml create mode 100644 .github/actions/run-in-docker-action/action.yml create mode 100644 .github/workflows/ci.yml create mode 100755 ci/ci.sh rename ci/test/{00_setup_env_mac_native_arm64.sh => 00_setup_env_mac_native.sh} (81%) delete mode 100755 ci/test/wrap-qemu.sh delete mode 100644 ci/test/wrapped-cl.bat create mode 100644 contrib/devtools/bitcoin-tidy/CMakeLists.txt create mode 100644 contrib/devtools/bitcoin-tidy/README create mode 100644 contrib/devtools/bitcoin-tidy/bitcoin-tidy.cpp create mode 100644 contrib/devtools/bitcoin-tidy/example_logprintf.cpp create mode 100644 contrib/devtools/bitcoin-tidy/logprintf.cpp create mode 100644 contrib/devtools/bitcoin-tidy/logprintf.h delete mode 100644 contrib/guix/patches/gcc-broken-longjmp.patch rename contrib/guix/patches/{gcc-10-remap-guix-store.patch => gcc-remap-guix-store.patch} (100%) create mode 100644 contrib/guix/patches/glibc-2.27-powerpc-ldbrx.patch delete mode 100644 contrib/guix/patches/nsis-disable-installer-reloc.patch delete mode 100644 contrib/guix/patches/nsis-gcc-10-memmove.patch delete mode 100644 contrib/macdeploy/background.tiff delete mode 100644 depends/packages/native_ds_store.mk delete mode 100644 depends/packages/native_mac_alias.mk delete mode 100644 depends/patches/native_cctools/no_fixup_chains.patch create mode 100644 depends/patches/native_libtapi/disable_zlib.patch create mode 100644 doc/release-notes-27213.md create mode 100644 doc/release-notes-28217.md create mode 100644 doc/release-notes-28354.md create mode 100644 doc/release-notes-28414.md create mode 100644 doc/release-notes-28448.md create mode 100644 src/addresstype.cpp create mode 100644 src/addresstype.h delete mode 100644 src/bench/chacha_poly_aead.cpp create mode 100644 src/bench/disconnected_transactions.cpp create mode 100644 src/bip324.cpp create mode 100644 src/bip324.h create mode 100644 src/crypto/chacha20poly1305.cpp create mode 100644 src/crypto/chacha20poly1305.h delete mode 100644 src/crypto/chacha_poly_aead.cpp delete mode 100644 src/crypto/chacha_poly_aead.h create mode 100644 src/kernel/disconnected_transactions.h create mode 100644 src/kernel/mempool_removal_reason.cpp create mode 100644 src/kernel/mempool_removal_reason.h create mode 100644 src/kernel/messagestartchars.h create mode 100644 src/qt/locale/bitcoin_az@latin.ts create mode 100644 src/qt/locale/bitcoin_br.ts create mode 100644 src/qt/locale/bitcoin_cmn.ts create mode 100644 src/qt/locale/bitcoin_de_AT.ts create mode 100644 src/qt/locale/bitcoin_de_CH.ts create mode 100644 src/qt/locale/bitcoin_es_SV.ts create mode 100644 src/qt/locale/bitcoin_fr_CM.ts create mode 100644 src/qt/locale/bitcoin_fr_LU.ts create mode 100644 src/qt/locale/bitcoin_ga_IE.ts create mode 100644 src/qt/locale/bitcoin_hak.ts create mode 100644 src/qt/locale/bitcoin_hi.ts create mode 100644 src/qt/locale/bitcoin_kn.ts create mode 100644 src/qt/locale/bitcoin_mg.ts create mode 100644 src/qt/locale/bitcoin_mr.ts create mode 100644 src/qt/locale/bitcoin_pt@qtfiletype.ts create mode 100644 src/qt/locale/bitcoin_so.ts create mode 100644 src/qt/locale/bitcoin_sr@ijekavianlatin.ts create mode 100644 src/qt/locale/bitcoin_yue.ts create mode 100644 src/qt/locale/bitcoin_zh-Hant.ts create mode 100644 src/script/solver.cpp create mode 100644 src/script/solver.h delete mode 100644 src/script/standard.cpp delete mode 100644 src/script/standard.h create mode 100644 src/test/bip324_tests.cpp create mode 100644 src/test/fuzz/bip324.cpp delete mode 100644 src/test/fuzz/crypto_chacha20_poly1305_aead.cpp create mode 100644 src/test/fuzz/package_eval.cpp create mode 100755 test/functional/feature_reindex_readonly.py create mode 100755 test/functional/p2p_net_deadlock.py create mode 100755 test/functional/p2p_orphan_handling.py create mode 100755 test/functional/wallet_spend_unconfirmed.py delete mode 100755 test/lint/lint-logs.py diff --git a/.cirrus.yml b/.cirrus.yml index 4a6e73ac85b708..96357a103d89d6 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -3,17 +3,11 @@ env: # Global defaults PACKAGE_MANAGER_INSTALL: "apt-get update && apt-get install -y" MAKEJOBS: "-j10" TEST_RUNNER_PORT_MIN: "14000" # Must be larger than 12321, which is used for the http cache. See https://cirrus-ci.org/guide/writing-tasks/#http-cache - CI_FAILFAST_TEST_LEAVE_DANGLING: "1" # Cirrus CI does not care about dangling process and setting this variable avoids killing the CI script itself on error + CI_FAILFAST_TEST_LEAVE_DANGLING: "1" # Cirrus CI does not care about dangling processes and setting this variable avoids killing the CI script itself on error CCACHE_MAXSIZE: "200M" CCACHE_DIR: "/tmp/ccache_dir" CCACHE_NOHASHDIR: "1" # Debug info might contain a stale path if the build dir changes, but this is fine -cirrus_ephemeral_worker_template_env: &CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - DANGER_RUN_CI_ON_HOST: "1" # Containers will be discarded after the run, so there is no risk that the ci scripts modify the system - -persistent_worker_template_env: &PERSISTENT_WORKER_TEMPLATE_ENV - RESTART_CI_DOCKER_BEFORE_RUN: "1" - # https://cirrus-ci.org/guide/persistent-workers/ # # It is possible to select a specific persistent worker by label. Refer to the @@ -24,14 +18,33 @@ persistent_worker_template_env: &PERSISTENT_WORKER_TEMPLATE_ENV # - apt-get is required due to PACKAGE_MANAGER_INSTALL # - podman-docker-4.1+ is required due to the use of `podman` when # RESTART_CI_DOCKER_BEFORE_RUN is set and 4.1+ due to the bugfix in 4.1 -# (https://github.com/bitcoin/bitcoin/pull/21652) -# - The ./ci/ depedencies should be installed: -# apt update && apt install screen python3 bash podman-docker curl -y +# (https://github.com/bitcoin/bitcoin/pull/21652#issuecomment-1657098200) +# - The ./ci/ depedencies (with cirrus-cli) should be installed: +# +# ``` +# apt update && apt install screen python3 bash podman-docker curl -y && curl -L -o cirrus "https://github.com/cirruslabs/cirrus-cli/releases/latest/download/cirrus-linux-$(dpkg --print-architecture)" && mv cirrus /usr/local/bin/cirrus && chmod +x /usr/local/bin/cirrus +# ``` +# +# - There are no strict requirements on the hardware, because having less CPUs +# runs the same CI script (maybe slower). To avoid rare and intermittent OOM +# due to short memory usage spikes, it is recommended to add (and persist) +# swap: +# +# ``` +# fallocate -l 16G /swapfile_ci && chmod 600 /swapfile_ci && mkswap /swapfile_ci && swapon /swapfile_ci && ( echo '/swapfile_ci none swap sw 0 0' | tee -a /etc/fstab ) +# ``` +# +# - To register the persistent worker, open a `screen` session and run: +# +# ``` +# RESTART_CI_DOCKER_BEFORE_RUN=1 screen cirrus worker run --labels type=todo_fill_in_type --token todo_fill_in_token +# ``` # # The following specific types should exist, with the following requirements: +# - small: For an x86_64 machine, recommended to have 2 CPUs and 8 GB of memory. +# - medium: For an x86_64 machine, recommended to have 4 CPUs and 16 GB of memory. # - lunar: For a machine running the Linux kernel shipped with Ubuntu Lunar 23.04. The machine is recommended to have 4 CPUs and 16 GB of memory. -persistent_worker_template: &PERSISTENT_WORKER_TEMPLATE - persistent_worker: {} # Only use this if the task does not care about the type at all +# - arm64: For an aarch64 machine, recommended to have 2 CPUs and 8 GB of memory. # https://cirrus-ci.org/guide/tips-and-tricks/#sharing-configuration-between-tasks filter_template: &FILTER_TEMPLATE @@ -50,25 +63,11 @@ base_template: &BASE_TEMPLATE main_template: &MAIN_TEMPLATE timeout_in: 120m # https://cirrus-ci.org/faq/#instance-timed-out - ccache_cache: - folder: "/tmp/ccache_dir" ci_script: - ./ci/test_run_all.sh -container_depends_template: &CONTAINER_DEPENDS_TEMPLATE - << : *BASE_TEMPLATE - container: - # https://cirrus-ci.org/faq/#are-there-any-limits - # Each project has 16 CPU in total, assign 2 to each container, so that 8 tasks run in parallel - cpu: 2 - memory: 8G # Set to 8GB to avoid OOM. https://cirrus-ci.org/guide/linux/#linux-containers - dockerfile: ci/test_imagefile # https://cirrus-ci.org/guide/docker-builder-vm/#dockerfile-as-a-ci-environment - depends_built_cache: - folder: "depends/built" - fingerprint_script: echo $CIRRUS_TASK_NAME $(git rev-parse HEAD:depends) - global_task_template: &GLOBAL_TASK_TEMPLATE - << : *CONTAINER_DEPENDS_TEMPLATE + << : *BASE_TEMPLATE << : *MAIN_TEMPLATE compute_credits_template: &CREDITS_TEMPLATE @@ -77,7 +76,7 @@ compute_credits_template: &CREDITS_TEMPLATE use_compute_credits: $CIRRUS_REPO_FULL_NAME == 'bitcoin/bitcoin' && $CIRRUS_PR != "" task: - name: 'lint [bookworm]' + name: 'lint' << : *BASE_TEMPLATE container: image: debian:bookworm @@ -92,187 +91,73 @@ task: - git fetch --unshallow --no-tags lint_script: - ./ci/lint_run_all.sh - env: - << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV task: - name: 'tidy [lunar]' + name: 'tidy' << : *GLOBAL_TASK_TEMPLATE - container: - cpu: 4 - memory: 5G - docker_arguments: - CI_IMAGE_NAME_TAG: ubuntu:lunar - FILE_ENV: "./ci/test/00_setup_env_native_tidy.sh" - env: - << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - -task: - name: "Win64 native [vs2022]" - << : *FILTER_TEMPLATE - windows_container: - cpu: 6 - memory: 12G - image: cirrusci/windowsservercore:visualstudio2022 - timeout_in: 120m + persistent_worker: + labels: + type: medium env: - PATH: 'C:\jom;C:\Python39;C:\Python39\Scripts;C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\MSBuild\Current\Bin;%PATH%' - PYTHONUTF8: 1 - CI_VCPKG_TAG: '2023.01.09' - VCPKG_DOWNLOADS: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\downloads' - VCPKG_DEFAULT_BINARY_CACHE: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\archives' - CCACHE_DIR: 'C:\Users\ContainerAdministrator\AppData\Local\ccache' - WRAPPED_CL: 'C:\Users\ContainerAdministrator\AppData\Local\Temp\cirrus-ci-build\ci\test\wrapped-cl.bat' - QT_DOWNLOAD_URL: 'https://download.qt.io/official_releases/qt/5.15/5.15.5/single/qt-everywhere-opensource-src-5.15.5.zip' - QT_LOCAL_PATH: 'C:\qt-everywhere-opensource-src-5.15.5.zip' - QT_SOURCE_DIR: 'C:\qt-everywhere-src-5.15.5' - QTBASEDIR: 'C:\Qt_static' - x64_NATIVE_TOOLS: '"C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build\vcvars64.bat"' - QT_CONFIGURE_COMMAND: '..\configure -release -silent -opensource -confirm-license -opengl desktop -static -static-runtime -mp -qt-zlib -qt-pcre -qt-libpng -nomake examples -nomake tests -nomake tools -no-angle -no-dbus -no-gif -no-gtk -no-ico -no-icu -no-libjpeg -no-libudev -no-sql-sqlite -no-sql-odbc -no-sqlite -no-vulkan -skip qt3d -skip qtactiveqt -skip qtandroidextras -skip qtcharts -skip qtconnectivity -skip qtdatavis3d -skip qtdeclarative -skip doc -skip qtdoc -skip qtgamepad -skip qtgraphicaleffects -skip qtimageformats -skip qtlocation -skip qtlottie -skip qtmacextras -skip qtmultimedia -skip qtnetworkauth -skip qtpurchasing -skip qtquick3d -skip qtquickcontrols -skip qtquickcontrols2 -skip qtquicktimeline -skip qtremoteobjects -skip qtscript -skip qtscxml -skip qtsensors -skip qtserialbus -skip qtserialport -skip qtspeech -skip qtsvg -skip qtvirtualkeyboard -skip qtwayland -skip qtwebchannel -skip qtwebengine -skip qtwebglplugin -skip qtwebsockets -skip qtwebview -skip qtx11extras -skip qtxmlpatterns -no-openssl -no-feature-bearermanagement -no-feature-printdialog -no-feature-printer -no-feature-printpreviewdialog -no-feature-printpreviewwidget -no-feature-sql -no-feature-sqlmodel -no-feature-textbrowser -no-feature-textmarkdownwriter -no-feature-textodfwriter -no-feature-xml' - IgnoreWarnIntDirInTempDetected: 'true' - merge_script: - - PowerShell -NoLogo -Command if ($env:CIRRUS_PR -ne $null) { git fetch $env:CIRRUS_REPO_CLONE_URL pull/$env:CIRRUS_PR/merge; git reset --hard FETCH_HEAD; } - msvc_qt_built_cache: - folder: "%QTBASEDIR%" - reupload_on_changes: false - fingerprint_script: - - echo %QT_DOWNLOAD_URL% %QT_CONFIGURE_COMMAND% - - msbuild -version - populate_script: - - curl -L -o C:\jom.zip http://download.qt.io/official_releases/jom/jom.zip - - mkdir C:\jom - - tar -xf C:\jom.zip -C C:\jom - - curl -L -o %QT_LOCAL_PATH% %QT_DOWNLOAD_URL% - - tar -xf %QT_LOCAL_PATH% -C C:\ - - '%x64_NATIVE_TOOLS%' - - cd %QT_SOURCE_DIR% - - mkdir build - - cd build - - '%QT_CONFIGURE_COMMAND% -prefix %QTBASEDIR%' - - jom - - jom install - vcpkg_tools_cache: - folder: '%VCPKG_DOWNLOADS%\tools' - reupload_on_changes: false - fingerprint_script: - - echo %CI_VCPKG_TAG% - - msbuild -version - vcpkg_binary_cache: - folder: '%VCPKG_DEFAULT_BINARY_CACHE%' - reupload_on_changes: true - fingerprint_script: - - echo %CI_VCPKG_TAG% - - type build_msvc\vcpkg.json - - msbuild -version - populate_script: - - mkdir %VCPKG_DEFAULT_BINARY_CACHE% - ccache_cache: - folder: '%CCACHE_DIR%' - install_tools_script: - - choco install --yes --no-progress ccache --version=4.7.4 - - choco install --yes --no-progress python3 --version=3.9.6 - - pip install zmq - - ccache --version - - python -VV - install_vcpkg_script: - - cd .. - - git clone --quiet https://github.com/microsoft/vcpkg.git - - cd vcpkg - - git -c advice.detachedHead=false checkout %CI_VCPKG_TAG% - - .\bootstrap-vcpkg -disableMetrics - - echo set(VCPKG_BUILD_TYPE release) >> triplets\x64-windows-static.cmake - - .\vcpkg integrate install - - .\vcpkg version - build_script: - - '%x64_NATIVE_TOOLS%' - - cd %CIRRUS_WORKING_DIR% - - ccache --zero-stats - - python build_msvc\msvc-autogen.py - - msbuild build_msvc\bitcoin.sln -property:CLToolExe=%WRAPPED_CL%;UseMultiToolTask=true;Configuration=Release -maxCpuCount -verbosity:minimal -noLogo - - ccache --show-stats - check_script: - - src\test_bitcoin.exe -l test_suite - - src\bench_bitcoin.exe --sanity-check - - python test\util\test_runner.py - - python test\util\rpcauth-test.py - functional_tests_script: - # Increase the dynamic port range to the maximum allowed value to mitigate "OSError: [WinError 10048] Only one usage of each socket address (protocol/network address/port) is normally permitted". - # See: https://learn.microsoft.com/en-us/biztalk/technical-guides/settings-that-can-be-modified-to-improve-network-performance - - netsh int ipv4 set dynamicport tcp start=1025 num=64511 - - netsh int ipv6 set dynamicport tcp start=1025 num=64511 - # Exclude feature_dbcrash for now due to timeout - - python test\functional\test_runner.py --nocleanup --ci --quiet --combinedlogslen=99999999 --jobs=6 --timeout-factor=8 --extended --exclude feature_dbcrash + FILE_ENV: "./ci/test/00_setup_env_native_tidy.sh" task: - name: 'ARM [unit tests, no functional tests] [bullseye]' + name: 'ARM, unit tests, no functional tests' << : *GLOBAL_TASK_TEMPLATE - container: - docker_arguments: - CI_IMAGE_NAME_TAG: debian:bullseye - FILE_ENV: "./ci/test/00_setup_env_arm.sh" - << : *CREDITS_TEMPLATE + persistent_worker: + labels: + type: arm64 # Use arm64 worker to sidestep qemu and avoid a slow CI: https://github.com/bitcoin/bitcoin/pull/28087#issuecomment-1649399453 env: - << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV + FILE_ENV: "./ci/test/00_setup_env_arm.sh" task: - name: 'Win64 [unit tests, no gui tests, no boost::process, no functional tests] [jammy]' + name: 'Win64, unit tests, no gui tests, no boost::process, no functional tests' << : *GLOBAL_TASK_TEMPLATE - container: - docker_arguments: - CI_IMAGE_NAME_TAG: ubuntu:jammy - FILE_ENV: "./ci/test/00_setup_env_win64.sh" - << : *CREDITS_TEMPLATE + persistent_worker: + labels: + type: small env: - << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV + FILE_ENV: "./ci/test/00_setup_env_win64.sh" task: - name: '32-bit + dash [gui] [CentOS 9]' + name: '32-bit CentOS, dash, gui' << : *GLOBAL_TASK_TEMPLATE - container: - docker_arguments: - CI_IMAGE_NAME_TAG: quay.io/centos/centos:stream9 - FILE_ENV: "./ci/test/00_setup_env_i686_centos.sh" - # For faster CI feedback, immediately schedule one task that runs all tests - << : *CREDITS_TEMPLATE + persistent_worker: + labels: + type: small env: - << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - PACKAGE_MANAGER_INSTALL: "yum install -y" + FILE_ENV: "./ci/test/00_setup_env_i686_centos.sh" task: - name: '[previous releases, qt5 dev package and depends packages, DEBUG] [focal]' - previous_releases_cache: - folder: "releases" + name: 'previous releases, qt5 dev package and depends packages, DEBUG' << : *GLOBAL_TASK_TEMPLATE - << : *PERSISTENT_WORKER_TEMPLATE + persistent_worker: + labels: + type: small env: - << : *PERSISTENT_WORKER_TEMPLATE_ENV FILE_ENV: "./ci/test/00_setup_env_native_qt5.sh" task: - name: '[TSan, depends, gui] [lunar]' + name: 'TSan, depends, gui' << : *GLOBAL_TASK_TEMPLATE - container: - cpu: 4 - memory: 16G # The default memory is too small, so double everything - docker_arguments: - CI_IMAGE_NAME_TAG: ubuntu:lunar - FILE_ENV: "./ci/test/00_setup_env_native_tsan.sh" + persistent_worker: + labels: + type: medium env: - << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV + FILE_ENV: "./ci/test/00_setup_env_native_tsan.sh" task: - name: '[MSan, depends] [jammy]' + name: 'MSan, depends' << : *GLOBAL_TASK_TEMPLATE - container: - docker_arguments: - CI_IMAGE_NAME_TAG: ubuntu:jammy - FILE_ENV: "./ci/test/00_setup_env_native_msan.sh" + persistent_worker: + labels: + type: small + timeout_in: 300m # Use longer timeout for the *rare* case where a full build (llvm + msan + depends + ...) needs to be done. env: - << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - MAKEJOBS: "-j4" # Avoid excessive memory use due to MSan + FILE_ENV: "./ci/test/00_setup_env_native_msan.sh" task: - name: '[ASan + LSan + UBSan + integer, no depends, USDT] [lunar]' + name: 'ASan + LSan + UBSan + integer, no depends, USDT' enable_bpfcc_script: # In the image build step, no external environment variables are available, # so any settings will need to be written to the settings env file: @@ -282,72 +167,40 @@ task: labels: type: lunar # Must use the lunar-specific worker (needed for USDT functional tests) env: - << : *PERSISTENT_WORKER_TEMPLATE_ENV FILE_ENV: "./ci/test/00_setup_env_native_asan.sh" task: - name: '[fuzzer,address,undefined,integer, no depends] [lunar]' + name: 'fuzzer,address,undefined,integer, no depends' << : *GLOBAL_TASK_TEMPLATE - container: - cpu: 4 # Increase CPU and memory to avoid timeout - memory: 16G - docker_arguments: - CI_IMAGE_NAME_TAG: ubuntu:lunar - FILE_ENV: "./ci/test/00_setup_env_native_fuzz.sh" + persistent_worker: + labels: + type: medium env: - << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV + FILE_ENV: "./ci/test/00_setup_env_native_fuzz.sh" task: - name: '[multiprocess, i686, DEBUG] [focal]' + name: 'multiprocess, i686, DEBUG' << : *GLOBAL_TASK_TEMPLATE - container: - cpu: 4 - memory: 16G # The default memory is too small, so double everything - docker_arguments: - CI_IMAGE_NAME_TAG: ubuntu:focal - FILE_ENV: "./ci/test/00_setup_env_i686_multiprocess.sh" + persistent_worker: + labels: + type: medium env: - << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV + FILE_ENV: "./ci/test/00_setup_env_i686_multiprocess.sh" task: - name: '[no wallet, libbitcoinkernel] [focal]' + name: 'no wallet, libbitcoinkernel' << : *GLOBAL_TASK_TEMPLATE - container: - docker_arguments: - CI_IMAGE_NAME_TAG: ubuntu:focal - FILE_ENV: "./ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh" - << : *CREDITS_TEMPLATE - env: - << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - -task: - name: 'macOS 11.0 [gui, no tests] [jammy]' - << : *CONTAINER_DEPENDS_TEMPLATE - container: - docker_arguments: - CI_IMAGE_NAME_TAG: ubuntu:jammy - FILE_ENV: "./ci/test/00_setup_env_mac.sh" - macos_sdk_cache: - folder: "depends/SDKs/$MACOS_SDK" - fingerprint_key: "$MACOS_SDK" - << : *MAIN_TEMPLATE + persistent_worker: + labels: + type: small env: - MACOS_SDK: "Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers" - << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV + FILE_ENV: "./ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh" task: - name: 'macOS 13 native arm64 [gui, sqlite only] [no depends]' - macos_instance: - # Use latest image, but hardcode version to avoid silent upgrades (and breaks) - image: ghcr.io/cirruslabs/macos-ventura-xcode:14.3.1 # https://cirrus-ci.org/guide/macOS - << : *BASE_TEMPLATE - check_clang_script: - - clang --version - brew_install_script: - - brew install boost libevent qt@5 miniupnpc libnatpmp ccache zeromq qrencode libtool automake gnu-getopt - << : *MAIN_TEMPLATE + name: 'macOS-cross 11.0, gui, no tests' + << : *GLOBAL_TASK_TEMPLATE + persistent_worker: + labels: + type: small env: - << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - CI_USE_APT_INSTALL: "no" - PACKAGE_MANAGER_INSTALL: "echo" # Nothing to do - FILE_ENV: "./ci/test/00_setup_env_mac_native_arm64.sh" + FILE_ENV: "./ci/test/00_setup_env_mac.sh" diff --git a/.github/ISSUE_TEMPLATE/good_first_issue.yml b/.github/ISSUE_TEMPLATE/good_first_issue.yml index c40dad9687a054..133937c011af6b 100644 --- a/.github/ISSUE_TEMPLATE/good_first_issue.yml +++ b/.github/ISSUE_TEMPLATE/good_first_issue.yml @@ -29,8 +29,10 @@ body: attributes: label: Useful Skills description: For example, “`std::thread`”, “Qt5 GUI and async GUI design” or “basic understanding of Bitcoin mining and the Bitcoin Core RPC interface”. - validations: - required: false + value: | + * Compiling Bitcoin Core from source + * Running the C++ unit tests and the Python functional tests + * ... - type: textarea attributes: label: Guidance for new contributors diff --git a/.github/actions/install-homebrew-valgrind/action.yml b/.github/actions/install-homebrew-valgrind/action.yml new file mode 100644 index 00000000000000..094ff891f71f73 --- /dev/null +++ b/.github/actions/install-homebrew-valgrind/action.yml @@ -0,0 +1,33 @@ +name: "Install Valgrind" +description: "Install Homebrew's Valgrind package and cache it." +runs: + using: "composite" + steps: + - run: | + brew tap LouisBrunner/valgrind + brew fetch --HEAD LouisBrunner/valgrind/valgrind + echo "CI_HOMEBREW_CELLAR_VALGRIND=$(brew --cellar valgrind)" >> "$GITHUB_ENV" + shell: bash + + - run: | + sw_vers > valgrind_fingerprint + brew --version >> valgrind_fingerprint + git -C "$(brew --cache)/valgrind--git" rev-parse HEAD >> valgrind_fingerprint + cat valgrind_fingerprint + shell: bash + + - uses: actions/cache@v3 + id: cache + with: + path: ${{ env.CI_HOMEBREW_CELLAR_VALGRIND }} + key: ${{ github.job }}-valgrind-${{ hashFiles('valgrind_fingerprint') }} + + - if: steps.cache.outputs.cache-hit != 'true' + run: | + brew install --HEAD LouisBrunner/valgrind/valgrind + shell: bash + + - if: steps.cache.outputs.cache-hit == 'true' + run: | + brew link valgrind + shell: bash diff --git a/.github/actions/run-in-docker-action/action.yml b/.github/actions/run-in-docker-action/action.yml new file mode 100644 index 00000000000000..d357c3cf751c8b --- /dev/null +++ b/.github/actions/run-in-docker-action/action.yml @@ -0,0 +1,49 @@ +name: 'Run in Docker with environment' +description: 'Run a command in a Docker container, while passing explicitly set environment variables into the container.' +inputs: + dockerfile: + description: 'A Dockerfile that defines an image' + required: true + tag: + description: 'A tag of an image' + required: true + command: + description: 'A command to run in a container' + required: false + default: ./ci/ci.sh +runs: + using: "composite" + steps: + - uses: docker/setup-buildx-action@v2 + + - uses: docker/build-push-action@v4 + id: main_builder + continue-on-error: true + with: + context: . + file: ${{ inputs.dockerfile }} + tags: ${{ inputs.tag }} + load: true + cache-from: type=gha + + - uses: docker/build-push-action@v4 + id: retry_builder + if: steps.main_builder.outcome == 'failure' + with: + context: . + file: ${{ inputs.dockerfile }} + tags: ${{ inputs.tag }} + load: true + cache-from: type=gha + + - # Tell Docker to pass environment variables in `env` into the container. + run: > + docker run \ + $(echo '${{ toJSON(env) }}' | jq -r 'keys[] | "--env \(.) "') \ + --volume ${{ github.workspace }}:${{ github.workspace }} \ + --workdir ${{ github.workspace }} \ + ${{ inputs.tag }} bash -c " + git config --global --add safe.directory ${{ github.workspace }} + ${{ inputs.command }} + " + shell: bash diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000000000..ab89533f114e48 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,832 @@ +name: CI +on: + pull_request: + push: + branches: + - '**' + tags-ignore: + - '**' + +concurrency: + group: ${{ github.event_name != 'pull_request' && github.run_id || github.ref }} + cancel-in-progress: true + +env: + ### compiler options + HOST: + WRAPPER_CMD: + # Specific warnings can be disabled with -Wno-error=foo. + # -pedantic-errors is not equivalent to -Werror=pedantic and thus not implied by -Werror according to the GCC manual. + WERROR_CFLAGS: '-Werror -pedantic-errors' + MAKEFLAGS: '-j4' + BUILD: 'check' + ### secp256k1 config + ECMULTWINDOW: 'auto' + ECMULTGENPRECISION: 'auto' + ASM: 'no' + WIDEMUL: 'auto' + WITH_VALGRIND: 'yes' + EXTRAFLAGS: + ### secp256k1 modules + EXPERIMENTAL: 'no' + ECDH: 'no' + RECOVERY: 'no' + SCHNORRSIG: 'no' + ELLSWIFT: 'no' + ### test options + SECP256K1_TEST_ITERS: + BENCH: 'yes' + SECP256K1_BENCH_ITERS: 2 + CTIMETESTS: 'yes' + # Compile and run the examples. + EXAMPLES: 'yes' + +jobs: + test-each-commit: + name: 'test each commit' + runs-on: ubuntu-22.04 + if: github.event_name == 'pull_request' && github.event.pull_request.commits != 1 + timeout-minutes: 360 # Use maximum time, see https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idtimeout-minutes. Assuming a worst case time of 1 hour per commit, this leads to a --max-count=6 below. + env: + MAX_COUNT: 6 + steps: + - run: echo "FETCH_DEPTH=$((${{ github.event.pull_request.commits }} + 2))" >> "$GITHUB_ENV" + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: ${{ env.FETCH_DEPTH }} + - run: | + git checkout HEAD~ + echo "COMMIT_AFTER_LAST_MERGE=$(git log $(git log --merges -1 --format=%H)..HEAD --format=%H --max-count=${{ env.MAX_COUNT }} | tail -1)" >> "$GITHUB_ENV" + - run: sudo apt install clang ccache build-essential libtool autotools-dev automake pkg-config bsdmainutils python3-zmq libevent-dev libboost-dev libsqlite3-dev libdb++-dev systemtap-sdt-dev libminiupnpc-dev libnatpmp-dev libqt5gui5 libqt5core5a libqt5dbus5 qttools5-dev qttools5-dev-tools qtwayland5 libqrencode-dev -y + - name: Compile and run tests + run: | + # Use clang++, because it is a bit faster and uses less memory than g++ + git rebase --exec "echo Running test-one-commit on \$( git log -1 ) && ./autogen.sh && CC=clang CXX=clang++ ./configure && make clean && make -j $(nproc) check && ./test/functional/test_runner.py -j $(( $(nproc) * 2 ))" ${{ env.COMMIT_AFTER_LAST_MERGE }}~1 + + macos-native-x86_64: + name: 'macOS 13 native, x86_64, no depends, sqlite only, gui' + # Use latest image, but hardcode version to avoid silent upgrades (and breaks). + # See: https://github.com/actions/runner-images#available-images. + runs-on: macos-13 # Use M1 once available https://github.com/github/roadmap/issues/528 + + - name: Build container + uses: docker/build-push-action@v4 + with: + file: ./ci/linux-debian.Dockerfile + tags: linux-debian-image + cache-from: type=gha + cache-to: type=gha,mode=min + + linux_debian: + name: "x86_64: Linux (Debian stable)" + runs-on: ubuntu-latest + needs: docker_cache + + strategy: + fail-fast: false + matrix: + configuration: + - env_vars: { WIDEMUL: 'int64', RECOVERY: 'yes' } + - env_vars: { WIDEMUL: 'int64', ECDH: 'yes', SCHNORRSIG: 'yes', ELLSWIFT: 'yes' } + - env_vars: { WIDEMUL: 'int128' } + - env_vars: { WIDEMUL: 'int128_struct', ELLSWIFT: 'yes' } + - env_vars: { WIDEMUL: 'int128', RECOVERY: 'yes', SCHNORRSIG: 'yes', ELLSWIFT: 'yes' } + - env_vars: { WIDEMUL: 'int128', ECDH: 'yes', SCHNORRSIG: 'yes' } + - env_vars: { WIDEMUL: 'int128', ASM: 'x86_64', ELLSWIFT: 'yes' } + - env_vars: { RECOVERY: 'yes', SCHNORRSIG: 'yes' } + - env_vars: { CTIMETESTS: 'no', RECOVERY: 'yes', ECDH: 'yes', SCHNORRSIG: 'yes', CPPFLAGS: '-DVERIFY' } + - env_vars: { BUILD: 'distcheck', WITH_VALGRIND: 'no', CTIMETESTS: 'no', BENCH: 'no' } + - env_vars: { CPPFLAGS: '-DDETERMINISTIC' } + - env_vars: { CFLAGS: '-O0', CTIMETESTS: 'no' } + - env_vars: { CFLAGS: '-O1', RECOVERY: 'yes', ECDH: 'yes', SCHNORRSIG: 'yes', ELLSWIFT: 'yes' } + - env_vars: { ECMULTGENPRECISION: 2, ECMULTWINDOW: 2 } + - env_vars: { ECMULTGENPRECISION: 8, ECMULTWINDOW: 4 } + cc: + - 'gcc' + - 'clang' + - 'gcc-snapshot' + - 'clang-snapshot' + + env: + CC: ${{ matrix.cc }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Clang version + run: clang --version + + - name: Install Homebrew packages + run: brew install automake libtool pkg-config gnu-getopt ccache boost libevent miniupnpc libnatpmp zeromq qt@5 qrencode + + - name: Set Ccache directory + run: echo "CCACHE_DIR=${RUNNER_TEMP}/ccache_dir" >> "$GITHUB_ENV" + + - name: Restore Ccache cache + id: ccache-cache + uses: actions/cache/restore@v3 + with: + dockerfile: ./ci/linux-debian.Dockerfile + tag: linux-debian-image + + - run: cat tests.log || true + if: ${{ always() }} + - run: cat noverify_tests.log || true + if: ${{ always() }} + - run: cat exhaustive_tests.log || true + if: ${{ always() }} + - run: cat ctime_tests.log || true + if: ${{ always() }} + - run: cat bench.log || true + if: ${{ always() }} + - run: cat config.log || true + if: ${{ always() }} + - run: cat test_env.log || true + if: ${{ always() }} + - name: CI env + run: env + if: ${{ always() }} + + i686_debian: + name: "i686: Linux (Debian stable)" + runs-on: ubuntu-latest + needs: docker_cache + + strategy: + fail-fast: false + matrix: + cc: + - 'i686-linux-gnu-gcc' + - 'clang --target=i686-pc-linux-gnu -isystem /usr/i686-linux-gnu/include' + + env: + HOST: 'i686-linux-gnu' + ECDH: 'yes' + RECOVERY: 'yes' + SCHNORRSIG: 'yes' + ELLSWIFT: 'yes' + CC: ${{ matrix.cc }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: CI script + uses: ./.github/actions/run-in-docker-action + with: + dockerfile: ./ci/linux-debian.Dockerfile + tag: linux-debian-image + + - run: cat tests.log || true + if: ${{ always() }} + - run: cat noverify_tests.log || true + if: ${{ always() }} + - run: cat exhaustive_tests.log || true + if: ${{ always() }} + - run: cat ctime_tests.log || true + if: ${{ always() }} + - run: cat bench.log || true + if: ${{ always() }} + - run: cat config.log || true + if: ${{ always() }} + - run: cat test_env.log || true + if: ${{ always() }} + - name: CI env + run: env + if: ${{ always() }} + + s390x_debian: + name: "s390x (big-endian): Linux (Debian stable, QEMU)" + runs-on: ubuntu-latest + needs: docker_cache + + env: + WRAPPER_CMD: 'qemu-s390x' + SECP256K1_TEST_ITERS: 16 + HOST: 's390x-linux-gnu' + WITH_VALGRIND: 'no' + ECDH: 'yes' + RECOVERY: 'yes' + SCHNORRSIG: 'yes' + ELLSWIFT: 'yes' + CTIMETESTS: 'no' + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: CI script + uses: ./.github/actions/run-in-docker-action + with: + dockerfile: ./ci/linux-debian.Dockerfile + tag: linux-debian-image + + - run: cat tests.log || true + if: ${{ always() }} + - run: cat noverify_tests.log || true + if: ${{ always() }} + - run: cat exhaustive_tests.log || true + if: ${{ always() }} + - run: cat ctime_tests.log || true + if: ${{ always() }} + - run: cat bench.log || true + if: ${{ always() }} + - run: cat config.log || true + if: ${{ always() }} + - run: cat test_env.log || true + if: ${{ always() }} + - name: CI env + run: env + if: ${{ always() }} + + arm32_debian: + name: "ARM32: Linux (Debian stable, QEMU)" + runs-on: ubuntu-latest + needs: docker_cache + + strategy: + fail-fast: false + matrix: + configuration: + - env_vars: {} + - env_vars: { EXPERIMENTAL: 'yes', ASM: 'arm32' } + + env: + WRAPPER_CMD: 'qemu-arm' + SECP256K1_TEST_ITERS: 16 + HOST: 'arm-linux-gnueabihf' + WITH_VALGRIND: 'no' + ECDH: 'yes' + RECOVERY: 'yes' + SCHNORRSIG: 'yes' + ELLSWIFT: 'yes' + CTIMETESTS: 'no' + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: CI script + env: ${{ matrix.configuration.env_vars }} + uses: ./.github/actions/run-in-docker-action + with: + dockerfile: ./ci/linux-debian.Dockerfile + tag: linux-debian-image + + - run: cat tests.log || true + if: ${{ always() }} + - run: cat noverify_tests.log || true + if: ${{ always() }} + - run: cat exhaustive_tests.log || true + if: ${{ always() }} + - run: cat ctime_tests.log || true + if: ${{ always() }} + - run: cat bench.log || true + if: ${{ always() }} + - run: cat config.log || true + if: ${{ always() }} + - run: cat test_env.log || true + if: ${{ always() }} + - name: CI env + run: env + if: ${{ always() }} + + arm64_debian: + name: "ARM64: Linux (Debian stable, QEMU)" + runs-on: ubuntu-latest + needs: docker_cache + + env: + WRAPPER_CMD: 'qemu-aarch64' + SECP256K1_TEST_ITERS: 16 + HOST: 'aarch64-linux-gnu' + WITH_VALGRIND: 'no' + ECDH: 'yes' + RECOVERY: 'yes' + SCHNORRSIG: 'yes' + ELLSWIFT: 'yes' + CTIMETESTS: 'no' + + strategy: + fail-fast: false + matrix: + configuration: + - env_vars: { } # gcc + - env_vars: # clang + CC: 'clang --target=aarch64-linux-gnu' + - env_vars: # clang-snapshot + CC: 'clang-snapshot --target=aarch64-linux-gnu' + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: CI script + env: ${{ matrix.configuration.env_vars }} + uses: ./.github/actions/run-in-docker-action + with: + dockerfile: ./ci/linux-debian.Dockerfile + tag: linux-debian-image + + - run: cat tests.log || true + if: ${{ always() }} + - run: cat noverify_tests.log || true + if: ${{ always() }} + - run: cat exhaustive_tests.log || true + if: ${{ always() }} + - run: cat ctime_tests.log || true + if: ${{ always() }} + - run: cat bench.log || true + if: ${{ always() }} + - run: cat config.log || true + if: ${{ always() }} + - run: cat test_env.log || true + if: ${{ always() }} + - name: CI env + run: env + if: ${{ always() }} + + ppc64le_debian: + name: "ppc64le: Linux (Debian stable, QEMU)" + runs-on: ubuntu-latest + needs: docker_cache + + env: + WRAPPER_CMD: 'qemu-ppc64le' + SECP256K1_TEST_ITERS: 16 + HOST: 'powerpc64le-linux-gnu' + WITH_VALGRIND: 'no' + ECDH: 'yes' + RECOVERY: 'yes' + SCHNORRSIG: 'yes' + ELLSWIFT: 'yes' + CTIMETESTS: 'no' + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: CI script + uses: ./.github/actions/run-in-docker-action + with: + dockerfile: ./ci/linux-debian.Dockerfile + tag: linux-debian-image + + - run: cat tests.log || true + if: ${{ always() }} + - run: cat noverify_tests.log || true + if: ${{ always() }} + - run: cat exhaustive_tests.log || true + if: ${{ always() }} + - run: cat ctime_tests.log || true + if: ${{ always() }} + - run: cat bench.log || true + if: ${{ always() }} + - run: cat config.log || true + if: ${{ always() }} + - run: cat test_env.log || true + if: ${{ always() }} + - name: CI env + run: env + if: ${{ always() }} + + valgrind_debian: + name: "Valgrind (memcheck)" + runs-on: ubuntu-latest + needs: docker_cache + + strategy: + fail-fast: false + matrix: + configuration: + - env_vars: { CC: 'clang', ASM: 'auto' } + - env_vars: { CC: 'i686-linux-gnu-gcc', HOST: 'i686-linux-gnu', ASM: 'auto' } + - env_vars: { CC: 'clang', ASM: 'no', ECMULTGENPRECISION: 2, ECMULTWINDOW: 2 } + - env_vars: { CC: 'i686-linux-gnu-gcc', HOST: 'i686-linux-gnu', ASM: 'no', ECMULTGENPRECISION: 2, ECMULTWINDOW: 2 } + + env: + # The `--error-exitcode` is required to make the test fail if valgrind found errors, + # otherwise it will return 0 (https://www.valgrind.org/docs/manual/manual-core.html). + WRAPPER_CMD: 'valgrind --error-exitcode=42' + ECDH: 'yes' + RECOVERY: 'yes' + SCHNORRSIG: 'yes' + ELLSWIFT: 'yes' + CTIMETESTS: 'no' + SECP256K1_TEST_ITERS: 2 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: CI script + env: ${{ matrix.configuration.env_vars }} + uses: ./.github/actions/run-in-docker-action + with: + dockerfile: ./ci/linux-debian.Dockerfile + tag: linux-debian-image + + - run: cat tests.log || true + if: ${{ always() }} + - run: cat noverify_tests.log || true + if: ${{ always() }} + - run: cat exhaustive_tests.log || true + if: ${{ always() }} + - run: cat ctime_tests.log || true + if: ${{ always() }} + - run: cat bench.log || true + if: ${{ always() }} + - run: cat config.log || true + if: ${{ always() }} + - run: cat test_env.log || true + if: ${{ always() }} + - name: CI env + run: env + if: ${{ always() }} + + sanitizers_debian: + name: "UBSan, ASan, LSan" + runs-on: ubuntu-latest + needs: docker_cache + + strategy: + fail-fast: false + matrix: + configuration: + - env_vars: { CC: 'clang', ASM: 'auto' } + - env_vars: { CC: 'i686-linux-gnu-gcc', HOST: 'i686-linux-gnu', ASM: 'auto' } + - env_vars: { CC: 'clang', ASM: 'no', ECMULTGENPRECISION: 2, ECMULTWINDOW: 2 } + - env_vars: { CC: 'i686-linux-gnu-gcc', HOST: 'i686-linux-gnu', ASM: 'no', ECMULTGENPRECISION: 2, ECMULTWINDOW: 2 } + + env: + ECDH: 'yes' + RECOVERY: 'yes' + SCHNORRSIG: 'yes' + ELLSWIFT: 'yes' + CTIMETESTS: 'no' + CFLAGS: '-fsanitize=undefined,address -g' + UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1' + ASAN_OPTIONS: 'strict_string_checks=1:detect_stack_use_after_return=1:detect_leaks=1' + LSAN_OPTIONS: 'use_unaligned=1' + SECP256K1_TEST_ITERS: 32 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: CI script + env: ${{ matrix.configuration.env_vars }} + uses: ./.github/actions/run-in-docker-action + with: + dockerfile: ./ci/linux-debian.Dockerfile + tag: linux-debian-image + + - run: cat tests.log || true + if: ${{ always() }} + - run: cat noverify_tests.log || true + if: ${{ always() }} + - run: cat exhaustive_tests.log || true + if: ${{ always() }} + - run: cat ctime_tests.log || true + if: ${{ always() }} + - run: cat bench.log || true + if: ${{ always() }} + - run: cat config.log || true + if: ${{ always() }} + - run: cat test_env.log || true + if: ${{ always() }} + - name: CI env + run: env + if: ${{ always() }} + + msan_debian: + name: "MSan" + runs-on: ubuntu-latest + needs: docker_cache + + strategy: + fail-fast: false + matrix: + configuration: + - env_vars: + CFLAGS: '-fsanitize=memory -fsanitize-recover=memory -g' + - env_vars: + ECMULTGENPRECISION: 2 + ECMULTWINDOW: 2 + CFLAGS: '-fsanitize=memory -fsanitize-recover=memory -g -O3' + + env: + ECDH: 'yes' + RECOVERY: 'yes' + SCHNORRSIG: 'yes' + ELLSWIFT: 'yes' + CTIMETESTS: 'yes' + CC: 'clang' + SECP256K1_TEST_ITERS: 32 + ASM: 'no' + WITH_VALGRIND: 'no' + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: CI script + env: ${{ matrix.configuration.env_vars }} + uses: ./.github/actions/run-in-docker-action + with: + dockerfile: ./ci/linux-debian.Dockerfile + tag: linux-debian-image + + - run: cat tests.log || true + if: ${{ always() }} + - run: cat noverify_tests.log || true + if: ${{ always() }} + - run: cat exhaustive_tests.log || true + if: ${{ always() }} + - run: cat ctime_tests.log || true + if: ${{ always() }} + - run: cat bench.log || true + if: ${{ always() }} + - run: cat config.log || true + if: ${{ always() }} + - run: cat test_env.log || true + if: ${{ always() }} + - name: CI env + run: env + if: ${{ always() }} + + mingw_debian: + name: ${{ matrix.configuration.job_name }} + runs-on: ubuntu-latest + needs: docker_cache + + env: + WRAPPER_CMD: 'wine' + WITH_VALGRIND: 'no' + ECDH: 'yes' + RECOVERY: 'yes' + SCHNORRSIG: 'yes' + ELLSWIFT: 'yes' + CTIMETESTS: 'no' + + strategy: + fail-fast: false + matrix: + configuration: + - job_name: 'x86_64 (mingw32-w64): Windows (Debian stable, Wine)' + env_vars: + HOST: 'x86_64-w64-mingw32' + - job_name: 'i686 (mingw32-w64): Windows (Debian stable, Wine)' + env_vars: + HOST: 'i686-w64-mingw32' + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: CI script + env: ${{ matrix.configuration.env_vars }} + uses: ./.github/actions/run-in-docker-action + with: + dockerfile: ./ci/linux-debian.Dockerfile + tag: linux-debian-image + + - run: cat tests.log || true + if: ${{ always() }} + - run: cat noverify_tests.log || true + if: ${{ always() }} + - run: cat exhaustive_tests.log || true + if: ${{ always() }} + - run: cat ctime_tests.log || true + if: ${{ always() }} + - run: cat bench.log || true + if: ${{ always() }} + - run: cat config.log || true + if: ${{ always() }} + - run: cat test_env.log || true + if: ${{ always() }} + - name: CI env + run: env + if: ${{ always() }} + + macos-native: + name: "x86_64: macOS Monterey" + # See: https://github.com/actions/runner-images#available-images. + runs-on: macos-12 # Use M1 once available https://github.com/github/roadmap/issues/528 + + env: + CC: 'clang' + HOMEBREW_NO_AUTO_UPDATE: 1 + HOMEBREW_NO_INSTALL_CLEANUP: 1 + + strategy: + fail-fast: false + matrix: + env_vars: + - { WIDEMUL: 'int64', RECOVERY: 'yes', ECDH: 'yes', SCHNORRSIG: 'yes', ELLSWIFT: 'yes' } + - { WIDEMUL: 'int128_struct', ECMULTGENPRECISION: 2, ECMULTWINDOW: 4 } + - { WIDEMUL: 'int128', ECDH: 'yes', SCHNORRSIG: 'yes', ELLSWIFT: 'yes' } + - { WIDEMUL: 'int128', RECOVERY: 'yes' } + - { WIDEMUL: 'int128', RECOVERY: 'yes', ECDH: 'yes', SCHNORRSIG: 'yes', ELLSWIFT: 'yes' } + - { WIDEMUL: 'int128', RECOVERY: 'yes', ECDH: 'yes', SCHNORRSIG: 'yes', ELLSWIFT: 'yes', CC: 'gcc' } + - { WIDEMUL: 'int128', RECOVERY: 'yes', ECDH: 'yes', SCHNORRSIG: 'yes', ELLSWIFT: 'yes', WRAPPER_CMD: 'valgrind --error-exitcode=42', SECP256K1_TEST_ITERS: 2 } + - { WIDEMUL: 'int128', RECOVERY: 'yes', ECDH: 'yes', SCHNORRSIG: 'yes', ELLSWIFT: 'yes', CC: 'gcc', WRAPPER_CMD: 'valgrind --error-exitcode=42', SECP256K1_TEST_ITERS: 2 } + - { WIDEMUL: 'int128', RECOVERY: 'yes', ECDH: 'yes', SCHNORRSIG: 'yes', ELLSWIFT: 'yes', CPPFLAGS: '-DVERIFY', CTIMETESTS: 'no' } + - BUILD: 'distcheck' + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Homebrew packages + run: | + brew install automake libtool gcc + ln -s $(brew --prefix gcc)/bin/gcc-?? /usr/local/bin/gcc + + - name: Install and cache Valgrind + uses: ./.github/actions/install-homebrew-valgrind + + - name: CI script + env: ${{ matrix.env_vars }} + run: ./ci/ci.sh + + - run: cat tests.log || true + if: ${{ always() }} + - run: cat noverify_tests.log || true + if: ${{ always() }} + - run: cat exhaustive_tests.log || true + if: ${{ always() }} + - run: cat ctime_tests.log || true + if: ${{ always() }} + - run: cat bench.log || true + if: ${{ always() }} + - run: cat config.log || true + if: ${{ always() }} + - run: cat test_env.log || true + if: ${{ always() }} + - name: CI env + run: env + if: ${{ always() }} + + win64-native: + name: ${{ matrix.configuration.job_name }} + # See: https://github.com/actions/runner-images#available-images. + runs-on: windows-2022 + + strategy: + fail-fast: false + matrix: + configuration: + - job_name: 'x64 (MSVC): Windows (VS 2022, shared)' + cmake_options: '-A x64 -DBUILD_SHARED_LIBS=ON' + - job_name: 'x64 (MSVC): Windows (VS 2022, static)' + cmake_options: '-A x64 -DBUILD_SHARED_LIBS=OFF' + - job_name: 'x64 (MSVC): Windows (VS 2022, int128_struct)' + cmake_options: '-A x64 -DSECP256K1_TEST_OVERRIDE_WIDE_MULTIPLY=int128_struct' + - job_name: 'x64 (MSVC): Windows (VS 2022, int128_struct with __(u)mulh)' + cmake_options: '-A x64 -DSECP256K1_TEST_OVERRIDE_WIDE_MULTIPLY=int128_struct' + cpp_flags: '/DSECP256K1_MSVC_MULH_TEST_OVERRIDE' + - job_name: 'x86 (MSVC): Windows (VS 2022)' + cmake_options: '-A Win32' + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Generate buildsystem + run: cmake -E env CFLAGS="/WX ${{ matrix.configuration.cpp_flags }}" cmake -B build -DSECP256K1_ENABLE_MODULE_RECOVERY=ON -DSECP256K1_BUILD_EXAMPLES=ON ${{ matrix.configuration.cmake_options }} + + - name: Build + run: cmake --build build --config RelWithDebInfo -- /p:UseMultiToolTask=true /maxCpuCount + + - name: Binaries info + # Use the bash shell included with Git for Windows. + shell: bash + run: | + cd build/src/RelWithDebInfo && file *tests.exe bench*.exe libsecp256k1-*.dll || true + + - name: Check + run: | + ctest -C RelWithDebInfo --test-dir build -j ([int]$env:NUMBER_OF_PROCESSORS + 1) + build\src\RelWithDebInfo\bench_ecmult.exe + build\src\RelWithDebInfo\bench_internal.exe + build\src\RelWithDebInfo\bench.exe + + win64-native-headers: + name: "x64 (MSVC): C++ (public headers)" + # See: https://github.com/actions/runner-images#available-images. + runs-on: windows-2022 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Add cl.exe to PATH + uses: ilammy/msvc-dev-cmd@v1 + + - name: C++ (public headers) + run: | + cl.exe -c -WX -TP include/*.h + + cxx_fpermissive_debian: + name: "C++ -fpermissive (entire project)" + runs-on: ubuntu-latest + needs: docker_cache + + env: + CC: 'g++' + CFLAGS: '-fpermissive -g' + CPPFLAGS: '-DSECP256K1_CPLUSPLUS_TEST_OVERRIDE' + WERROR_CFLAGS: + ECDH: 'yes' + RECOVERY: 'yes' + SCHNORRSIG: 'yes' + ELLSWIFT: 'yes' + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: CI script + uses: ./.github/actions/run-in-docker-action + with: + dockerfile: ./ci/linux-debian.Dockerfile + tag: linux-debian-image + + - run: cat tests.log || true + if: ${{ always() }} + - run: cat noverify_tests.log || true + if: ${{ always() }} + - run: cat exhaustive_tests.log || true + if: ${{ always() }} + - run: cat ctime_tests.log || true + if: ${{ always() }} + - run: cat bench.log || true + if: ${{ always() }} + - run: cat config.log || true + if: ${{ always() }} + - run: cat test_env.log || true + if: ${{ always() }} + - name: CI env + run: env + if: ${{ always() }} + + cxx_headers_debian: + name: "C++ (public headers)" + runs-on: ubuntu-latest + needs: docker_cache + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: CI script + uses: ./.github/actions/run-in-docker-action + with: + dockerfile: ./ci/linux-debian.Dockerfile + tag: linux-debian-image + command: | + g++ -Werror include/*.h + clang -Werror -x c++-header include/*.h + + sage: + name: "SageMath prover" + runs-on: ubuntu-latest + container: + image: sagemath/sagemath:latest + options: --user root + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: CI script + run: | + cd sage + sage prove_group_implementations.sage + + release: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - run: ./autogen.sh && ./configure --enable-dev-mode && make distcheck + + - name: Check installation with Autotools + env: + CI_INSTALL: ${{ runner.temp }}/${{ github.run_id }}${{ github.action }} + run: | + ./autogen.sh && ./configure --prefix=${{ env.CI_INSTALL }} && make clean && make install && ls -RlAh ${{ env.CI_INSTALL }} + gcc -o ecdsa examples/ecdsa.c $(PKG_CONFIG_PATH=${{ env.CI_INSTALL }}/lib/pkgconfig pkg-config --cflags --libs libsecp256k1) -Wl,-rpath,"${{ env.CI_INSTALL }}/lib" && ./ecdsa + + - name: Check installation with CMake + env: + CI_BUILD: ${{ runner.temp }}/${{ github.run_id }}${{ github.action }}/build + CI_INSTALL: ${{ runner.temp }}/${{ github.run_id }}${{ github.action }}/install + run: | + cmake -B ${{ env.CI_BUILD }} -DCMAKE_INSTALL_PREFIX=${{ env.CI_INSTALL }} && cmake --build ${{ env.CI_BUILD }} --target install && ls -RlAh ${{ env.CI_INSTALL }} + gcc -o ecdsa examples/ecdsa.c -I ${{ env.CI_INSTALL }}/include -L ${{ env.CI_INSTALL }}/lib*/ -l secp256k1 -Wl,-rpath,"${{ env.CI_INSTALL }}/lib",-rpath,"${{ env.CI_INSTALL }}/lib64" && ./ecdsa diff --git a/.gitignore b/.gitignore index 6ca9d39a16b360..c77303f50e2ee9 100644 --- a/.gitignore +++ b/.gitignore @@ -74,7 +74,7 @@ src/qt/bitcoin-qt.includes *.log *.trs -*.dmg +*.zip *.json.h *.raw.h diff --git a/.tx/config b/.tx/config index ab1fb933d833ee..e75ce0afcb7cf1 100644 --- a/.tx/config +++ b/.tx/config @@ -1,7 +1,7 @@ [main] host = https://www.transifex.com -[o:bitcoin:p:bitcoin:r:qt-translation-025x] +[o:bitcoin:p:bitcoin:r:qt-translation-026x] file_filter = src/qt/locale/bitcoin_.xlf source_file = src/qt/locale/bitcoin_en.xlf source_lang = en diff --git a/Makefile.am b/Makefile.am index 8b61763ae4d6cb..7c3cd82e7f0b98 100644 --- a/Makefile.am +++ b/Makefile.am @@ -37,7 +37,7 @@ space := $(empty) $(empty) OSX_APP=Bitcoin-Qt.app OSX_VOLNAME = $(subst $(space),-,$(PACKAGE_NAME)) -OSX_DMG = $(OSX_VOLNAME).dmg +OSX_ZIP = $(OSX_VOLNAME).zip OSX_DEPLOY_SCRIPT=$(top_srcdir)/contrib/macdeploy/macdeployqtplus OSX_INSTALLER_ICONS=$(top_srcdir)/src/qt/res/icons/bitcoin.icns OSX_PLIST=$(top_builddir)/share/qt/Info.plist #not installed @@ -124,15 +124,16 @@ osx_volname: echo $(OSX_VOLNAME) >$@ if BUILD_DARWIN -$(OSX_DMG): $(OSX_APP_BUILT) $(OSX_PACKAGING) - $(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) $(OSX_VOLNAME) -translations-dir=$(QT_TRANSLATION_DIR) -dmg +$(OSX_ZIP): $(OSX_APP_BUILT) $(OSX_PACKAGING) + $(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) $(OSX_VOLNAME) -translations-dir=$(QT_TRANSLATION_DIR) -zip -deploydir: $(OSX_DMG) +deploydir: $(OSX_ZIP) else !BUILD_DARWIN APP_DIST_DIR=$(top_builddir)/dist -$(OSX_DMG): deploydir - $(XORRISOFS) -D -l -V "$(OSX_VOLNAME)" -no-pad -r -dir-mode 0755 -o $@ $(APP_DIST_DIR) -- $(if $(SOURCE_DATE_EPOCH),-volume_date all_file_dates =$(SOURCE_DATE_EPOCH)) +$(OSX_ZIP): deploydir + if [ -n "$(SOURCE_DATE_EPOCH)" ]; then find $(APP_DIST_DIR) -exec touch -d @$(SOURCE_DATE_EPOCH) {} +; fi + cd $(APP_DIST_DIR) && find . | sort | $(ZIP) -X@ $@ $(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Bitcoin-Qt: $(OSX_APP_BUILT) $(OSX_PACKAGING) INSTALL_NAME_TOOL=$(INSTALL_NAME_TOOL) OTOOL=$(OTOOL) STRIP=$(STRIP) $(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) $(OSX_VOLNAME) -translations-dir=$(QT_TRANSLATION_DIR) @@ -140,7 +141,7 @@ $(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Bitcoin-Qt: $(OSX_APP_BUILT) $(OSX_PAC deploydir: $(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Bitcoin-Qt endif !BUILD_DARWIN -deploy: $(OSX_DMG) +deploy: $(OSX_ZIP) endif $(BITCOIN_QT_BIN): FORCE @@ -313,7 +314,7 @@ EXTRA_DIST += \ test/util/data/txcreatesignv2.hex \ test/util/rpcauth-test.py -CLEANFILES = $(OSX_DMG) $(BITCOIN_WIN_INSTALLER) +CLEANFILES = $(OSX_ZIP) $(BITCOIN_WIN_INSTALLER) DISTCHECK_CONFIGURE_FLAGS = --enable-man diff --git a/ci/README.md b/ci/README.md index b4158d01832e3a..f89af329dd84cd 100644 --- a/ci/README.md +++ b/ci/README.md @@ -20,10 +20,11 @@ requires `bash`, `docker`, and `python3` to be installed. To install all require sudo apt install bash docker.io python3 ``` -To run the test stage with a specific configuration, +It is recommended to run the ci system in a clean env. To run the test stage +with a specific configuration, ``` -FILE_ENV="./ci/test/00_setup_env_arm.sh" ./ci/test_run_all.sh +env -i HOME="$HOME" PATH="$PATH" USER="$USER" bash -c 'FILE_ENV="./ci/test/00_setup_env_arm.sh" ./ci/test_run_all.sh' ``` ### Configurations @@ -38,14 +39,11 @@ the system package manager to install build dependencies. This guarantees that the tester is using the same versions as the release builds, which also use `./depends`. -If no `FILE_ENV` has been specified or values are left out, `00_setup_env.sh` -is used as the default configuration with fallback values. - It is also possible to force a specific configuration without modifying the file. For example, ``` -MAKEJOBS="-j1" FILE_ENV="./ci/test/00_setup_env_arm.sh" ./ci/test_run_all.sh +env -i HOME="$HOME" PATH="$PATH" USER="$USER" bash -c 'MAKEJOBS="-j1" FILE_ENV="./ci/test/00_setup_env_arm.sh" ./ci/test_run_all.sh' ``` The files starting with `0n` (`n` greater than 0) are the scripts that are run diff --git a/ci/ci.sh b/ci/ci.sh new file mode 100755 index 00000000000000..719e7851efdf94 --- /dev/null +++ b/ci/ci.sh @@ -0,0 +1,132 @@ +#!/bin/sh + +set -eux + +export LC_ALL=C + +# Print commit and relevant CI environment to allow reproducing the job outside of CI. +git show --no-patch +print_environment() { + # Turn off -x because it messes up the output + set +x + # There are many ways to print variable names and their content. This one + # does not rely on bash. + for var in WERROR_CFLAGS MAKEFLAGS BUILD \ + ECMULTWINDOW ECMULTGENPRECISION ASM WIDEMUL WITH_VALGRIND EXTRAFLAGS \ + EXPERIMENTAL ECDH RECOVERY SCHNORRSIG ELLSWIFT \ + SECP256K1_TEST_ITERS BENCH SECP256K1_BENCH_ITERS CTIMETESTS\ + EXAMPLES \ + HOST WRAPPER_CMD \ + CC CFLAGS CPPFLAGS AR NM + do + eval "isset=\${$var+x}" + if [ -n "$isset" ]; then + eval "val=\${$var}" + # shellcheck disable=SC2154 + printf '%s="%s" ' "$var" "$val" + fi + done + echo "$0" + set -x +} +print_environment + +env >> test_env.log + +# If gcc is requested, assert that it's in fact gcc (and not some symlinked Apple clang). +case "${CC:-undefined}" in + *gcc*) + $CC -v 2>&1 | grep -q "gcc version" || exit 1; + ;; +esac + +if [ -n "${CC+x}" ]; then + # The MSVC compiler "cl" doesn't understand "-v" + $CC -v || true +fi +if [ "$WITH_VALGRIND" = "yes" ]; then + valgrind --version +fi +if [ -n "$WRAPPER_CMD" ]; then + $WRAPPER_CMD --version +fi + +# Workaround for https://bugs.kde.org/show_bug.cgi?id=452758 (fixed in valgrind 3.20.0). +case "${CC:-undefined}" in + clang*) + if [ "$CTIMETESTS" = "yes" ] && [ "$WITH_VALGRIND" = "yes" ] + then + export CFLAGS="${CFLAGS:+$CFLAGS }-gdwarf-4" + else + case "$WRAPPER_CMD" in + valgrind*) + export CFLAGS="${CFLAGS:+$CFLAGS }-gdwarf-4" + ;; + esac + fi + ;; +esac + +./autogen.sh + +./configure \ + --enable-experimental="$EXPERIMENTAL" \ + --with-test-override-wide-multiply="$WIDEMUL" --with-asm="$ASM" \ + --with-ecmult-window="$ECMULTWINDOW" \ + --with-ecmult-gen-precision="$ECMULTGENPRECISION" \ + --enable-module-ecdh="$ECDH" --enable-module-recovery="$RECOVERY" \ + --enable-module-ellswift="$ELLSWIFT" \ + --enable-module-schnorrsig="$SCHNORRSIG" \ + --enable-examples="$EXAMPLES" \ + --enable-ctime-tests="$CTIMETESTS" \ + --with-valgrind="$WITH_VALGRIND" \ + --host="$HOST" $EXTRAFLAGS + +# We have set "-j" in MAKEFLAGS. +make + +# Print information about binaries so that we can see that the architecture is correct +file *tests* || true +file bench* || true +file .libs/* || true + +# This tells `make check` to wrap test invocations. +export LOG_COMPILER="$WRAPPER_CMD" + +make "$BUILD" + +# Using the local `libtool` because on macOS the system's libtool has nothing to do with GNU libtool +EXEC='./libtool --mode=execute' +if [ -n "$WRAPPER_CMD" ] +then + EXEC="$EXEC $WRAPPER_CMD" +fi + +if [ "$BENCH" = "yes" ] +then + { + $EXEC ./bench_ecmult + $EXEC ./bench_internal + $EXEC ./bench + } >> bench.log 2>&1 +fi + +if [ "$CTIMETESTS" = "yes" ] +then + if [ "$WITH_VALGRIND" = "yes" ]; then + ./libtool --mode=execute valgrind --error-exitcode=42 ./ctime_tests > ctime_tests.log 2>&1 + else + $EXEC ./ctime_tests > ctime_tests.log 2>&1 + fi +fi + +# Rebuild precomputed files (if not cross-compiling). +if [ -z "$HOST" ] +then + make clean-precomp clean-testvectors + make precomp testvectors +fi + +# Check that no repo files have been modified by the build. +# (This fails for example if the precomp files need to be updated in the repo.) +git diff --exit-code diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index 722a87789039d3..8b2adbb0966904 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -8,10 +8,14 @@ export LC_ALL=C.UTF-8 set -ex -# The root dir. +# The source root dir, usually from git, usually read-only. # The ci system copies this folder. -BASE_ROOT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../../ >/dev/null 2>&1 && pwd ) -export BASE_ROOT_DIR +BASE_READ_ONLY_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../../ >/dev/null 2>&1 && pwd ) +export BASE_READ_ONLY_DIR +# The destination root dir inside the container. +# This folder will also hold any SDKs. +# This folder only exists on the ci guest and will be a copy of BASE_READ_ONLY_DIR +export BASE_ROOT_DIR="${BASE_ROOT_DIR:-/ci_container_base}" # The depends dir. # This folder exists only on the ci guest, and on the ci host as a volume. export DEPENDS_DIR=${DEPENDS_DIR:-$BASE_ROOT_DIR/depends} @@ -32,10 +36,6 @@ fi echo "Fallback to default values in env (if not yet set)" # The number of parallel jobs to pass down to make and test_runner.py export MAKEJOBS=${MAKEJOBS:--j4} -# What host to compile for. See also ./depends/README.md -# Tests that need cross-compilation export the appropriate HOST. -# Tests that run natively guess the host -export HOST=${HOST:-$("$BASE_ROOT_DIR/depends/config.guess")} # Whether to prefer BusyBox over GNU utilities export USE_BUSY_BOX=${USE_BUSY_BOX:-false} @@ -61,15 +61,13 @@ export CCACHE_COMPRESS=${CCACHE_COMPRESS:-1} # This folder exists only on the ci guest, and on the ci host as a volume. export CCACHE_DIR=${CCACHE_DIR:-$BASE_SCRATCH_DIR/.ccache} # Folder where the build result is put (bin and lib). -export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_SCRATCH_DIR/out/$HOST} +export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_SCRATCH_DIR/out} # Folder where the build is done (dist and out-of-tree build). export BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_SCRATCH_DIR/build} # The folder for previous release binaries. # This folder exists only on the ci guest, and on the ci host as a volume. -export PREVIOUS_RELEASES_DIR=${PREVIOUS_RELEASES_DIR:-$BASE_ROOT_DIR/releases/$HOST} -export DIR_IWYU="${BASE_SCRATCH_DIR}/iwyu" -export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks} -export CI_BASE_PACKAGES=${CI_BASE_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git procps bison} +export PREVIOUS_RELEASES_DIR=${PREVIOUS_RELEASES_DIR:-$BASE_ROOT_DIR/prev_releases} +export CI_BASE_PACKAGES=${CI_BASE_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git procps bison e2fsprogs} export GOAL=${GOAL:-install} export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_SCRATCH_DIR}/qa-assets} export CI_RETRY_EXE=${CI_RETRY_EXE:-"retry --"} diff --git a/ci/test/00_setup_env_android.sh b/ci/test/00_setup_env_android.sh index 1834bd0bc4ae78..97a6bd73181a47 100755 --- a/ci/test/00_setup_env_android.sh +++ b/ci/test/00_setup_env_android.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2022 The Bitcoin Core developers +# Copyright (c) 2019-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -9,7 +9,7 @@ export LC_ALL=C.UTF-8 export HOST=aarch64-linux-android export PACKAGES="unzip openjdk-8-jdk gradle" export CONTAINER_NAME=ci_android -export CI_IMAGE_NAME_TAG="ubuntu:jammy" +export CI_IMAGE_NAME_TAG="docker.io/amd64/ubuntu:22.04" export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false diff --git a/ci/test/00_setup_env_arm.sh b/ci/test/00_setup_env_arm.sh index ac0c0be96a7b36..65d37f01d98a90 100755 --- a/ci/test/00_setup_env_arm.sh +++ b/ci/test/00_setup_env_arm.sh @@ -7,18 +7,10 @@ export LC_ALL=C.UTF-8 export HOST=arm-linux-gnueabihf -# The host arch is unknown, so we run the tests through qemu. -# If the host is arm and wants to run the tests natively, it can set QEMU_USER_CMD to the empty string. -if [ -z ${QEMU_USER_CMD+x} ]; then export QEMU_USER_CMD="${QEMU_USER_CMD:-"qemu-arm -L /usr/arm-linux-gnueabihf/"}"; fi export DPKG_ADD_ARCH="armhf" export PACKAGES="python3-zmq g++-arm-linux-gnueabihf busybox libc6:armhf libstdc++6:armhf libfontconfig1:armhf libxcb1:armhf" -if [ -n "$QEMU_USER_CMD" ]; then - # Likely cross-compiling, so install the needed gcc and qemu-user - export PACKAGES="$PACKAGES qemu-user" -fi export CONTAINER_NAME=ci_arm_linux -# Use debian to avoid 404 apt errors when cross compiling -export CI_IMAGE_NAME_TAG="debian:bullseye" +export CI_IMAGE_NAME_TAG="docker.io/arm64v8/debian:bookworm" export USE_BUSY_BOX=true export RUN_UNIT_TESTS=true export RUN_FUNCTIONAL_TESTS=false diff --git a/ci/test/00_setup_env_i686_centos.sh b/ci/test/00_setup_env_i686_centos.sh index 606c28e252fbfb..d509c721410188 100755 --- a/ci/test/00_setup_env_i686_centos.sh +++ b/ci/test/00_setup_env_i686_centos.sh @@ -8,8 +8,8 @@ export LC_ALL=C.UTF-8 export HOST=i686-pc-linux-gnu export CONTAINER_NAME=ci_i686_centos -export CI_IMAGE_NAME_TAG="quay.io/centos/centos:stream9" -export CI_BASE_PACKAGES="gcc-c++ glibc-devel.x86_64 libstdc++-devel.x86_64 glibc-devel.i686 libstdc++-devel.i686 ccache libtool make git python3 python3-pip which patch lbzip2 xz procps-ng dash rsync coreutils bison util-linux" +export CI_IMAGE_NAME_TAG="quay.io/centos/amd64:stream9" +export CI_BASE_PACKAGES="gcc-c++ glibc-devel.x86_64 libstdc++-devel.x86_64 glibc-devel.i686 libstdc++-devel.i686 ccache libtool make git python3 python3-pip which patch lbzip2 xz procps-ng dash rsync coreutils bison util-linux e2fsprogs" export PIP_PACKAGES="pyzmq" export GOAL="install" export NO_WERROR=1 # Suppress error: #warning _FORTIFY_SOURCE > 2 is treated like 2 on this platform [-Werror=cpp] diff --git a/ci/test/00_setup_env_i686_multiprocess.sh b/ci/test/00_setup_env_i686_multiprocess.sh index 7911c1912fc05b..b11a387660e44f 100755 --- a/ci/test/00_setup_env_i686_multiprocess.sh +++ b/ci/test/00_setup_env_i686_multiprocess.sh @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 export HOST=i686-pc-linux-gnu export CONTAINER_NAME=ci_i686_multiprocess -export CI_IMAGE_NAME_TAG=ubuntu:20.04 +export CI_IMAGE_NAME_TAG="docker.io/amd64/ubuntu:20.04" export PACKAGES="cmake llvm clang g++-multilib" export DEP_OPTS="DEBUG=1 MULTIPROCESS=1" export GOAL="install" diff --git a/ci/test/00_setup_env_mac.sh b/ci/test/00_setup_env_mac.sh index 65c6119fcdf92e..6fabb4ab96d78a 100755 --- a/ci/test/00_setup_env_mac.sh +++ b/ci/test/00_setup_env_mac.sh @@ -1,15 +1,17 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2021 The Bitcoin Core developers +# Copyright (c) 2019-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 +export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks} + export CONTAINER_NAME=ci_macos_cross -export CI_IMAGE_NAME_TAG=ubuntu:22.04 +export CI_IMAGE_NAME_TAG="docker.io/ubuntu:22.04" export HOST=x86_64-apple-darwin -export PACKAGES="cmake libz-dev libtinfo5 python3-setuptools xorriso" +export PACKAGES="cmake libz-dev python3-setuptools zip" export XCODE_VERSION=12.2 export XCODE_BUILD_ID=12B45b export RUN_UNIT_TESTS=false diff --git a/ci/test/00_setup_env_mac_native_arm64.sh b/ci/test/00_setup_env_mac_native.sh similarity index 81% rename from ci/test/00_setup_env_mac_native_arm64.sh rename to ci/test/00_setup_env_mac_native.sh index eee72db4354c90..c9f65bf397bebc 100755 --- a/ci/test/00_setup_env_mac_native_arm64.sh +++ b/ci/test/00_setup_env_mac_native.sh @@ -1,18 +1,18 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2022 The Bitcoin Core developers +# Copyright (c) 2019-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -export HOST=arm64-apple-darwin +export HOST=x86_64-apple-darwin export PIP_PACKAGES="zmq" export GOAL="install" export BITCOIN_CONFIG="--with-gui --with-miniupnpc --with-natpmp --enable-reduce-exports" export CI_OS_NAME="macos" export NO_DEPENDS=1 export OSX_SDK="" -export CCACHE_MAXSIZE=300M +export CCACHE_MAXSIZE=400M export RUN_FUZZ_TESTS=true export FUZZ_TESTS_CONFIG="--exclude banman" # https://github.com/bitcoin/bitcoin/issues/27924 diff --git a/ci/test/00_setup_env_native_asan.sh b/ci/test/00_setup_env_native_asan.sh index dff86b1ffec6dd..0f69ebdd9c8325 100755 --- a/ci/test/00_setup_env_native_asan.sh +++ b/ci/test/00_setup_env_native_asan.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2022 The Bitcoin Core developers +# Copyright (c) 2019-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -17,9 +17,10 @@ fi export CONTAINER_NAME=ci_native_asan export PACKAGES="systemtap-sdt-dev clang-16 llvm-16 libclang-rt-16-dev python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libsqlite3-dev ${BPFCC_PACKAGE}" -export CI_IMAGE_NAME_TAG=ubuntu:23.04 # Version 23.04 will reach EOL in Jan 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version). +export CI_IMAGE_NAME_TAG="docker.io/ubuntu:23.04" # This version will reach EOL in Jan 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version). export NO_DEPENDS=1 export GOAL="install" export BITCOIN_CONFIG="--enable-c++20 --enable-usdt --enable-zmq --with-incompatible-bdb --with-gui=qt5 \ CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' \ ---with-sanitizers=address,float-divide-by-zero,integer,undefined CC=clang-16 CXX=clang++-16" +--with-sanitizers=address,float-divide-by-zero,integer,undefined \ +CC='clang-16 -ftrivial-auto-var-init=pattern' CXX='clang++-16 -ftrivial-auto-var-init=pattern'" diff --git a/ci/test/00_setup_env_native_fuzz.sh b/ci/test/00_setup_env_native_fuzz.sh index bfd51be6d00eca..122f044b583f18 100755 --- a/ci/test/00_setup_env_native_fuzz.sh +++ b/ci/test/00_setup_env_native_fuzz.sh @@ -1,14 +1,14 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2022 The Bitcoin Core developers +# Copyright (c) 2019-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -export CI_IMAGE_NAME_TAG="ubuntu:23.04" # Version 23.04 will reach EOL in Jan 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version). +export CI_IMAGE_NAME_TAG="docker.io/ubuntu:23.10" # This version will reach EOL in Jul 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version). export CONTAINER_NAME=ci_native_fuzz -export PACKAGES="clang-16 llvm-16 libclang-rt-16-dev libevent-dev libboost-dev libsqlite3-dev" +export PACKAGES="clang-17 llvm-17 libclang-rt-17-dev libevent-dev libboost-dev libsqlite3-dev" export NO_DEPENDS=1 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false @@ -16,5 +16,5 @@ export RUN_FUZZ_TESTS=true export GOAL="install" export CI_CONTAINER_CAP="--cap-add SYS_PTRACE" # If run with (ASan + LSan), the container needs access to ptrace (https://github.com/google/sanitizers/issues/764) export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined,float-divide-by-zero,integer \ -CC='clang-16 -ftrivial-auto-var-init=pattern' CXX='clang++-16 -ftrivial-auto-var-init=pattern'" +CC='clang-17 -ftrivial-auto-var-init=pattern' CXX='clang++-17 -ftrivial-auto-var-init=pattern'" export CCACHE_MAXSIZE=200M diff --git a/ci/test/00_setup_env_native_fuzz_with_msan.sh b/ci/test/00_setup_env_native_fuzz_with_msan.sh index 89006bf95ef1c1..6055e0ad198af2 100755 --- a/ci/test/00_setup_env_native_fuzz_with_msan.sh +++ b/ci/test/00_setup_env_native_fuzz_with_msan.sh @@ -1,13 +1,13 @@ #!/usr/bin/env bash # -# Copyright (c) 2020-2022 The Bitcoin Core developers +# Copyright (c) 2020-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -export CI_IMAGE_NAME_TAG="ubuntu:22.04" -LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/cxx_build/" +export CI_IMAGE_NAME_TAG="docker.io/ubuntu:22.04" +LIBCXX_DIR="/msan/cxx_build/" export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls" LIBCXX_FLAGS="-nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${LIBCXX_DIR}lib -Wl,-rpath,${LIBCXX_DIR}lib -lc++ -lc++abi -lpthread -Wno-unused-command-line-argument" export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" diff --git a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh index 30f512c38ac7a5..1f60c4680368c6 100755 --- a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh +++ b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh @@ -1,12 +1,12 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2022 The Bitcoin Core developers +# Copyright (c) 2019-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -export CI_IMAGE_NAME_TAG="debian:bookworm" +export CI_IMAGE_NAME_TAG="docker.io/debian:bookworm" export CONTAINER_NAME=ci_native_fuzz_valgrind export PACKAGES="clang llvm libclang-rt-dev libevent-dev libboost-dev libsqlite3-dev valgrind" export NO_DEPENDS=1 diff --git a/ci/test/00_setup_env_native_msan.sh b/ci/test/00_setup_env_native_msan.sh index a96ed30dd3c34d..d73a7db0e6ec30 100755 --- a/ci/test/00_setup_env_native_msan.sh +++ b/ci/test/00_setup_env_native_msan.sh @@ -1,13 +1,13 @@ #!/usr/bin/env bash # -# Copyright (c) 2020-2023 The Bitcoin Core developers +# Copyright (c) 2020-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -export CI_IMAGE_NAME_TAG="ubuntu:22.04" -LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/cxx_build/" +export CI_IMAGE_NAME_TAG="docker.io/ubuntu:22.04" +LIBCXX_DIR="/msan/cxx_build/" export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls" LIBCXX_FLAGS="-nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${LIBCXX_DIR}lib -Wl,-rpath,${LIBCXX_DIR}lib -lc++ -lc++abi -lpthread -Wno-unused-command-line-argument" export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" diff --git a/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh b/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh index 8e3c935c6fdd52..454a4b8dff2e3c 100755 --- a/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh +++ b/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh @@ -1,13 +1,13 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2022 The Bitcoin Core developers +# Copyright (c) 2019-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_nowallet_libbitcoinkernel -export CI_IMAGE_NAME_TAG="ubuntu:20.04" +export CI_IMAGE_NAME_TAG="docker.io/ubuntu:20.04" # Use minimum supported python3.8 and clang-10, see doc/dependencies.md export PACKAGES="python3-zmq clang-10 llvm-10 libc++abi-10-dev libc++-10-dev" export DEP_OPTS="NO_WALLET=1 CC=clang-10 CXX='clang++-10 -stdlib=libc++'" diff --git a/ci/test/00_setup_env_native_qt5.sh b/ci/test/00_setup_env_native_qt5.sh index 3a1d7808f15239..0fad95e2cb884d 100755 --- a/ci/test/00_setup_env_native_qt5.sh +++ b/ci/test/00_setup_env_native_qt5.sh @@ -1,13 +1,13 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2022 The Bitcoin Core developers +# Copyright (c) 2019-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_qt5 -export CI_IMAGE_NAME_TAG="ubuntu:20.04" +export CI_IMAGE_NAME_TAG="docker.io/ubuntu:20.04" # Use minimum supported python3.8 and gcc-9, see doc/dependencies.md export PACKAGES="gcc-9 g++-9 python3-zmq qtbase5-dev qttools5-dev-tools libdbus-1-dev libharfbuzz-dev" export DEP_OPTS="NO_QT=1 NO_UPNP=1 NO_NATPMP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1 CC=gcc-9 CXX=g++-9" diff --git a/ci/test/00_setup_env_native_tidy.sh b/ci/test/00_setup_env_native_tidy.sh index c86722c551d828..456166d3be0fc5 100755 --- a/ci/test/00_setup_env_native_tidy.sh +++ b/ci/test/00_setup_env_native_tidy.sh @@ -1,12 +1,12 @@ #!/usr/bin/env bash # -# Copyright (c) 2023 The Bitcoin Core developers +# Copyright (c) 2023-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -export CI_IMAGE_NAME_TAG="ubuntu:lunar" # Version 23.04 will reach EOL in Jan 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version). +export CI_IMAGE_NAME_TAG="docker.io/ubuntu:23.10" # This version will reach EOL in Jul 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version). export CONTAINER_NAME=ci_native_tidy export PACKAGES="clang-16 libclang-16-dev llvm-16-dev libomp-16-dev clang-tidy-16 jq bear cmake libevent-dev libboost-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev systemtap-sdt-dev libqt5gui5 libqt5core5a libqt5dbus5 qttools5-dev qttools5-dev-tools libqrencode-dev libsqlite3-dev libdb++-dev" export NO_DEPENDS=1 diff --git a/ci/test/00_setup_env_native_tsan.sh b/ci/test/00_setup_env_native_tsan.sh index 8ebb1fa563ccd1..67c29d4bc85bb9 100755 --- a/ci/test/00_setup_env_native_tsan.sh +++ b/ci/test/00_setup_env_native_tsan.sh @@ -1,14 +1,14 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2023 The Bitcoin Core developers +# Copyright (c) 2019-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_tsan -export CI_IMAGE_NAME_TAG=ubuntu:23.04 # Version 23.04 will reach EOL in Jan 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version). -export PACKAGES="clang-16 llvm-16 libclang-rt-16-dev libc++abi-16-dev libc++-16-dev python3-zmq" -export DEP_OPTS="CC=clang-16 CXX='clang++-16 -stdlib=libc++'" +export CI_IMAGE_NAME_TAG="docker.io/ubuntu:23.10" # This version will reach EOL in Jul 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version). +export PACKAGES="clang-17 llvm-17 libclang-rt-17-dev libc++abi-17-dev libc++-17-dev python3-zmq" +export DEP_OPTS="CC=clang-17 CXX='clang++-17 -stdlib=libc++'" export GOAL="install" export BITCOIN_CONFIG="--enable-zmq CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER -DDEBUG_LOCKCONTENTION' CXXFLAGS='-g' --with-sanitizers=thread" diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh index fa2ae81fddf9cf..daa1a0cdb311d3 100755 --- a/ci/test/00_setup_env_native_valgrind.sh +++ b/ci/test/00_setup_env_native_valgrind.sh @@ -1,12 +1,12 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2022 The Bitcoin Core developers +# Copyright (c) 2019-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -export CI_IMAGE_NAME_TAG="debian:bookworm" +export CI_IMAGE_NAME_TAG="docker.io/debian:bookworm" export CONTAINER_NAME=ci_native_valgrind export PACKAGES="valgrind clang llvm libclang-rt-dev python3-zmq libevent-dev libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libsqlite3-dev" export USE_VALGRIND=1 diff --git a/ci/test/00_setup_env_s390x.sh b/ci/test/00_setup_env_s390x.sh index 523e81c94a2da6..ca84ecce5153ca 100755 --- a/ci/test/00_setup_env_s390x.sh +++ b/ci/test/00_setup_env_s390x.sh @@ -1,25 +1,15 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2022 The Bitcoin Core developers +# Copyright (c) 2019-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 export HOST=s390x-linux-gnu -# The host arch is unknown, so we run the tests through qemu. -# If the host is s390x and wants to run the tests natively, it can set QEMU_USER_CMD to the empty string. -if [ -z ${QEMU_USER_CMD+x} ]; then export QEMU_USER_CMD="${QEMU_USER_CMD:-"qemu-s390x"}"; fi export PACKAGES="python3-zmq" -if [ -n "$QEMU_USER_CMD" ]; then - # Likely cross-compiling, so install the needed gcc and qemu-user - export DPKG_ADD_ARCH="s390x" - export PACKAGES="$PACKAGES g++-s390x-linux-gnu qemu-user libc6:s390x libstdc++6:s390x" -fi -# Use debian to avoid 404 apt errors export CONTAINER_NAME=ci_s390x -export CI_IMAGE_NAME_TAG="debian:bookworm" -export TEST_RUNNER_ENV="LC_ALL=C" +export CI_IMAGE_NAME_TAG="docker.io/s390x/debian:bookworm" export TEST_RUNNER_EXTRA="--exclude feature_init,rpc_bind,feature_bind_extra" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 export RUN_FUNCTIONAL_TESTS=true export GOAL="install" diff --git a/ci/test/00_setup_env_win64.sh b/ci/test/00_setup_env_win64.sh index e42828ac1fa3dd..ebd4487c527310 100755 --- a/ci/test/00_setup_env_win64.sh +++ b/ci/test/00_setup_env_win64.sh @@ -1,13 +1,13 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2022 The Bitcoin Core developers +# Copyright (c) 2019-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_win64 -export CI_IMAGE_NAME_TAG=ubuntu:22.04 # Check that Jammy can cross-compile to win64 +export CI_IMAGE_NAME_TAG="docker.io/amd64/ubuntu:22.04" # Check that Jammy can cross-compile to win64 export HOST=x86_64-w64-mingw32 export DPKG_ADD_ARCH="i386" export PACKAGES="nsis g++-mingw-w64-x86-64-posix wine-binfmt wine64 wine32 file" diff --git a/ci/test/01_base_install.sh b/ci/test/01_base_install.sh index 76cde42161a132..424dca52dc6c08 100755 --- a/ci/test/01_base_install.sh +++ b/ci/test/01_base_install.sh @@ -22,7 +22,7 @@ fi if [[ $CI_IMAGE_NAME_TAG == *centos* ]]; then bash -c "dnf -y install epel-release" bash -c "dnf -y --allowerasing install $CI_BASE_PACKAGES $PACKAGES" -elif [ "$CI_USE_APT_INSTALL" != "no" ]; then +elif [ "$CI_OS_NAME" != "macos" ]; then if [[ -n "${APPEND_APT_SOURCES_LIST}" ]]; then echo "${APPEND_APT_SOURCES_LIST}" >> /etc/apt/sources.list fi @@ -42,39 +42,40 @@ if [ -n "$PIP_PACKAGES" ]; then fi if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then - git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-16.0.6 "${BASE_SCRATCH_DIR}"/msan/llvm-project - - cmake -G Ninja -B "${BASE_SCRATCH_DIR}"/msan/clang_build/ -DLLVM_ENABLE_PROJECTS="clang" \ - -DCMAKE_BUILD_TYPE=Release \ - -DLLVM_TARGETS_TO_BUILD=Native \ - -DLLVM_ENABLE_RUNTIMES="compiler-rt;libcxx;libcxxabi;libunwind" \ - -S "${BASE_SCRATCH_DIR}"/msan/llvm-project/llvm - - ninja -C "${BASE_SCRATCH_DIR}"/msan/clang_build/ "$MAKEJOBS" - ninja -C "${BASE_SCRATCH_DIR}"/msan/clang_build/ install-runtimes - - update-alternatives --install /usr/bin/clang++ clang++ "${BASE_SCRATCH_DIR}"/msan/clang_build/bin/clang++ 100 - update-alternatives --install /usr/bin/clang clang "${BASE_SCRATCH_DIR}"/msan/clang_build/bin/clang 100 - update-alternatives --install /usr/bin/llvm-symbolizer llvm-symbolizer "${BASE_SCRATCH_DIR}"/msan/clang_build/bin/llvm-symbolizer 100 - - cmake -G Ninja -B "${BASE_SCRATCH_DIR}"/msan/cxx_build/ -DLLVM_ENABLE_RUNTIMES='libcxx;libcxxabi' \ - -DCMAKE_BUILD_TYPE=Release \ - -DLLVM_USE_SANITIZER=MemoryWithOrigins \ - -DCMAKE_C_COMPILER=clang \ - -DCMAKE_CXX_COMPILER=clang++ \ - -DLLVM_TARGETS_TO_BUILD=Native \ - -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF \ - -DLIBCXX_ENABLE_DEBUG_MODE=ON \ - -DLIBCXX_ENABLE_ASSERTIONS=ON \ - -S "${BASE_SCRATCH_DIR}"/msan/llvm-project/runtimes - - ninja -C "${BASE_SCRATCH_DIR}"/msan/cxx_build/ "$MAKEJOBS" + git clone --depth=1 https://github.com/llvm/llvm-project -b "llvmorg-17.0.0-rc4" /msan/llvm-project + + cmake -G Ninja -B /msan/clang_build/ \ + -DLLVM_ENABLE_PROJECTS="clang" \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_TARGETS_TO_BUILD=Native \ + -DLLVM_ENABLE_RUNTIMES="compiler-rt;libcxx;libcxxabi;libunwind" \ + -S /msan/llvm-project/llvm + + ninja -C /msan/clang_build/ "-j$( nproc )" # Use nproc, because MAKEJOBS is the default in docker image builds + ninja -C /msan/clang_build/ install-runtimes + + update-alternatives --install /usr/bin/clang++ clang++ /msan/clang_build/bin/clang++ 100 + update-alternatives --install /usr/bin/clang clang /msan/clang_build/bin/clang 100 + update-alternatives --install /usr/bin/llvm-symbolizer llvm-symbolizer /msan/clang_build/bin/llvm-symbolizer 100 + + cmake -G Ninja -B /msan/cxx_build/ \ + -DLLVM_ENABLE_RUNTIMES='libcxx;libcxxabi' \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_USE_SANITIZER=MemoryWithOrigins \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DLLVM_TARGETS_TO_BUILD=Native \ + -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF \ + -DLIBCXX_HARDENING_MODE=debug \ + -S /msan/llvm-project/runtimes + + ninja -C /msan/cxx_build/ "-j$( nproc )" # Use nproc, because MAKEJOBS is the default in docker image builds fi if [[ "${RUN_TIDY}" == "true" ]]; then - git clone --depth=1 https://github.com/include-what-you-use/include-what-you-use -b clang_16 "${DIR_IWYU}"/include-what-you-use - cmake -B "${DIR_IWYU}"/build/ -G 'Unix Makefiles' -DCMAKE_PREFIX_PATH=/usr/lib/llvm-16 -S "${DIR_IWYU}"/include-what-you-use - make -C "${DIR_IWYU}"/build/ install "$MAKEJOBS" + git clone --depth=1 https://github.com/include-what-you-use/include-what-you-use -b clang_16 /include-what-you-use + cmake -B /iwyu-build/ -G 'Unix Makefiles' -DCMAKE_PREFIX_PATH=/usr/lib/llvm-16 -S /include-what-you-use + make -C /iwyu-build/ install "-j$( nproc )" # Use nproc, because MAKEJOBS is the default in docker image builds fi mkdir -p "${DEPENDS_DIR}/SDKs" "${DEPENDS_DIR}/sdk-sources" @@ -85,7 +86,7 @@ if [ -n "$XCODE_VERSION" ] && [ ! -d "${DEPENDS_DIR}/SDKs/${OSX_SDK_BASENAME}" ] OSX_SDK_FILENAME="${OSX_SDK_BASENAME}.tar.gz" OSX_SDK_PATH="${DEPENDS_DIR}/sdk-sources/${OSX_SDK_FILENAME}" if [ ! -f "$OSX_SDK_PATH" ]; then - curl --location --fail "${SDK_URL}/${OSX_SDK_FILENAME}" -o "$OSX_SDK_PATH" + ${CI_RETRY_EXE} curl --location --fail "${SDK_URL}/${OSX_SDK_FILENAME}" -o "$OSX_SDK_PATH" fi tar -C "${DEPENDS_DIR}/SDKs" -xf "$OSX_SDK_PATH" fi @@ -93,7 +94,7 @@ fi if [ -n "$ANDROID_HOME" ] && [ ! -d "$ANDROID_HOME" ]; then ANDROID_TOOLS_PATH=${DEPENDS_DIR}/sdk-sources/android-tools.zip if [ ! -f "$ANDROID_TOOLS_PATH" ]; then - curl --location --fail "${ANDROID_TOOLS_URL}" -o "$ANDROID_TOOLS_PATH" + ${CI_RETRY_EXE} curl --location --fail "${ANDROID_TOOLS_URL}" -o "$ANDROID_TOOLS_PATH" fi mkdir -p "$ANDROID_HOME" unzip -o "$ANDROID_TOOLS_PATH" -d "$ANDROID_HOME" diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index 205c79328a501f..01faf9fff96be9 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -1,24 +1,11 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2022 The Bitcoin Core developers +# Copyright (c) 2018-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -if [[ $QEMU_USER_CMD == qemu-s390* ]]; then - export LC_ALL=C -fi - -# Create folders that are mounted into the docker -mkdir -p "${CCACHE_DIR}" -mkdir -p "${PREVIOUS_RELEASES_DIR}" - -export ASAN_OPTIONS="detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1" -export LSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/lsan" -export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/tsan:halt_on_error=1:log_path=${BASE_SCRATCH_DIR}/sanitizer-output/tsan" -export UBSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1" - if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then # Export all env vars to avoid missing some. # Though, exclude those with newlines to avoid parsing problems. @@ -27,11 +14,11 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then docker run --rm "${CI_IMAGE_NAME_TAG}" bash -c "env | grep --extended-regexp '^(HOME|PATH|USER)='" | tee --append /tmp/env echo "Creating $CI_IMAGE_NAME_TAG container to run in" DOCKER_BUILDKIT=1 docker build \ - --file "${BASE_ROOT_DIR}/ci/test_imagefile" \ + --file "${BASE_READ_ONLY_DIR}/ci/test_imagefile" \ --build-arg "CI_IMAGE_NAME_TAG=${CI_IMAGE_NAME_TAG}" \ --build-arg "FILE_ENV=${FILE_ENV}" \ --tag="${CONTAINER_NAME}" \ - "${BASE_ROOT_DIR}" + "${BASE_READ_ONLY_DIR}" docker volume create "${CONTAINER_NAME}_ccache" || true docker volume create "${CONTAINER_NAME}_depends" || true docker volume create "${CONTAINER_NAME}_previous_releases" || true @@ -44,18 +31,23 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then fi # shellcheck disable=SC2086 - CI_CONTAINER_ID=$(docker run $CI_CONTAINER_CAP --rm --interactive --detach --tty \ - --mount type=bind,src=$BASE_ROOT_DIR,dst=/ro_base,readonly \ + + + CI_CONTAINER_ID=$(docker run --cap-add LINUX_IMMUTABLE $CI_CONTAINER_CAP --rm --interactive --detach --tty \ + --mount "type=bind,src=$BASE_READ_ONLY_DIR,dst=$BASE_READ_ONLY_DIR,readonly" \ --mount "type=volume,src=${CONTAINER_NAME}_ccache,dst=$CCACHE_DIR" \ --mount "type=volume,src=${CONTAINER_NAME}_depends,dst=$DEPENDS_DIR" \ --mount "type=volume,src=${CONTAINER_NAME}_previous_releases,dst=$PREVIOUS_RELEASES_DIR" \ --env-file /tmp/env \ - --name $CONTAINER_NAME \ - $CONTAINER_NAME) + --name "$CONTAINER_NAME" \ + "$CONTAINER_NAME") export CI_CONTAINER_ID export CI_EXEC_CMD_PREFIX="docker exec ${CI_CONTAINER_ID}" else echo "Running on host system without docker wrapper" + echo "Create missing folders" + mkdir -p "${CCACHE_DIR}" + mkdir -p "${PREVIOUS_RELEASES_DIR}" fi CI_EXEC () { @@ -63,9 +55,10 @@ CI_EXEC () { } export -f CI_EXEC -CI_EXEC rsync --archive --stats --human-readable /ci_base_install/ "${BASE_ROOT_DIR}" || echo "/ci_base_install/ missing" +# Normalize all folders to BASE_ROOT_DIR +CI_EXEC rsync --archive --stats --human-readable "${BASE_READ_ONLY_DIR}/" "${BASE_ROOT_DIR}" || echo "Nothing to copy from ${BASE_READ_ONLY_DIR}/" CI_EXEC "${BASE_ROOT_DIR}/ci/test/01_base_install.sh" -CI_EXEC rsync --archive --stats --human-readable /ro_base/ "${BASE_ROOT_DIR}" || echo "Nothing to copy from ro_base" + # Fixes permission issues when there is a container UID/GID mismatch with the owner # of the git source code directory. CI_EXEC git config --global --add safe.directory \"*\" diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index 6e921d3377d0f4..f5ba48e4559616 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -8,6 +8,11 @@ export LC_ALL=C.UTF-8 set -ex +export ASAN_OPTIONS="detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1" +export LSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/lsan" +export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/tsan:halt_on_error=1:log_path=${BASE_SCRATCH_DIR}/sanitizer-output/tsan" +export UBSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1" + if [ "$CI_OS_NAME" == "macos" ]; then top -l 1 -s 0 | awk ' /PhysMem/ {print}' echo "Number of CPUs: $(sysctl -n hw.logicalcpu)" @@ -19,6 +24,34 @@ fi echo "Free disk space:" df -h +# What host to compile for. See also ./depends/README.md +# Tests that need cross-compilation export the appropriate HOST. +# Tests that run natively guess the host +export HOST=${HOST:-$("$BASE_ROOT_DIR/depends/config.guess")} + +( + # compact->outputs[i].file_size is uninitialized memory, so reading it is UB. + # The statistic bytes_written is only used for logging, which is disabled in + # CI, so as a temporary minimal fix to work around UB and CI failures, leave + # bytes_written unmodified. + # See https://github.com/bitcoin/bitcoin/pull/28359#issuecomment-1698694748 + echo 'diff --git a/src/leveldb/db/db_impl.cc b/src/leveldb/db/db_impl.cc +index 65e31724bc..f61b471953 100644 +--- a/src/leveldb/db/db_impl.cc ++++ b/src/leveldb/db/db_impl.cc +@@ -1028,9 +1028,6 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) { + stats.bytes_read += compact->compaction->input(which, i)->file_size; + } + } +- for (size_t i = 0; i < compact->outputs.size(); i++) { +- stats.bytes_written += compact->outputs[i].file_size; +- } + + mutex_.Lock(); + stats_[compact->compaction->level() + 1].Add(stats);' | patch -p1 + git diff +) + if [ "$RUN_FUZZ_TESTS" = "true" ]; then export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/ if [ ! -d "$DIR_FUZZ_IN" ]; then @@ -33,7 +66,7 @@ elif [ "$RUN_UNIT_TESTS" = "true" ] || [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ] export DIR_UNIT_TEST_DATA=${DIR_QA_ASSETS}/unit_test_data/ if [ ! -d "$DIR_UNIT_TEST_DATA" ]; then mkdir -p "$DIR_UNIT_TEST_DATA" - curl --location --fail https://github.com/bitcoin-core/qa-assets/raw/main/unit_test_data/script_assets_test.json -o "${DIR_UNIT_TEST_DATA}/script_assets_test.json" + ${CI_RETRY_EXE} curl --location --fail https://github.com/bitcoin-core/qa-assets/raw/main/unit_test_data/script_assets_test.json -o "${DIR_UNIT_TEST_DATA}/script_assets_test.json" fi fi @@ -124,13 +157,6 @@ if [[ $HOST = *-mingw32 ]]; then "${BASE_ROOT_DIR}/ci/test/wrap-wine.sh" fi -if [ -n "$QEMU_USER_CMD" ]; then - # Generate all binaries, so that they can be wrapped - make "$MAKEJOBS" -C src/secp256k1 VERBOSE=1 - make "$MAKEJOBS" -C src minisketch/test VERBOSE=1 - "${BASE_ROOT_DIR}/ci/test/wrap-qemu.sh" -fi - if [ -n "$USE_VALGRIND" ]; then "${BASE_ROOT_DIR}/ci/test/wrap-valgrind.sh" fi @@ -148,9 +174,13 @@ if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then fi if [ "${RUN_TIDY}" = "true" ]; then + cmake -B /tidy-build -DLLVM_DIR=/usr/lib/llvm-16/cmake -DCMAKE_BUILD_TYPE=Release -S "${BASE_ROOT_DIR}"/contrib/devtools/bitcoin-tidy + cmake --build /tidy-build "$MAKEJOBS" + cmake --build /tidy-build --target bitcoin-tidy-tests "$MAKEJOBS" + set -eo pipefail cd "${BASE_BUILD_DIR}/bitcoin-$HOST/src/" - ( run-clang-tidy-16 -quiet "${MAKEJOBS}" ) | grep -C5 "error" + ( run-clang-tidy-16 -quiet -load="/tidy-build/libbitcoin-tidy.so" "${MAKEJOBS}" ) | grep -C5 "error" # Filter out files by regex here, because regex may not be # accepted in src/.bear-tidy-config # Filter out: @@ -158,13 +188,13 @@ if [ "${RUN_TIDY}" = "true" ]; then jq 'map(select(.file | test("src/qt/qrc_.*\\.cpp$|/moc_.*\\.cpp$") | not))' ../compile_commands.json > tmp.json mv tmp.json ../compile_commands.json cd "${BASE_BUILD_DIR}/bitcoin-$HOST/" - python3 "${DIR_IWYU}/include-what-you-use/iwyu_tool.py" \ + python3 "/include-what-you-use/iwyu_tool.py" \ -p . "${MAKEJOBS}" \ -- -Xiwyu --cxx17ns -Xiwyu --mapping_file="${BASE_BUILD_DIR}/bitcoin-$HOST/contrib/devtools/iwyu/bitcoin.core.imp" \ -Xiwyu --max_line_length=160 \ 2>&1 | tee /tmp/iwyu_ci.out cd "${BASE_ROOT_DIR}/src" - python3 "${DIR_IWYU}/include-what-you-use/fix_includes.py" --nosafe_headers < /tmp/iwyu_ci.out + python3 "/include-what-you-use/fix_includes.py" --nosafe_headers < /tmp/iwyu_ci.out git --no-pager diff fi diff --git a/ci/test/wrap-qemu.sh b/ci/test/wrap-qemu.sh deleted file mode 100755 index e028ede37888ff..00000000000000 --- a/ci/test/wrap-qemu.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2022 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C.UTF-8 - -for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/minisketch/test{,-verify},src/univalue/{test_json,unitester,object}}; do - # shellcheck disable=SC2044 - for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name "$(basename "$b_name")"); do - echo "Wrap $b ..." - mv "$b" "${b}_orig" - echo '#!/usr/bin/env bash' > "$b" - echo "$QEMU_USER_CMD \"${b}_orig\" \"\$@\"" >> "$b" - chmod +x "$b" - done -done diff --git a/ci/test/wrapped-cl.bat b/ci/test/wrapped-cl.bat deleted file mode 100644 index fc2a604c580bed..00000000000000 --- a/ci/test/wrapped-cl.bat +++ /dev/null @@ -1 +0,0 @@ -ccache cl %* diff --git a/ci/test_imagefile b/ci/test_imagefile index 8dfb5916b1b9ef..f8b5eea1c88abc 100644 --- a/ci/test_imagefile +++ b/ci/test_imagefile @@ -11,6 +11,6 @@ ARG FILE_ENV ENV FILE_ENV=${FILE_ENV} COPY ./ci/retry/retry /usr/bin/retry -COPY ./ci/test/00_setup_env.sh ./${FILE_ENV} ./ci/test/01_base_install.sh /ci_base_install/ci/test/ +COPY ./ci/test/00_setup_env.sh ./${FILE_ENV} ./ci/test/01_base_install.sh /ci_container_base/ci/test/ -RUN ["bash", "-c", "cd /ci_base_install/ && set -o errexit && source ./ci/test/00_setup_env.sh && ./ci/test/01_base_install.sh"] +RUN ["bash", "-c", "cd /ci_container_base/ && set -o errexit && source ./ci/test/00_setup_env.sh && ./ci/test/01_base_install.sh"] diff --git a/configure.ac b/configure.ac index f0ebc50623331b..de84e6b30a09dd 100644 --- a/configure.ac +++ b/configure.ac @@ -1,15 +1,24 @@ -AC_PREREQ([2.69]) -define(_CLIENT_VERSION_MAJOR, 25) -define(_CLIENT_VERSION_MINOR, 99) -define(_CLIENT_VERSION_BUILD, 0) -define(_CLIENT_VERSION_RC, 0) -define(_CLIENT_VERSION_IS_RELEASE, false) -define(_COPYRIGHT_YEAR, 2023) -define(_COPYRIGHT_HOLDERS,[The %s developers]) -define(_COPYRIGHT_HOLDERS_SUBSTITUTION,[[Bitcoin Core]]) -AC_INIT([Bitcoin Core],m4_join([.], _CLIENT_VERSION_MAJOR, _CLIENT_VERSION_MINOR, _CLIENT_VERSION_BUILD)m4_if(_CLIENT_VERSION_RC, [0], [], [rc]_CLIENT_VERSION_RC),[https://github.com/bitcoin/bitcoin/issues],[bitcoin],[https://bitcoincore.org/]) -AC_CONFIG_SRCDIR([src/validation.cpp]) -AC_CONFIG_HEADERS([src/config/bitcoin-config.h]) +AC_PREREQ([2.60]) + +# The package (a.k.a. release) version is based on semantic versioning 2.0.0 of +# the API. All changes in experimental modules are treated as +# backwards-compatible and therefore at most increase the minor version. +define(_PKG_VERSION_MAJOR, 0) +define(_PKG_VERSION_MINOR, 4) +define(_PKG_VERSION_PATCH, 0) +define(_PKG_VERSION_IS_RELEASE, true) + +# The library version is based on libtool versioning of the ABI. The set of +# rules for updating the version can be found here: +# https://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html +# All changes in experimental modules are treated as if they don't affect the +# interface and therefore only increase the revision. +define(_LIB_VERSION_CURRENT, 3) +define(_LIB_VERSION_REVISION, 0) +define(_LIB_VERSION_AGE, 1) + +AC_INIT([libsecp256k1],m4_join([.], _PKG_VERSION_MAJOR, _PKG_VERSION_MINOR, _PKG_VERSION_PATCH)m4_if(_PKG_VERSION_IS_RELEASE, [true], [], [-dev]),[https://github.com/bitcoin-core/secp256k1/issues],[libsecp256k1],[https://github.com/bitcoin-core/secp256k1]) + AC_CONFIG_AUX_DIR([build-aux]) AC_CONFIG_MACRO_DIR([build-aux/m4]) @@ -725,6 +734,10 @@ case $host in dnl We require Windows 7 (NT 6.1) or later AX_CHECK_LINK_FLAG([-Wl,--major-subsystem-version -Wl,6 -Wl,--minor-subsystem-version -Wl,1], [CORE_LDFLAGS="$CORE_LDFLAGS -Wl,--major-subsystem-version -Wl,6 -Wl,--minor-subsystem-version -Wl,1"], [], [$LDFLAG_WERROR]) + + dnl Avoid the use of aligned vector instructions when building for Windows. + dnl See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=54412. + AX_CHECK_COMPILE_FLAG([-Wa,-muse-unaligned-vector-move], [CORE_CXXFLAGS="$CORE_CXXFLAGS -Wa,-muse-unaligned-vector-move"], [], [$CXXFLAG_WERROR]) ;; *darwin*) TARGET_OS=darwin @@ -795,7 +808,7 @@ case $host in AC_PATH_TOOL([DSYMUTIL], [dsymutil], [dsymutil]) AC_PATH_TOOL([INSTALL_NAME_TOOL], [install_name_tool], [install_name_tool]) AC_PATH_TOOL([OTOOL], [otool], [otool]) - AC_PATH_PROGS([XORRISOFS], [xorrisofs], [xorrisofs]) + AC_PATH_PROG([ZIP], [zip], [zip]) dnl libtool will try to strip the static lib, which is a problem for dnl cross-builds because strip attempts to call a hard-coded ld, @@ -937,9 +950,8 @@ if test "$TARGET_OS" != "windows"; then AX_CHECK_COMPILE_FLAG([-fPIC], [PIC_FLAGS="-fPIC"]) fi -dnl Versions of gcc prior to 12.1 (commit -dnl https://github.com/gcc-mirror/gcc/commit/551aa75778a4c5165d9533cd447c8fc822f583e1) -dnl are subject to a bug, see the gccbug_90348 test case and +dnl Currently all versions of gcc are subject to a class of bugs, see the +dnl gccbug_90348 test case (only reproduces on GCC 11 and earlier) and the related bugs of dnl https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90348. To work around that, set dnl -fstack-reuse=none for all gcc builds. (Only gcc understands this flag) AX_CHECK_COMPILE_FLAG([-fstack-reuse=none], [HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fstack-reuse=none"]) @@ -1327,12 +1339,74 @@ else dnl sets $bitcoin_enable_qt, $bitcoin_enable_qt_test, $bitcoin_enable_qt_dbus BITCOIN_QT_CONFIGURE([5.11.3]) - dnl Keep a copy of the original $QT_INCLUDES and use it when invoking qt's moc - QT_INCLUDES_UNSUPPRESSED=$QT_INCLUDES - if test "$suppress_external_warnings" != "no" ; then - QT_INCLUDES=SUPPRESS_WARNINGS($QT_INCLUDES) - QT_DBUS_INCLUDES=SUPPRESS_WARNINGS($QT_DBUS_INCLUDES) - QT_TEST_INCLUDES=SUPPRESS_WARNINGS($QT_TEST_INCLUDES) +# Set ecmult gen precision +if test x"$req_ecmult_gen_precision" = x"auto"; then + set_ecmult_gen_precision=4 +else + set_ecmult_gen_precision=$req_ecmult_gen_precision +fi + +case $set_ecmult_gen_precision in +2|4|8) + SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES -DECMULT_GEN_PREC_BITS=$set_ecmult_gen_precision" + ;; +*) + AC_MSG_ERROR(['ecmult gen precision not 2, 4, 8 or "auto"']) + ;; +esac + +if test x"$enable_valgrind" = x"yes"; then + SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES $VALGRIND_CPPFLAGS -DVALGRIND" +fi + +# Add -Werror and similar flags passed from the outside (for testing, e.g., in CI). +# We don't want to set the user variable CFLAGS in CI because this would disable +# autoconf's logic for setting default CFLAGS, which we would like to test in CI. +SECP_CFLAGS="$SECP_CFLAGS $WERROR_CFLAGS" + +### +### Handle module options +### + +if test x"$enable_module_ecdh" = x"yes"; then + SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES -DENABLE_MODULE_ECDH=1" +fi + +if test x"$enable_module_recovery" = x"yes"; then + SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES -DENABLE_MODULE_RECOVERY=1" +fi + +if test x"$enable_module_schnorrsig" = x"yes"; then + SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES -DENABLE_MODULE_SCHNORRSIG=1" + enable_module_extrakeys=yes +fi + +if test x"$enable_module_ellswift" = x"yes"; then + SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES -DENABLE_MODULE_ELLSWIFT=1" +fi + +# Test if extrakeys is set after the schnorrsig module to allow the schnorrsig +# module to set enable_module_extrakeys=yes +if test x"$enable_module_extrakeys" = x"yes"; then + SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES -DENABLE_MODULE_EXTRAKEYS=1" +fi + +if test x"$enable_external_default_callbacks" = x"yes"; then + SECP_CONFIG_DEFINES="$SECP_CONFIG_DEFINES -DUSE_EXTERNAL_DEFAULT_CALLBACKS=1" +fi + +### +### Check for --enable-experimental if necessary +### + +if test x"$enable_experimental" = x"yes"; then + AC_MSG_NOTICE([******]) + AC_MSG_NOTICE([WARNING: experimental build]) + AC_MSG_NOTICE([Experimental features do not have stable APIs or properties, and may not be safe for production use.]) + AC_MSG_NOTICE([******]) +else + if test x"$set_asm" = x"arm32"; then + AC_MSG_ERROR([ARM32 assembly optimization is experimental. Use --enable-experimental to allow.]) fi fi @@ -1933,7 +2007,6 @@ AC_CONFIG_FILES([contrib/devtools/split-debug.sh],[chmod +x contrib/devtools/spl AM_COND_IF([HAVE_DOXYGEN], [AC_CONFIG_FILES([doc/Doxyfile])]) AC_CONFIG_LINKS([contrib/devtools/iwyu/bitcoin.core.imp:contrib/devtools/iwyu/bitcoin.core.imp]) AC_CONFIG_LINKS([contrib/filter-lcov.py:contrib/filter-lcov.py]) -AC_CONFIG_LINKS([contrib/macdeploy/background.tiff:contrib/macdeploy/background.tiff]) AC_CONFIG_LINKS([src/.bear-tidy-config:src/.bear-tidy-config]) AC_CONFIG_LINKS([src/.clang-tidy:src/.clang-tidy]) AC_CONFIG_LINKS([test/functional/test_runner.py:test/functional/test_runner.py]) diff --git a/contrib/devtools/bitcoin-tidy/CMakeLists.txt b/contrib/devtools/bitcoin-tidy/CMakeLists.txt new file mode 100644 index 00000000000000..35e60d1d87e2bd --- /dev/null +++ b/contrib/devtools/bitcoin-tidy/CMakeLists.txt @@ -0,0 +1,56 @@ +cmake_minimum_required(VERSION 3.9) + +project(bitcoin-tidy VERSION 1.0.0 DESCRIPTION "clang-tidy checks for Bitcoin Core") + +include(GNUInstallDirs) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED True) +set(CMAKE_CXX_EXTENSIONS False) + +# TODO: Figure out how to avoid the terminfo check +find_package(LLVM REQUIRED CONFIG) +find_program(CLANG_TIDY_EXE NAMES "clang-tidy-${LLVM_VERSION_MAJOR}" "clang-tidy" HINTS ${LLVM_TOOLS_BINARY_DIR}) +message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}") +message(STATUS "Found clang-tidy: ${CLANG_TIDY_EXE}") + +add_library(bitcoin-tidy MODULE bitcoin-tidy.cpp logprintf.cpp) +target_include_directories(bitcoin-tidy SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS}) + +# Disable RTTI and exceptions as necessary +if (MSVC) + target_compile_options(bitcoin-tidy PRIVATE /GR-) +else() + target_compile_options(bitcoin-tidy PRIVATE -fno-rtti) + target_compile_options(bitcoin-tidy PRIVATE -fno-exceptions) +endif() + +if(CMAKE_HOST_APPLE) + # ld64 expects no undefined symbols by default + target_link_options(bitcoin-tidy PRIVATE -Wl,-flat_namespace) + target_link_options(bitcoin-tidy PRIVATE -Wl,-undefined -Wl,suppress) +endif() + +# Add warnings +if (MSVC) + target_compile_options(bitcoin-tidy PRIVATE /W4) +else() + target_compile_options(bitcoin-tidy PRIVATE -Wall) + target_compile_options(bitcoin-tidy PRIVATE -Wextra) +endif() + +if(CMAKE_VERSION VERSION_LESS 3.27) + set(CLANG_TIDY_COMMAND "${CLANG_TIDY_EXE}" "--load=${CMAKE_BINARY_DIR}/${CMAKE_SHARED_MODULE_PREFIX}bitcoin-tidy${CMAKE_SHARED_MODULE_SUFFIX}" "-checks=-*,bitcoin-*") +else() + # CLANG_TIDY_COMMAND supports generator expressions as of 3.27 + set(CLANG_TIDY_COMMAND "${CLANG_TIDY_EXE}" "--load=$" "-checks=-*,bitcoin-*") +endif() + +# Create a dummy library that runs clang-tidy tests as a side-effect of building +add_library(bitcoin-tidy-tests OBJECT EXCLUDE_FROM_ALL example_logprintf.cpp) +add_dependencies(bitcoin-tidy-tests bitcoin-tidy) + +set_target_properties(bitcoin-tidy-tests PROPERTIES CXX_CLANG_TIDY "${CLANG_TIDY_COMMAND}") + + +install(TARGETS bitcoin-tidy LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) diff --git a/contrib/devtools/bitcoin-tidy/README b/contrib/devtools/bitcoin-tidy/README new file mode 100644 index 00000000000000..c15e07c4ede544 --- /dev/null +++ b/contrib/devtools/bitcoin-tidy/README @@ -0,0 +1,11 @@ +# Bitcoin Tidy + +Example Usage: + +```bash +cmake -S . -B build -DLLVM_DIR=$(llvm-config --cmakedir) -DCMAKE_BUILD_TYPE=Release + +cmake --build build -j$(nproc) + +cmake --build build --target bitcoin-tidy-tests -j$(nproc) +``` diff --git a/contrib/devtools/bitcoin-tidy/bitcoin-tidy.cpp b/contrib/devtools/bitcoin-tidy/bitcoin-tidy.cpp new file mode 100644 index 00000000000000..0f34d377938776 --- /dev/null +++ b/contrib/devtools/bitcoin-tidy/bitcoin-tidy.cpp @@ -0,0 +1,22 @@ +// Copyright (c) 2023 Bitcoin Developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "logprintf.h" + +#include +#include + +class BitcoinModule final : public clang::tidy::ClangTidyModule +{ +public: + void addCheckFactories(clang::tidy::ClangTidyCheckFactories& CheckFactories) override + { + CheckFactories.registerCheck("bitcoin-unterminated-logprintf"); + } +}; + +static clang::tidy::ClangTidyModuleRegistry::Add + X("bitcoin-module", "Adds bitcoin checks."); + +volatile int BitcoinModuleAnchorSource = 0; diff --git a/contrib/devtools/bitcoin-tidy/example_logprintf.cpp b/contrib/devtools/bitcoin-tidy/example_logprintf.cpp new file mode 100644 index 00000000000000..a12a666c086d68 --- /dev/null +++ b/contrib/devtools/bitcoin-tidy/example_logprintf.cpp @@ -0,0 +1,108 @@ +// Copyright (c) 2023 Bitcoin Developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include + +// Test for bitcoin-unterminated-logprintf + +enum LogFlags { + NONE +}; + +enum Level { + None +}; + +template +static inline void LogPrintf_(const std::string& logging_function, const std::string& source_file, const int source_line, const LogFlags flag, const Level level, const char* fmt, const Args&... args) +{ +} + +#define LogPrintLevel_(category, level, ...) LogPrintf_(__func__, __FILE__, __LINE__, category, level, __VA_ARGS__) +#define LogPrintf(...) LogPrintLevel_(LogFlags::NONE, Level::None, __VA_ARGS__) + +#define LogPrint(category, ...) \ + do { \ + LogPrintf(__VA_ARGS__); \ + } while (0) + + +class CWallet +{ + std::string GetDisplayName() const + { + return "default wallet"; + } + +public: + template + void WalletLogPrintf(const char* fmt, Params... parameters) const + { + LogPrintf(("%s " + std::string{fmt}).c_str(), GetDisplayName(), parameters...); + }; +}; + +struct ScriptPubKeyMan +{ + std::string GetDisplayName() const + { + return "default wallet"; + } + + template + void WalletLogPrintf(const char* fmt, Params... parameters) const + { + LogPrintf(("%s " + std::string{fmt}).c_str(), GetDisplayName(), parameters...); + }; +}; + +void good_func() +{ + LogPrintf("hello world!\n"); +} +void good_func2() +{ + CWallet wallet; + wallet.WalletLogPrintf("hi\n"); + ScriptPubKeyMan spkm; + spkm.WalletLogPrintf("hi\n"); + + const CWallet& walletref = wallet; + walletref.WalletLogPrintf("hi\n"); + + auto* walletptr = new CWallet(); + walletptr->WalletLogPrintf("hi\n"); + delete walletptr; +} +void bad_func() +{ + LogPrintf("hello world!"); +} +void bad_func2() +{ + LogPrintf(""); +} +void bad_func3() +{ + // Ending in "..." has no special meaning. + LogPrintf("hello world!..."); +} +void bad_func4_ignored() +{ + LogPrintf("hello world!"); // NOLINT(bitcoin-unterminated-logprintf) +} +void bad_func5() +{ + CWallet wallet; + wallet.WalletLogPrintf("hi"); + ScriptPubKeyMan spkm; + spkm.WalletLogPrintf("hi"); + + const CWallet& walletref = wallet; + walletref.WalletLogPrintf("hi"); + + auto* walletptr = new CWallet(); + walletptr->WalletLogPrintf("hi"); + delete walletptr; +} diff --git a/contrib/devtools/bitcoin-tidy/logprintf.cpp b/contrib/devtools/bitcoin-tidy/logprintf.cpp new file mode 100644 index 00000000000000..36beac28c86c31 --- /dev/null +++ b/contrib/devtools/bitcoin-tidy/logprintf.cpp @@ -0,0 +1,60 @@ +// Copyright (c) 2023 Bitcoin Developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "logprintf.h" + +#include +#include + + +namespace { +AST_MATCHER(clang::StringLiteral, unterminated) +{ + size_t len = Node.getLength(); + if (len > 0 && Node.getCodeUnit(len - 1) == '\n') { + return false; + } + return true; +} +} // namespace + +namespace bitcoin { + +void LogPrintfCheck::registerMatchers(clang::ast_matchers::MatchFinder* finder) +{ + using namespace clang::ast_matchers; + + /* + Logprintf(..., ..., ..., ..., ..., "foo", ...) + */ + + finder->addMatcher( + callExpr( + callee(functionDecl(hasName("LogPrintf_"))), + hasArgument(5, stringLiteral(unterminated()).bind("logstring"))), + this); + + /* + auto walletptr = &wallet; + wallet.WalletLogPrintf("foo"); + wallet->WalletLogPrintf("foo"); + */ + finder->addMatcher( + cxxMemberCallExpr( + callee(cxxMethodDecl(hasName("WalletLogPrintf"))), + hasArgument(0, stringLiteral(unterminated()).bind("logstring"))), + this); +} + +void LogPrintfCheck::check(const clang::ast_matchers::MatchFinder::MatchResult& Result) +{ + if (const clang::StringLiteral* lit = Result.Nodes.getNodeAs("logstring")) { + const clang::ASTContext& ctx = *Result.Context; + const auto user_diag = diag(lit->getEndLoc(), "Unterminated format string used with LogPrintf"); + const auto& loc = lit->getLocationOfByte(lit->getByteLength(), *Result.SourceManager, ctx.getLangOpts(), ctx.getTargetInfo()); + user_diag << clang::FixItHint::CreateInsertion(loc, "\\n"); + } +} + +} // namespace bitcoin diff --git a/contrib/devtools/bitcoin-tidy/logprintf.h b/contrib/devtools/bitcoin-tidy/logprintf.h new file mode 100644 index 00000000000000..db95dfe143e843 --- /dev/null +++ b/contrib/devtools/bitcoin-tidy/logprintf.h @@ -0,0 +1,29 @@ +// Copyright (c) 2023 Bitcoin Developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef LOGPRINTF_CHECK_H +#define LOGPRINTF_CHECK_H + +#include + +namespace bitcoin { + +// Warn about any use of LogPrintf that does not end with a newline. +class LogPrintfCheck final : public clang::tidy::ClangTidyCheck +{ +public: + LogPrintfCheck(clang::StringRef Name, clang::tidy::ClangTidyContext* Context) + : clang::tidy::ClangTidyCheck(Name, Context) {} + + bool isLanguageVersionSupported(const clang::LangOptions& LangOpts) const override + { + return LangOpts.CPlusPlus; + } + void registerMatchers(clang::ast_matchers::MatchFinder* Finder) override; + void check(const clang::ast_matchers::MatchFinder::MatchResult& Result) override; +}; + +} // namespace bitcoin + +#endif // LOGPRINTF_CHECK_H diff --git a/contrib/guix/INSTALL.md b/contrib/guix/INSTALL.md index bbd88e58f3dc94..49e92acdf95534 100644 --- a/contrib/guix/INSTALL.md +++ b/contrib/guix/INSTALL.md @@ -578,7 +578,7 @@ sudo --login guix pull --commit= ``` `guix pull` is quite a long process (especially if you're using -`--no-substitute`). If you encounter build problems, please refer to the +`--no-substitutes`). If you encounter build problems, please refer to the [troubleshooting section](#troubleshooting). Note that running a bare `guix pull` with no commit or branch specified will @@ -616,7 +616,7 @@ systemctl enable guix-daemon systemctl start guix-daemon ``` -Remember to set `--no-substitute` in `$libdir/systemd/system/guix-daemon.service` and other customizations if you used them for `guix-daemon-original.service`. +Remember to set `--no-substitutes` in `$libdir/systemd/system/guix-daemon.service` and other customizations if you used them for `guix-daemon-original.service`. ##### If you installed Guix via the Debian/Ubuntu distribution packages diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index e0bd15493f884f..c49553bec3ce41 100755 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -52,7 +52,8 @@ BASEPREFIX="${PWD}/depends" store_path() { grep --extended-regexp "/[^-]{32}-${1}-[^-]+${2:+-${2}}" "${GUIX_ENVIRONMENT}/manifest" \ | head --lines=1 \ - | sed --expression='s|^[[:space:]]*"||' \ + | sed --expression='s|\x29*$||' \ + --expression='s|^[[:space:]]*"||' \ --expression='s|"[[:space:]]*$||' } @@ -314,7 +315,7 @@ mkdir -p "$DISTSRC" | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}-unsigned.tar.gz" \ || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}-unsigned.tar.gz" && exit 1 ) ) - make deploy ${V:+V=1} OSX_DMG="${OUTDIR}/${DISTNAME}-${HOST}-unsigned.dmg" + make deploy ${V:+V=1} OSX_ZIP="${OUTDIR}/${DISTNAME}-${HOST}-unsigned.zip" ;; esac ( diff --git a/contrib/guix/libexec/codesign.sh b/contrib/guix/libexec/codesign.sh index 6ffa0f07b2e90d..0b5f77d01ea2ba 100755 --- a/contrib/guix/libexec/codesign.sh +++ b/contrib/guix/libexec/codesign.sh @@ -85,11 +85,8 @@ mkdir -p "$DISTSRC" # Apply detached codesignatures to dist/ (in-place) signapple apply dist/Bitcoin-Qt.app codesignatures/osx/dist - # Make a DMG from dist/ - xorrisofs -D -l -V "$(< osx_volname)" -no-pad -r -dir-mode 0755 \ - -o "${OUTDIR}/${DISTNAME}-${HOST}.dmg" \ - dist \ - -- -volume_date all_file_dates ="$SOURCE_DATE_EPOCH" + # Make a .zip from dist/ + zip "${OUTDIR}/${DISTNAME}-${HOST}.zip" dist/* ;; *) exit 1 diff --git a/contrib/guix/libexec/prelude.bash b/contrib/guix/libexec/prelude.bash index 3eb8fc02dae65c..4d5b06a54dd386 100644 --- a/contrib/guix/libexec/prelude.bash +++ b/contrib/guix/libexec/prelude.bash @@ -51,7 +51,7 @@ fi time-machine() { # shellcheck disable=SC2086 guix time-machine --url=https://git.savannah.gnu.org/git/guix.git \ - --commit=998eda3067c7d21e0d9bb3310d2f5a14b8f1c681 \ + --commit=160f78a4d92205df986ed9efcce7d3aac188cb24 \ --cores="$JOBS" \ --keep-failed \ --fallback \ diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index 31ac44f8e210fe..3c4ad6cdbc9fca 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -3,7 +3,6 @@ ((gnu packages bash) #:select (bash-minimal)) (gnu packages bison) ((gnu packages certs) #:select (nss-certs)) - ((gnu packages cdrom) #:select (xorriso)) ((gnu packages cmake) #:select (cmake-minimal)) (gnu packages commencement) (gnu packages compression) @@ -40,41 +39,7 @@ FILE-NAME found in ./patches relative to the current file." ((%patch-path (list (string-append (dirname (current-filename)) "/patches")))) (list (search-patch file-name) ...))) -(define (make-ssp-fixed-gcc xgcc) - "Given a XGCC package, return a modified package that uses the SSP function -from glibc instead of from libssp.so. Our `symbol-check' script will complain if -we link against libssp.so, and thus will ensure that this works properly. - -Taken from: -http://www.linuxfromscratch.org/hlfs/view/development/chapter05/gcc-pass1.html" - (package - (inherit xgcc) - (arguments - (substitute-keyword-arguments (package-arguments xgcc) - ((#:make-flags flags) - `(cons "gcc_cv_libc_provides_ssp=yes" ,flags)))))) - -(define (make-gcc-rpath-link xgcc) - "Given a XGCC package, return a modified package that replace each instance of --rpath in the default system spec that's inserted by Guix with -rpath-link" - (package - (inherit xgcc) - (arguments - (substitute-keyword-arguments (package-arguments xgcc) - ((#:phases phases) - `(modify-phases ,phases - (add-after 'pre-configure 'replace-rpath-with-rpath-link - (lambda _ - (substitute* (cons "gcc/config/rs6000/sysv4.h" - (find-files "gcc/config" - "^gnu-user.*\\.h$")) - (("-rpath=") "-rpath-link=")) - #t)))))))) - -(define building-on (string-append (list-ref (string-split (%current-system) #\-) 0) "-guix-linux-gnu")) - -(define (explicit-cross-configure package) - (package-with-extra-configure-variable package "--build" building-on)) +(define building-on (string-append "--build=" (list-ref (string-split (%current-system) #\-) 0) "-guix-linux-gnu")) (define (make-cross-toolchain target base-gcc-for-libc @@ -85,28 +50,28 @@ http://www.linuxfromscratch.org/hlfs/view/development/chapter05/gcc-pass1.html" (let* ((xbinutils (cross-binutils target)) ;; 1. Build a cross-compiling gcc without targeting any libc, derived ;; from BASE-GCC-FOR-LIBC - (xgcc-sans-libc (explicit-cross-configure (cross-gcc target - #:xgcc base-gcc-for-libc - #:xbinutils xbinutils))) + (xgcc-sans-libc (cross-gcc target + #:xgcc base-gcc-for-libc + #:xbinutils xbinutils)) ;; 2. Build cross-compiled kernel headers with XGCC-SANS-LIBC, derived ;; from BASE-KERNEL-HEADERS (xkernel (cross-kernel-headers target - base-kernel-headers - xgcc-sans-libc - xbinutils)) + #:linux-headers base-kernel-headers + #:xgcc xgcc-sans-libc + #:xbinutils xbinutils)) ;; 3. Build a cross-compiled libc with XGCC-SANS-LIBC and XKERNEL, ;; derived from BASE-LIBC - (xlibc (explicit-cross-configure (cross-libc target - base-libc - xgcc-sans-libc - xbinutils - xkernel))) + (xlibc (cross-libc target + #:libc base-libc + #:xgcc xgcc-sans-libc + #:xbinutils xbinutils + #:xheaders xkernel)) ;; 4. Build a cross-compiling gcc targeting XLIBC, derived from ;; BASE-GCC - (xgcc (explicit-cross-configure (cross-gcc target - #:xgcc base-gcc - #:xbinutils xbinutils - #:libc xlibc)))) + (xgcc (cross-gcc target + #:xgcc base-gcc + #:xbinutils xbinutils + #:libc xlibc))) ;; Define a meta-package that propagates the resulting XBINUTILS, XLIBC, and ;; XGCC (package @@ -130,21 +95,12 @@ chain for " target " development.")) (define base-gcc gcc-10) (define base-linux-kernel-headers linux-libre-headers-5.15) -;; https://gcc.gnu.org/install/configure.html -(define (hardened-gcc gcc) - (package-with-extra-configure-variable ( - package-with-extra-configure-variable ( - package-with-extra-configure-variable gcc - "--enable-initfini-array" "yes") - "--enable-default-ssp" "yes") - "--enable-default-pie" "yes")) - (define* (make-bitcoin-cross-toolchain target #:key - (base-gcc-for-libc base-gcc) + (base-gcc-for-libc linux-base-gcc) (base-kernel-headers base-linux-kernel-headers) - (base-libc (hardened-glibc glibc-2.27)) - (base-gcc (make-gcc-rpath-link (hardened-gcc base-gcc)))) + (base-libc glibc-2.27) + (base-gcc linux-base-gcc)) "Convenience wrapper around MAKE-CROSS-TOOLCHAIN with default values desirable for building Bitcoin Core release binaries." (make-cross-toolchain target @@ -153,26 +109,19 @@ desirable for building Bitcoin Core release binaries." base-libc base-gcc)) -(define (make-gcc-with-pthreads gcc) - (package-with-extra-configure-variable - (package-with-extra-patches gcc - (search-our-patches "gcc-10-remap-guix-store.patch")) - "--enable-threads" "posix")) - -(define (make-mingw-w64-cross-gcc cross-gcc) - (package-with-extra-patches cross-gcc - (search-our-patches "vmov-alignment.patch" - "gcc-broken-longjmp.patch"))) +(define (gcc-mingw-patches gcc) + (package-with-extra-patches gcc + (search-our-patches "gcc-remap-guix-store.patch" + "vmov-alignment.patch"))) (define (make-mingw-pthreads-cross-toolchain target) "Create a cross-compilation toolchain package for TARGET" (let* ((xbinutils (cross-binutils target)) (pthreads-xlibc mingw-w64-x86_64-winpthreads) - (pthreads-xgcc (make-gcc-with-pthreads - (cross-gcc target - #:xgcc (make-ssp-fixed-gcc (make-mingw-w64-cross-gcc base-gcc)) + (pthreads-xgcc (cross-gcc target + #:xgcc (gcc-mingw-patches mingw-w64-base-gcc) #:xbinutils xbinutils - #:libc pthreads-xlibc)))) + #:libc pthreads-xlibc))) ;; Define a meta-package that propagates the resulting XBINUTILS, XLIBC, and ;; XGCC (package @@ -192,11 +141,6 @@ chain for " target " development.")) (home-page (package-home-page pthreads-xgcc)) (license (package-license pthreads-xgcc))))) -(define (make-nsis-for-gcc-10 base-nsis) - (package-with-extra-patches base-nsis - (search-our-patches "nsis-gcc-10-memmove.patch" - "nsis-disable-installer-reloc.patch"))) - ;; While LIEF is packaged in Guix, we maintain our own package, ;; to simplify building, and more easily apply updates. ;; Moreover, the Guix's package uses cmake, which caused build @@ -292,7 +236,7 @@ thus should be able to compile on most platforms where these exist.") (define-public python-oscrypto (package (name "python-oscrypto") - (version "1.2.1") + (version "1.3.0") (source (origin (method git-fetch) @@ -302,7 +246,7 @@ thus should be able to compile on most platforms where these exist.") (file-name (git-file-name name version)) (sha256 (base32 - "1d4d8s4z340qhvb3g5m5v3436y3a71yc26wk4749q64m09kxqc3l")) + "1v5wkmzcyiqy39db8j2dvkdrv2nlsc48556h73x4dzjwd6kg4q0a")) (patches (search-our-patches "oscrypto-hard-code-openssl.patch")))) (build-system python-build-system) (native-search-paths @@ -532,16 +476,45 @@ and endian independent.") inspecting signatures in Mach-O binaries.") (license license:expat)))) -;; https://www.gnu.org/software/libc/manual/html_node/Configuring-and-compiling.html -;; We don't use --disable-werror directly, as that would be passed through to bash, -;; and cause it's build to fail. -(define (hardened-glibc glibc) - (package-with-extra-configure-variable ( - package-with-extra-configure-variable ( - package-with-extra-configure-variable glibc - "enable_werror" "no") - "--enable-stack-protector" "all") - "--enable-bind-now" "yes")) +(define-public mingw-w64-base-gcc + (package + (inherit base-gcc) + (arguments + (substitute-keyword-arguments (package-arguments base-gcc) + ((#:configure-flags flags) + `(append ,flags + ;; https://gcc.gnu.org/install/configure.html + (list "--enable-threads=posix", + building-on))) + ((#:make-flags flags) + ;; Uses the SSP functions from glibc instead of from libssp.so. + ;; Our 'symbol-check' script will complain if we link against libssp.so, + ;; and thus will ensure that this works properly. + `(cons "gcc_cv_libc_provides_ssp=yes" ,flags)))))) + +(define-public linux-base-gcc + (package + (inherit base-gcc) + (arguments + (substitute-keyword-arguments (package-arguments base-gcc) + ((#:configure-flags flags) + `(append ,flags + ;; https://gcc.gnu.org/install/configure.html + (list "--enable-initfini-array=yes", + "--enable-default-ssp=yes", + "--enable-default-pie=yes", + building-on))) + ((#:phases phases) + `(modify-phases ,phases + ;; Given a XGCC package, return a modified package that replace each instance of + ;; -rpath in the default system spec that's inserted by Guix with -rpath-link + (add-after 'pre-configure 'replace-rpath-with-rpath-link + (lambda _ + (substitute* (cons "gcc/config/rs6000/sysv4.h" + (find-files "gcc/config" + "^gnu-user.*\\.h$")) + (("-rpath=") "-rpath-link=")) + #t)))))))) (define-public glibc-2.27 (package @@ -559,7 +532,29 @@ inspecting signatures in Mach-O binaries.") (patches (search-our-patches "glibc-2.27-riscv64-Use-__has_include-to-include-asm-syscalls.h.patch" "glibc-2.27-fcommon.patch" "glibc-2.27-guix-prefix.patch" - "glibc-2.27-no-librt.patch")))))) + "glibc-2.27-no-librt.patch" + "glibc-2.27-powerpc-ldbrx.patch")))) + (arguments + (substitute-keyword-arguments (package-arguments glibc) + ((#:configure-flags flags) + `(append ,flags + ;; https://www.gnu.org/software/libc/manual/html_node/Configuring-and-compiling.html + (list "--enable-stack-protector=all", + "--enable-bind-now", + "--disable-werror", + building-on))) + ((#:phases phases) + `(modify-phases ,phases + (add-before 'configure 'set-etc-rpc-installation-directory + (lambda* (#:key outputs #:allow-other-keys) + ;; Install the rpc data base file under `$out/etc/rpc'. + ;; Otherwise build will fail with "Permission denied." + (let ((out (assoc-ref outputs "out"))) + (substitute* "sunrpc/Makefile" + (("^\\$\\(inst_sysconfdir\\)/rpc(.*)$" _ suffix) + (string-append out "/etc/rpc" suffix "\n")) + (("^install-others =.*$") + (string-append "install-others = " out "/etc/rpc\n")))))))))))) (packages->manifest (append @@ -585,7 +580,7 @@ inspecting signatures in Mach-O binaries.") xz ;; Build tools gnu-make - libtool-2.4.7 + libtool autoconf-2.71 automake pkg-config @@ -594,7 +589,7 @@ inspecting signatures in Mach-O binaries.") gcc-toolchain-10 (list gcc-toolchain-10 "static") ;; Scripting - python-minimal ;; (3.9) + python-minimal ;; (3.10) ;; Git git-minimal ;; Tests @@ -604,11 +599,11 @@ inspecting signatures in Mach-O binaries.") ;; Windows (list zip (make-mingw-pthreads-cross-toolchain "x86_64-w64-mingw32") - (make-nsis-for-gcc-10 nsis-x86_64) + nsis-x86_64 nss-certs osslsigncode)) ((string-contains target "-linux-") (list (make-bitcoin-cross-toolchain target))) ((string-contains target "darwin") - (list clang-toolchain-11 binutils cmake-minimal xorriso python-signapple)) + (list clang-toolchain-15 binutils cmake-minimal python-signapple zip)) (else '()))))) diff --git a/contrib/guix/patches/gcc-broken-longjmp.patch b/contrib/guix/patches/gcc-broken-longjmp.patch deleted file mode 100644 index 56568813c0d834..00000000000000 --- a/contrib/guix/patches/gcc-broken-longjmp.patch +++ /dev/null @@ -1,70 +0,0 @@ -commit eb5698897c52702498938592d7f76e67d126451f -Author: Eric Botcazou -Date: Wed May 5 22:48:51 2021 +0200 - - Fix PR target/100402 - - This is a regression for 64-bit Windows present from mainline down to the 9 - branch and introduced by the fix for PR target/99234. Again SEH, but with - a twist related to the way MinGW implements setjmp/longjmp, which turns out - to be piggybacked on SEH with recent versions of MinGW, i.e. the longjmp - performs a bona-fide unwinding of the stack, because it calls RtlUnwindEx - with the second argument initially passed to setjmp, which is the result of - __builtin_frame_address (0) in the MinGW header file: - - define setjmp(BUF) _setjmp((BUF), __builtin_frame_address (0)) - - This means that we directly expose the frame pointer to the SEH machinery - here (unlike with regular exception handling where we use an intermediate - CFA) and thus that we cannot do whatever we want with it. The old code - would leave it unaligned, i.e. not multiple of 16, whereas the new code - aligns it, but this breaks for some reason; at least it appears that a - .seh_setframe directive with 0 as second argument always works, so the - fix aligns it this way. - - gcc/ - PR target/100402 - * config/i386/i386.c (ix86_compute_frame_layout): For a SEH target, - always return the establisher frame for __builtin_frame_address (0). - gcc/testsuite/ - * gcc.c-torture/execute/20210505-1.c: New test. - - This patch can be dropped when we are building with GCC 10.4.0 or later. - -diff --git a/gcc/config/i386/i386.c b/gcc/config/i386/i386.c -index 2f838840e96..06ad1b2274e 100644 ---- a/gcc/config/i386/i386.c -+++ b/gcc/config/i386/i386.c -@@ -6356,12 +6356,29 @@ ix86_compute_frame_layout (void) - area, see the SEH code in config/i386/winnt.c for the rationale. */ - frame->hard_frame_pointer_offset = frame->sse_reg_save_offset; - -- /* If we can leave the frame pointer where it is, do so. Also, return -+ /* If we can leave the frame pointer where it is, do so; however return - the establisher frame for __builtin_frame_address (0) or else if the -- frame overflows the SEH maximum frame size. */ -+ frame overflows the SEH maximum frame size. -+ -+ Note that the value returned by __builtin_frame_address (0) is quite -+ constrained, because setjmp is piggybacked on the SEH machinery with -+ recent versions of MinGW: -+ -+ # elif defined(__SEH__) -+ # if defined(__aarch64__) || defined(_ARM64_) -+ # define setjmp(BUF) _setjmp((BUF), __builtin_sponentry()) -+ # elif (__MINGW_GCC_VERSION < 40702) -+ # define setjmp(BUF) _setjmp((BUF), mingw_getsp()) -+ # else -+ # define setjmp(BUF) _setjmp((BUF), __builtin_frame_address (0)) -+ # endif -+ -+ and the second argument passed to _setjmp, if not null, is forwarded -+ to the TargetFrame parameter of RtlUnwindEx by longjmp (after it has -+ built an ExceptionRecord on the fly describing the setjmp buffer). */ - const HOST_WIDE_INT diff - = frame->stack_pointer_offset - frame->hard_frame_pointer_offset; -- if (diff <= 255) -+ if (diff <= 255 && !crtl->accesses_prior_frames) - { - /* The resulting diff will be a multiple of 16 lower than 255, - i.e. at most 240 as required by the unwind data structure. */ diff --git a/contrib/guix/patches/gcc-10-remap-guix-store.patch b/contrib/guix/patches/gcc-remap-guix-store.patch similarity index 100% rename from contrib/guix/patches/gcc-10-remap-guix-store.patch rename to contrib/guix/patches/gcc-remap-guix-store.patch diff --git a/contrib/guix/patches/glibc-2.27-powerpc-ldbrx.patch b/contrib/guix/patches/glibc-2.27-powerpc-ldbrx.patch new file mode 100644 index 00000000000000..26716054c8fcfa --- /dev/null +++ b/contrib/guix/patches/glibc-2.27-powerpc-ldbrx.patch @@ -0,0 +1,245 @@ +From 50b0b3c9ff71ffd7ebbd74ae46844c3566478123 Mon Sep 17 00:00:00 2001 +From: "Gabriel F. T. Gomes" +Date: Mon, 27 May 2019 15:21:22 -0300 +Subject: [PATCH] powerpc: Fix build failures with current GCC + +Since GCC commit 271500 (svn), also known as the following commit on the +git mirror: + +commit e154242724b084380e3221df7c08fcdbd8460674 +Author: amodra +Date: Wed May 22 04:34:26 2019 +0000 + + [RS6000] Don't pass -many to the assembler + +glibc builds are failing when an assembly implementation does not +declare the correct '.machine' directive, or when no such directive is +declared at all. For example, when a POWER6 instruction is used, but +'.machine power6' is not declared, the assembler will fail with an error +similar to the following: + + ../sysdeps/powerpc/powerpc64/power8/strcmp.S: Assembler messages: + 24 ../sysdeps/powerpc/powerpc64/power8/strcmp.S:55: Error: unrecognized opcode: `cmpb' + +This patch adds '.machine powerN' directives where none existed, as well +as it updates '.machine power7' directives on POWER8 files, because the +minimum binutils version required to build glibc (binutils 2.25) now +provides this machine version. It also adds '-many' to the assembler +command used to build tst-set_ppr.c. + +Tested for powerpc, powerpc64, and powerpc64le, as well as with +build-many-glibcs.py for powerpc targets. + +Reviewed-by: Tulio Magno Quites Machado Filho +--- + sysdeps/powerpc/Makefile | 5 +++ + sysdeps/powerpc/powerpc64/power4/memcmp.S | 7 ++++ + sysdeps/powerpc/powerpc64/power7/strncmp.S | 1 + + .../powerpc/powerpc64/power8/fpu/s_llround.S | 1 + + sysdeps/powerpc/powerpc64/power8/strcasecmp.S | 36 ++++++------------- + sysdeps/powerpc/powerpc64/power8/strcasestr.S | 14 ++------ + sysdeps/powerpc/powerpc64/power8/strcmp.S | 1 + + 7 files changed, 28 insertions(+), 37 deletions(-) + +diff --git a/sysdeps/powerpc/Makefile b/sysdeps/powerpc/Makefile +index 6aa683b03f..23126147df 100644 +--- a/sysdeps/powerpc/Makefile ++++ b/sysdeps/powerpc/Makefile +@@ -45,6 +45,11 @@ ifeq ($(subdir),misc) + sysdep_headers += sys/platform/ppc.h + tests += test-gettimebase + tests += tst-set_ppr ++ ++# This test is expected to run and exit with EXIT_UNSUPPORTED on ++# processors that do not implement the Power ISA 2.06 or greater. ++# But the test makes use of instructions from Power ISA 2.06 and 2.07. ++CFLAGS-tst-set_ppr.c += -Wa,-many + endif + + ifneq (,$(filter %le,$(config-machine))) +diff --git a/sysdeps/powerpc/powerpc64/power4/memcmp.S b/sysdeps/powerpc/powerpc64/power4/memcmp.S +index e5319f101f..38dcf4c9a1 100644 +--- a/sysdeps/powerpc/powerpc64/power4/memcmp.S ++++ b/sysdeps/powerpc/powerpc64/power4/memcmp.S +@@ -26,7 +26,14 @@ + # define MEMCMP memcmp + #endif + ++#ifndef __LITTLE_ENDIAN__ + .machine power4 ++#else ++/* Little endian is only available since POWER8, so it's safe to ++ specify .machine as power8 (or older), even though this is a POWER4 ++ file. Since the little-endian code uses 'ldbrx', power7 is enough. */ ++ .machine power7 ++#endif + ENTRY_TOCLESS (MEMCMP, 4) + CALL_MCOUNT 3 + +diff --git a/sysdeps/powerpc/powerpc64/power7/strncmp.S b/sysdeps/powerpc/powerpc64/power7/strncmp.S +index 0c7429d19f..10f898c5a3 100644 +--- a/sysdeps/powerpc/powerpc64/power7/strncmp.S ++++ b/sysdeps/powerpc/powerpc64/power7/strncmp.S +@@ -28,6 +28,7 @@ + const char *s2 [r4], + size_t size [r5]) */ + ++ .machine power7 + ENTRY_TOCLESS (STRNCMP, 5) + CALL_MCOUNT 3 + +diff --git a/sysdeps/powerpc/powerpc64/power8/fpu/s_llround.S b/sysdeps/powerpc/powerpc64/power8/fpu/s_llround.S +index a22fc63bb3..84c76ba0f9 100644 +--- a/sysdeps/powerpc/powerpc64/power8/fpu/s_llround.S ++++ b/sysdeps/powerpc/powerpc64/power8/fpu/s_llround.S +@@ -26,6 +26,7 @@ + + /* long long [r3] llround (float x [fp1]) */ + ++ .machine power8 + ENTRY_TOCLESS (__llround) + CALL_MCOUNT 0 + frin fp1,fp1 /* Round to nearest +-0.5. */ +diff --git a/sysdeps/powerpc/powerpc64/power8/strcasecmp.S b/sysdeps/powerpc/powerpc64/power8/strcasecmp.S +index 3a2efe2a64..eeacd40c7f 100644 +--- a/sysdeps/powerpc/powerpc64/power8/strcasecmp.S ++++ b/sysdeps/powerpc/powerpc64/power8/strcasecmp.S +@@ -91,21 +91,7 @@ + 3: \ + TOLOWER() + +-#ifdef _ARCH_PWR8 +-# define VCLZD_V8_v7 vclzd v8, v7; +-# define MFVRD_R3_V1 mfvrd r3, v1; +-# define VSUBUDM_V9_V8 vsubudm v9, v9, v8; +-# define VPOPCNTD_V8_V8 vpopcntd v8, v8; +-# define VADDUQM_V7_V8 vadduqm v9, v7, v8; +-#else +-# define VCLZD_V8_v7 .long 0x11003fc2 +-# define MFVRD_R3_V1 .long 0x7c230067 +-# define VSUBUDM_V9_V8 .long 0x112944c0 +-# define VPOPCNTD_V8_V8 .long 0x110047c3 +-# define VADDUQM_V7_V8 .long 0x11274100 +-#endif +- +- .machine power7 ++ .machine power8 + + ENTRY (__STRCASECMP) + #ifdef USE_AS_STRNCASECMP +@@ -265,15 +251,15 @@ L(different): + #ifdef __LITTLE_ENDIAN__ + /* Count trailing zero. */ + vspltisb v8, -1 +- VADDUQM_V7_V8 ++ vadduqm v9, v7, v8 + vandc v8, v9, v7 +- VPOPCNTD_V8_V8 ++ vpopcntd v8, v8 + vspltb v6, v8, 15 + vcmpequb. v6, v6, v1 + blt cr6, L(shift8) + #else + /* Count leading zero. */ +- VCLZD_V8_v7 ++ vclzd v8, v7 + vspltb v6, v8, 7 + vcmpequb. v6, v6, v1 + blt cr6, L(shift8) +@@ -291,7 +277,7 @@ L(skipsum): + /* Merge and move to GPR. */ + vmrglb v6, v6, v7 + vslo v1, v6, v1 +- MFVRD_R3_V1 ++ mfvrd r3, v1 + /* Place the characters that are different in first position. */ + sldi rSTR2, rRTN, 56 + srdi rSTR2, rSTR2, 56 +@@ -301,7 +287,7 @@ L(skipsum): + vslo v6, v5, v8 + vslo v7, v4, v8 + vmrghb v1, v6, v7 +- MFVRD_R3_V1 ++ mfvrd r3, v1 + srdi rSTR2, rRTN, 48 + sldi rSTR2, rSTR2, 56 + srdi rSTR2, rSTR2, 56 +@@ -320,15 +306,15 @@ L(null_found): + #ifdef __LITTLE_ENDIAN__ + /* Count trailing zero. */ + vspltisb v8, -1 +- VADDUQM_V7_V8 ++ vadduqm v9, v7, v8 + vandc v8, v9, v7 +- VPOPCNTD_V8_V8 ++ vpopcntd v8, v8 + vspltb v6, v8, 15 + vcmpequb. v6, v6, v10 + blt cr6, L(shift_8) + #else + /* Count leading zero. */ +- VCLZD_V8_v7 ++ vclzd v8, v7 + vspltb v6, v8, 7 + vcmpequb. v6, v6, v10 + blt cr6, L(shift_8) +@@ -343,10 +329,10 @@ L(skipsum1): + vspltisb v10, 7 + vslb v10, v10, v10 + vsldoi v9, v0, v10, 1 +- VSUBUDM_V9_V8 ++ vsubudm v9, v9, v8 + vspltisb v8, 8 + vsldoi v8, v0, v8, 1 +- VSUBUDM_V9_V8 ++ vsubudm v9, v9, v8 + /* Shift and remove junk after null character. */ + #ifdef __LITTLE_ENDIAN__ + vslo v5, v5, v9 +diff --git a/sysdeps/powerpc/powerpc64/power8/strcasestr.S b/sysdeps/powerpc/powerpc64/power8/strcasestr.S +index 9fc24c29f9..e10f06fd86 100644 +--- a/sysdeps/powerpc/powerpc64/power8/strcasestr.S ++++ b/sysdeps/powerpc/powerpc64/power8/strcasestr.S +@@ -73,18 +73,8 @@ + vor reg, v8, reg; \ + vcmpequb. v6, reg, v4; + +-/* TODO: change these to the actual instructions when the minimum required +- binutils allows it. */ +-#ifdef _ARCH_PWR8 +-#define VCLZD_V8_v7 vclzd v8, v7; +-#else +-#define VCLZD_V8_v7 .long 0x11003fc2 +-#endif +- + #define FRAMESIZE (FRAME_MIN_SIZE+48) +-/* TODO: change this to .machine power8 when the minimum required binutils +- allows it. */ +- .machine power7 ++ .machine power8 + ENTRY (STRCASESTR, 4) + CALL_MCOUNT 2 + mflr r0 /* Load link register LR to r0. */ +@@ -291,7 +281,7 @@ L(nullchk1): + vcmpequb. v6, v0, v7 + /* Shift r3 by 16 bytes and proceed. */ + blt cr6, L(shift16) +- VCLZD_V8_v7 ++ vclzd v8, v7 + #ifdef __LITTLE_ENDIAN__ + vspltb v6, v8, 15 + #else +diff --git a/sysdeps/powerpc/powerpc64/power8/strcmp.S b/sysdeps/powerpc/powerpc64/power8/strcmp.S +index 15e7351d1b..d592266d1d 100644 +--- a/sysdeps/powerpc/powerpc64/power8/strcmp.S ++++ b/sysdeps/powerpc/powerpc64/power8/strcmp.S +@@ -31,6 +31,7 @@ + 64K as default, the page cross handling assumes minimum page size of + 4k. */ + ++ .machine power8 + ENTRY_TOCLESS (STRCMP, 4) + li r0,0 + +-- +2.41.0 diff --git a/contrib/guix/patches/nsis-disable-installer-reloc.patch b/contrib/guix/patches/nsis-disable-installer-reloc.patch deleted file mode 100644 index 4914527e56bebe..00000000000000 --- a/contrib/guix/patches/nsis-disable-installer-reloc.patch +++ /dev/null @@ -1,30 +0,0 @@ -Patch NSIS so that it's installer stubs, produced at NSIS build time, -do not contain .reloc sections, which will exist by default when using -binutils/ld 2.36+. - -This ultimately fixes an issue when running the installer with the -"Force randomization for images (Mandatory ASLR)" setting active. - -This patch has not yet been sent upstream, because it's not clear if this -is the best fix, for the underlying issue, which seems to be that makensis -doesn't account for .reloc sections when it builds installers. - -The existence of a reloc section shouldn't be a problem, and, if anything, -is actually a requirement for working ASLR. All other Windows binaries we -produce contain them, and function correctly when under the same -"Force randomization for images (Mandatory ASLR)" setting. - -See: -https://github.com/bitcoin/bitcoin/issues/25726 -https://sourceforge.net/p/nsis/bugs/1131/ - ---- a/SCons/Config/gnu -+++ b/SCons/Config/gnu -@@ -102,6 +102,7 @@ stub_env.Append(LINKFLAGS = ['-mwindows']) # build windows executables - stub_env.Append(LINKFLAGS = ['$NODEFLIBS_FLAG']) # no standard libraries - stub_env.Append(LINKFLAGS = ['$ALIGN_FLAG']) # 512 bytes align - stub_env.Append(LINKFLAGS = ['$MAP_FLAG']) # generate map file -+stub_env.Append(LINKFLAGS = ['-Wl,--disable-reloc-section']) - - conf = FlagsConfigure(stub_env) - conf.CheckCompileFlag('-fno-tree-loop-distribute-patterns') # GCC 10: Don't generate msvcrt!memmove calls (bug #1248) diff --git a/contrib/guix/patches/nsis-gcc-10-memmove.patch b/contrib/guix/patches/nsis-gcc-10-memmove.patch deleted file mode 100644 index a1aadfd4f36186..00000000000000 --- a/contrib/guix/patches/nsis-gcc-10-memmove.patch +++ /dev/null @@ -1,23 +0,0 @@ -commit f6df41524e703dc471e283e566a48e05a735b7f2 -Author: Anders -Date: Sat Jun 27 23:18:45 2020 +0000 - - Don't let GCC 10 generate memmove calls (bug #1248) - - git-svn-id: https://svn.code.sf.net/p/nsis/code/NSIS/trunk@7189 212acab6-be3b-0410-9dea-997c60f758d6 - -diff --git a/SCons/Config/gnu b/SCons/Config/gnu -index bfcb362d..21fa446b 100644 ---- a/SCons/Config/gnu -+++ b/SCons/Config/gnu -@@ -103,6 +103,10 @@ stub_env.Append(LINKFLAGS = ['$NODEFLIBS_FLAG']) # no standard libraries - stub_env.Append(LINKFLAGS = ['$ALIGN_FLAG']) # 512 bytes align - stub_env.Append(LINKFLAGS = ['$MAP_FLAG']) # generate map file - -+conf = FlagsConfigure(stub_env) -+conf.CheckCompileFlag('-fno-tree-loop-distribute-patterns') # GCC 10: Don't generate msvcrt!memmove calls (bug #1248) -+conf.Finish() -+ - stub_uenv = stub_env.Clone() - stub_uenv.Append(CPPDEFINES = ['_UNICODE', 'UNICODE']) - diff --git a/contrib/macdeploy/README.md b/contrib/macdeploy/README.md index 599a0bfa6ce497..16fb0dad219d68 100644 --- a/contrib/macdeploy/README.md +++ b/contrib/macdeploy/README.md @@ -6,7 +6,7 @@ The `macdeployqtplus` script should not be run manually. Instead, after building make deploy ``` -When complete, it will have produced `Bitcoin-Core.dmg`. +When complete, it will have produced `Bitcoin-Core.zip`. ## SDK Extraction @@ -60,10 +60,10 @@ previous stage) as the first argument. The `sha256sum` of the generated TAR.GZ archive should be `df75d30ecafc429e905134333aeae56ac65fac67cb4182622398fd717df77619`. -## Deterministic macOS DMG Notes +## Deterministic macOS App Notes -Working macOS DMGs are created in Linux by combining a recent `clang`, the Apple -`binutils` (`ld`, `ar`, etc) and DMG authoring tools. +macOS Applications are created in Linux by combining a recent `clang` and the Apple +`binutils` (`ld`, `ar`, etc). Apple uses `clang` extensively for development and has upstreamed the necessary functionality so that a vanilla clang can take advantage. It supports the use of `-F`, @@ -93,20 +93,15 @@ created using these tools. The build process has been designed to avoid includin SDK's files in Guix's outputs. All interim tarballs are fully deterministic and may be freely redistributed. -[`xorrisofs`](https://www.gnu.org/software/xorriso/) is used to create the DMG. - -A background image is added to DMG files by inserting a `.DS_Store` during creation. - As of OS X 10.9 Mavericks, using an Apple-blessed key to sign binaries is a requirement in order to satisfy the new Gatekeeper requirements. Because this private key cannot be shared, we'll have to be a bit creative in order for the build process to remain somewhat deterministic. Here's how it works: -- Builders use Guix to create an unsigned release. This outputs an unsigned DMG which +- Builders use Guix to create an unsigned release. This outputs an unsigned ZIP which users may choose to bless and run. It also outputs an unsigned app structure in the form - of a tarball, which also contains all of the tools that have been previously (deterministically) - built in order to create a final DMG. + of a tarball. - The Apple keyholder uses this unsigned app to create a detached signature, using the script that is also included there. Detached signatures are available from this [repository](https://github.com/bitcoin-core/bitcoin-detached-sigs). - Builders feed the unsigned app + detached signature back into Guix. It uses the - pre-built tools to recombine the pieces into a deterministic DMG. + pre-built tools to recombine the pieces into a deterministic ZIP. diff --git a/contrib/macdeploy/background.tiff b/contrib/macdeploy/background.tiff deleted file mode 100644 index 1fb088c8374ac3acd5c4f4eb6ea6b5869723fd2f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 18464 zcmeHtc|4Tg`}d3)OV&_Cwvs)vWEs22l08M1LQG8d8Pv#9DJ_a*9mW!(5=xdVQA#p2 zM1@G}%v82AA^UUgTYW!$zMt>&{Fdk6c#Zoy*SXI7dSA;qbJy!wSt%mcaUu{$UybK> zC#LTed;CW2fvl}M=Nkasjk(NKXM?5+jZUu0F8O3r#LZt=@%D&HseG>t8+Uipo& zN}Uz}FMAN4(iuFL&_T1q%f^#a$;NUI-@HDY%Uvs>qaN|`WWYmJ;w!1SCYQh%l#{4z zru#BjDKYmq15ePl70p`Qsh#C)%n*F-Vz{1&@yqS!>NPNB@1mp$=H!~NB?TU6+DQ$g zw{ur6w^Tr0FFBk1LiMPx!Y1}&V?ae#RxR$4D^EYES4@h%IMt3V{Gu)HGnnkOW4mN) zckr2X`^yM>T(AxGm1SM(yEiu3dA++eF;cnV>c<0p1gz1RvMbFg7!6E-^i{irH?g>Y zUDiV%kK5FxB7GB6_wAPT(|oR0b?PjtDi3gxBUkq|vD+UJ*#BduKO)C`op9F9#3t`^ zvB&1ZGYlF+F22nOS(DwwE20v-JeR+6aUyo_B*N>qRETTgWSxpWF?jOYD^lw^TSKGy z_cdkq{dxP8A}`;mKHD$*?3+Qqd7VV`j?pcCDbgXk3QZ*!XHTbyhiu)g+u>RTdb6Xe z`D=iCNeDj}dpvPZ^`Cz@oY~)<6vijfFPk4HHjv+PV<=gFE8+4TyObk-Sxf9u6#?I# zYo(`>WxS{6Wxn|2VTVeD0>~e?hNWia79S^S5q;|)Sy2yeCvpjh%X{U$j+;VGJos?^ zbVo8L$*)d)uTF<6`bpzHp0=(#ZO0CLi@SX1l+A%sGgOw};Ecj-E*YaaN7H8PIxt$2 zoGFu9{b8x8UuDZHO_zDRcGv4RgQEh0-gfs6ygTU<*ZerbTIMvW^qiqnC~>J^?N{{2%z=tmuT`}by$=kIZgzW} zd89i%nt$89@uhyxazg46>1W@=za`$qF?i1?+c@RgxC!uhHGhA5s1#X#B04YItgXu> zR((RV;00CR;2bBZGJij9Jl$qHaNGG-yMyD+V zrufZ{kUOHuHtXqD*6Y+gKaKbrZs-9mxT5ooT2rdTYs-nQ7R@SEd6ubHX*d)bZnLRi z@1*bL%ui>8h4S=QCIo8ir|X1lB~s-x^BSh=sn&0z9Jdtg9rN|D{Jt>L=c+ZgeE305 zL{?!(<>Ku7BP|zPX$PFqnRPwqzbk+PbE94FsyDv=h1axHd;A((+<3Ow3+b%h zU7IWzX{FWXwn<;q>~L8g!J7MWW!vMeY;e>37Sm=W)tpbBVV){nJT9vZEzdlR-hNx* zjJn0BCRaP@VP zxs~{PPPXS;7W4P8saixJQyHY9JB;JbkHlWYuvfFt$sft=)rRy9k7JBSy_3G|kX^~d zEZo^)%tb0{FgNCsU^u>Ri*?Sx=<4zMpYC#e9m1t?93zK$gDlU-+&ISPAJgS%@M7ua zRzkQChd+^iZKC<~XK!i?$sb+)HjMX$Cd(@X{hEr`lo%mAQoQGUR#C$Tp^{4gVG>!M ziIMm&*%JkHmpUp{{o*n>{5KKy^n=#_z_-W#kWs*K%qM;J>Ky>J?zF-osuh)JJnEP9 zWd!B6jj$&<$|&F@<`bI5EeUgDA;dc>>kjEK#hwS}+(x=RVbeWUx2gEugeblV%I=iEl!}9QLm-!AXeZE6T?LWx)9=33I8H zwS2o=d@pOcI(x75$1+U#7uFyj-r^ajta8jHHdf`FRFg9&g`S@H&PmMS%Xb^w-xJw& zt|rt(>FJ4T4&wGRc8vu=Pfw)9aypTi(}~e0qhQv=#F}pGc6u?8H`l_1i-71YDj3gV zQXffu>>%&67Orj;dlDxwB{%#T0p>@ao_M~^KNoXJV&%0n;b4$sPa)>gflVPymA#1y z{(RY}&ugcAd5d>GiN}P$XAKJAEpETvoDF=%nsm}1y;ArnOpq$yF+%6jt-Lf#Hx@lB zM;IZ=3s+Of`;MK`&qCn@raS?x;8YIg(maR2FoS}_@fN#HT=%)UNwE(D6JZ1&LIuI+?L!DPad5DKtb;>>vC#AHKQk|3 zCx|{}2Jzy@RSI3Fnsyf;zG!aFF)TS{F;UCIF;atIgrA6P?+{(pXg0wW;a z(2peea1#QYtie+S@IU42Z3fA@K(6xf-suEn9UzZI;+*Y(Yz(?7a)rQ*(M;w8d|#<$cKUt+Svmc_(Gla zA(*-WSsuu({#YxhW?O*#1s}2-vQb1J(7e7_vzI4z36=oI3&w05;r@Fp zfnFWRd&2R1ptXPxTf!lnxfPJjfqcUsZwcu^ylh`X1K~acIbm1v#_zNMdXN+Lqxgv3 zP#!>x?8VsNy-+?ukFbv*oRQuLEFuJP5d8H)1S8%dq!0uI4iO1Ueh6T0N(lHsBMA8u zjO_E%qYCh7DsdGN0p|S?V_@DNNVbT#u;wUW7zDIesz?J5Uo z9`f?}r@Pj~Xy_UNV!2&tDB?sp)hZmz9=0>=YoM(PFnu16ntd-VUK zs()4U7q`Dx)&BRsfji0fAAJLSvG%g|u#T`kM@WI^N7fgtpMW&PIt+dX|HbF88bCdT z1NzmTSnW0+a7h2+U;;7|2+szAY(SO5!gaC$>%2jJ2_SnBpzoksD}xUBy%vA(5U8hr zpNBeP)w>_ST^;?^@jkEy>JZ33)Ghza&;MkH{I)+11z!>k4Lwj2y?_l}tshi7>>0Zm zJH*MYs<%|fREt$_sxnna;4HxPQ|(tBSAC&+16oc1Hyr-0onh`;o+Y<`}baj zGX8rHLAm_zG5S5%e?LK1s}Ju|UTI!6UUptZUQJ#TUL@}(@T<;i3?ymZ&Ac|e!eG{t zSB}?=cjLeE@vCBiKm&CU)OmmP;_A8Q3RwQ?G$=lQP<=dn%6b2c_upfb`dgIH35ffB zpCNg`PG0r2dWQV+`75t*h5vSf?ggH3h$Dap9^@4|9fHBxBlR~M=j*?O%2ksZDZR7WDI15mO!hat-z^gkB~x}pf%CP zKV|47)Ca%NIk=i5Wwa5PHAI`Cb)Yly_h`x_lz=k*hj1^k7ABf_JcW z2oZz?LK=LlD+8|D2tDvUyA!-O*dz8JToDHVJAY71JjiV{;shcAk%+j2NJnHN@)5;| z>xc?O6`~H&gm{2>jOax4AYLGbz&8*LF^c$#m_aNckw|tV4^j{*f|NwcAeE5nNFAgh zawpOTxf|(<^g{X}gOL%)XkW0SoX8{v4pWiv7BN_V##7DWVy*w%kqGwljQ}=Tb5ClX%qs* zg%Uz-LMfwkP^Kt*lpE?03Xh6KokwM$3Q-l{6lh2Fq28j#P;+QDv>;jvtqf{oiQbF$ zL5HDZ(HGF!=xgX&^dod1`W^Z!dYP4nRg6`MRhQL@bswuA>k-xj)(qAX)*99}aB9<7 zXV}=-gg{?tgMM*kJIr>B?E+gaTP53lwr6Y~*rwUp*+tlu*bUh2*}d7r*%R1F>^Iq4 z*n8P&>_0fTI3zjLIm|iSID$D&a%6Da;ArOP(am{fH za4T|~a(i$`a3^t>a^L43jl1|$t|{9 zqPLW7c_p((Mn}e9CR3(eW>!{C)MdG@<#Gu@`dux71$JX z6ao}-6?zm|6g3t76>}83l~|OtlyFM9O3${kZpCa3-deb|UzuCkL^)EqO!=LPkcy4U zNtHU4iEUf9xou0?*1m07RZ}$(e7_B;38?K-JE_*7Hl?np?xUWg{#=7c!(1ay1yeQ>sIMb>M835=-tr!q%WtB)i2inV6fT1+u(}9TSF;BFT(=Ew?>NYX^iQ8GnAQy*?BYSPJx|!cV_P#G2d+NXI^Ih z)k4$an8kfdR!eKk6w7`qaVsCI>sDWPY43{J^~jpr+Q~ZG`n`>!4c?}~mc@3LZL00C zos3(Jyxq5LU(CKvS5a4->s>c?H)ppJw`upC?&?8vDki;>TfXLEgXd-II*D)Pni z6Z7W_a0LTbEUwfR$`)o6u@yxY(Tkmn+pp?ey&tDg+b^rEt6J~q+^MLRsV3J5)|{_J z))H&K*7?`Hz3Y1SS^cj1)&{+Xnnsnz(xy#KIrrAzOK#?DKGnR^64f$&KlJ|C1HT6! zT0L8b9`1Yi{E_1$YMXW2lgH+dAGVvcw>&X;($t~bQBTpK)OBig)^=%j)jZXFT0_;M z)^=-m-|fNlG(6LP*4%5<`=HOP@9}f1=bbO?UOaob_vJvpNB`Ra-+@nqfrH;(MZQ`X zIu2g!&%a*#I%7m^r0|Wxo65IZZ|}V`d)M{e>HW|LpAVxS2_F|}r|3NN42C4*`X}{I zO`pv__k3~xLK_VoT^LIk=N~6e$WK&H8c%k8-S?ID4gYOr>H<@QS^9nZ_tt5L>5-YB znT6T&KSX|9o70(VpLdyOEF4*6U(8xkSZY|dULIZvT3G@Ab2#j1H>$dBrz72HkQW)7 zvbKZ=H#*ixolGy=5~b9x)7uzxb}YNwv}mfTsO2a#*{Y&cjK-cBrb5v-A!6Qpq!CX} z;UCw_Gto5KS%M>dNt%UmyS#9Lppb^4jPf(oBRdU0^@%ffk5nyi zc7ru*SJ$kebc!$vy9CoRrKvkfa`M~!$cgHclZraR*b$@6Z#~S@=X&}KsaKup&8;h9 zVJgB~32Bnx=~O$GYVjhH`$huAK}xrC}XQKF}a zMhpQ(LjFXoNSI2$A|WmLf==c*MU2)LmqNe=4#qUNU)0I$3>t%C<0WBo-KRihc= zG-d_}wJEOZKFH*92Mfe0ho6d#BH0gI2vtE*q8B%9V)V)bz%qUYo}_Lu_w_7c@OqbH zEOi@RWt$2i?akLja_9QmG5M7cfY|{suh80t5>EP~B-G7pgS zwtVyuz$Q(ZN>j1@0I|vQqsPcTx_rwgz>blDq3C&VFsj}Gq*Tx&lY>=7ixezWO;00; zU)|GsZWX3lh${8!e!88IPNo^DXpwQ2FlUTuIrGlLVPeu-M>uWa*aj*qZCN6L;9Ui& zv&L4{{oVBi!!1G3To6G;einF;pPf}Db5WLrB^f92D{bw-AnqHa_KYTTrT$dUV*qB| zQw$ASl7yiw4bjQ?w1rNcb}`y=Vlsi^IMMRs-v;Ckc!o3vWuP;~j(kQ`{C zkup`yqJE_sHtKsXMo3$*Py9KNj!9iAZNtzqra>m%BRl6o`05t*Ef)ybB>YM(#$^=+ z8sNezx?r$$K(Dk7G6R?{beoxcAqHrszs|bZJ0Ng|2Er3=E_~*aKuGK=Jmf@5EIi zzz+xq45+>T<(qiO|$-%UfXiF03#mc zDEZ=3;%G;3N!v2+9F&0nOOS9}_Fw~+lJl8I0gx8soVtbUSWAY^8*nBGEmUm7kHk~~ z2gGyC^8B$wZewM&aEJV!{kT}TqC@YYf*2Vw;z2%d-^PT1G9O&2el!<}O{yALIq*KZ z4Z?1ZO<{JWzP3qR8WmvZ84NzH9;gfY@Q+$9!u?U#HrJzWLEIBaZG=D=GSoW2wypW_ zIp&K_G3@QhmCXz;aHt=YT--@L5>u6fyIuL_@m#BYq=xcL{O?o&ub*YE_Y|6EFrv*# zMf!SL3~(H|j(KM4yaDHt&_b;YRT%VTo8j9;vQZ198T9-ZohsKz5?V+KB3CkfwfhcA z)?EA#Iqg;=v~2B4d`hf>mURj+8o?=a(6i;yTq{JKZ+r51BVYjKg!rK!lg+G-tz2LI0?rfuPnZq?Xvyc zMYLa@jcM_tl1%-~*VLKT#hXTPh}0JCYmpL5yB!p}6iT)68EOvF@hiC<0=}P^V7CXe_fgfy-ntKFFlL^4(&6O!vDMA<2oY$E#w7SO}gEiI3hbz8PwojwlEm zsVx}p4~@KcNw98EUgH4YeiHE)EH0Eo6Ttdd_}(QYe>L5a=c= zKdaf7x!ltqKb~X9itSWCQC~?i^mEDQ%YUhXO5*{{n-4WicQ@C`8^*HZ25Xd6XEn*n zanp6rT1(FtV_(1?wI(NyG|ni?rDiz7lrTj5759u?*gLjk}X^lCb&&8jPpuc2u(2`3H z=~^;CaZ`x9-w@Q~-BUX>XcBz(@r1k!(eus@TC%A=UHc{SWHSU`wHl7}vryFd&JFLL z`2KZ|5ToO!t=y#ynW3rk_5JSx<+s(Bj4KyopVUV0o;ae@`B5Je#*`_>w%2+boH(k} zNz+%T@UU?^6xg|mR&nOel|_z84>gvwi#I)&Rc8&z>qGOpVu)MihxWA;3}@aHpr+Oa zEe4T!NYRc|4db0;xoLy(fn29WwHoacRY;PUBb9WBMwUBGJ#*L?6Gd9| zqLSziI`)@G^CVG}+kEGm9D>LcS=DriRLA7C1~)`@TbJbB?b>n^Y6r<<=g$-g%q@pkrl>y@)z#$ z9mhv=rI>V$OZbV-p5|-*VZ|kHQ5Or&;i<4gy~|?0aSN#hV!1XiRkdTtIGwEcukMB@ z+9zMUX+5i}icbDl8F>r2R2jyqsBcLeO|q*LbEoEt)b3+%KXRBzvSyj*`=33{cm|6GVxm}+uQt&P_N zZNB??&!i8I=p!cI?(fbfKTBBhi+VeI)o#dD}2bll#e2Qc$HEhR%YX@pslblb0ajw*R|m$g7S>A zPL0Of$rEY3t)f{ZO*%v2K$k(rv9HTV;-5P#^m;fX&54=c2E9Dr-F(cDx98!H@t^{+ zY0QPqZNuZ!`AgPDT}@|d$3K{0CI}mLTGcZ~jRxZl1DwR!nNNb0%)9EBHNste+IDV@ zF65xx9&(OOi=PVhO~)I)KOs{fwlcq7oL->6)%7&FKe&k{jpwFkiX$)OVcOou#1Bf~ z^e`#8tIAG$M^Svgcv4v|zG@BWfj?e=*=QM;30FKw*_k z3;n@@8&4?orkS}ilfkY_@&Xi-`dw9cjAZ{YE%_l4fwmJEc^7MESw>$SnT3*&V5a84 zOL>u+PEquEko~L}+flpuJ~`utywVxw4i}db)i#696AGnQZ%`<2n`##nd(~@WY2pO8 z8x?_}RO@@?#n`8{Q??l8z3y7_4B?;(b}T`iCJN@G+)~-+4%3qM?sTY?b0VinZ~P*W z!@l@v;QdrvL7-#F@NPTn>tkUQW6vwGy`35+gC8%+S7!C?)~)KB{oaCgpWNb5i`hbv zqgAJ6+^kutbZCk85trX6E^jI z$RMRSfR$avJThI6!!bpQu}7CE8v$0uSk>f0!;y&uLCemnAR0Hoa;<$^hV;yiud*fK zcVd*Us<<~Xvl_p=+>fF>#!>|7)oHPAvn_Go4T2109BNgZ)}=|8y^`?UPyBvv@rznP zV60`DFa@WOb9yndgFjB*Df&CFL!{~!${MgTeXORPuh9=bTVF8dZJEXe>`Vs}N0*1L z^?V!H;1H>{g@OWR*T=Suw2?GriBS*K*s?dq@`QY; z43GxL9lce7B;6UhiPHq5?447sp00qPSDy;%r8t*#DU{0ewnn~G$6j0o%~Wuw;LcUw z5;rcfQG@3=Nb1Gp3`?#dZ(j;8;k~SKAp0(BOo5n4t;pr@*=vETevq@*Y+yfJ^=q-l zu%A{+nj!3mbF0&DG7`f?ni?c16V z@ILV}VE2&mIy3LdxKb<2eJnpth`Wle`S5^l*I(MA;HX=-P`uPI8a2|qkl)1SZq+=y zO(L(K;Xa``uxqNiWnjEM&Ff)BYYpzvVEZF+KY#$(s5{Q+reE&T+A z&)dhs9xvK;UoGyt#V*Qh)X%(Z_fX<+*W@>kUAjESZmO*`Fn0Ii28u4uWg0+pDiC6> zd#Sf@i*p@g_eq|{`I-iUOo2c};gw5}uc3e!EiI6*$a3%YMe|i;3Fgo$GO;UU6?sgp zlkhf<$bLJRM;093-LyPqm+^1-Tlp z;gpL#nMGBmlCu;deq0q#2=_{*|h%@gnsO5dP zlsraLVB(zpQP%*jE$waoSUaOK&FfhC{WFVckSjNSpJp2D;c-^wYKqk9WqHvieEFms>w#W6Tp6IuCiYV0JKMX;l1KFx6lib0FrX^-xMGm=0Xh0yvp$5w?9 zRb&P#($(n!Fg^`t2Lb@>zC)d=bog)VK*v_Ad9G$go2~(SM!`9IkRXnsgI)sdekH?G zAM|7;cG(w7A|Rv>X@0Vkf;hWNQW77Z+E&=sdwqV zW;!O_`yp>{(|CQTw-e^cR0%_uXDGliqfWmc2T)7qS_!!c<8HW&ZLlJ}bgl-ns9{lV zneiy~ZgeiD3wB4im3sTR8GiG?M2JS-p!>n#?QYBOAaJ{vpv55Y!Kru{d@y6u00K|1 zn474xdl*?B+Nr(jZL%I3hIdDZ)&{g4tLOt%UlVB0aRZ<8oJ6$7ZZU%10i=Z$Lpvr3 zh9YKkuhO~(Y6kI4$*_m9*WZ|HgB907qF`rbAT5x^gN%Hj09{u;Hv(e=I*~2cxHm@L zrMc&gZV&X4D4d%RuYL5S6FQ>YgTWDncASXIp*%dWQ@}N8%Wm?V6vz3#*-zRVjz>7A z&oG|Y`P4KO!_)Y^W=h?2bX9e0 zDt%S8StIWAs_M=Zmhthskuh%Ud6!?VwMZmxY=GjIOAw9xyfR5piu(f0E)oyZXTidu ztWxLY76Lf0V#>dFg+NBMfHs%q7noA-%x32kT`Lh(+(3He-YZkM`M2B0DE&85h@}L4 zYf_7I3S|hgH;g28xkG|}WG-bR5Ng%6mnV_ogE8s=4*UMpiD?tT%EFGkP~vllu1W)0m#{a>rH(ajJKbM zHGQ1g@!9q9Y#C61p7ZK#dJUc%nbj0eU}$JBl8lRxLUR3j;3d@TQZ!t8J+&i zbF~2M7&+j2v=4A+K&|m@E(h6`V7Mv1d5OUoZ2Yu>; zF<!90s8E?^Z%&)&IRu0uQEur{A)#JtevZ0M8`wIYM&~@K$C&wL<~x zBjG$}4Krn`d#-k3ENpnV#itAPpi zohIMnYs~2C?p=kH0E)c1R09Jhk?}FDN-f#9;S$7L;>)%dMw&9e-K_nD0(T#O-cmiz z0`4{bl%Wf#|J|FltiU!@>%YEP8xQQDo3)U=2eAKjv(^U4zi!s<0rLO$X6-4k7W#p% zH6nj|)`1w`f{z0Tst8=(&;*|fF$4lDg+T1wf@y8@J$I -Date: Thu Jun 29 11:52:43 2023 +0100 - - ld64: add support for -no_fixup_chains - - This is added in later versions, and is required if we want to be able - to disable fixup_chains, for use in security tests. - -diff --git a/cctools/ld64/src/ld/Options.cpp b/cctools/ld64/src/ld/Options.cpp -index 15e8e88..b6580af 100644 ---- a/cctools/ld64/src/ld/Options.cpp -+++ b/cctools/ld64/src/ld/Options.cpp -@@ -4128,6 +4128,9 @@ void Options::parse(int argc, const char* argv[]) - else if ( strcmp(arg, "-fixup_chains") == 0 ) { - fMakeChainedFixups = true; - } -+ else if ( strcmp(arg, "-no_fixup_chains") == 0 ) { -+ fMakeChainedFixups = false; -+ } - else if (strcmp(arg, "-debug_variant") == 0) { - fDebugVariant = true; - } diff --git a/depends/patches/native_libtapi/disable_zlib.patch b/depends/patches/native_libtapi/disable_zlib.patch new file mode 100644 index 00000000000000..6c7691214aa84c --- /dev/null +++ b/depends/patches/native_libtapi/disable_zlib.patch @@ -0,0 +1,17 @@ +build: disable zlib + +This isn't needed, and causes issues when clang-tblgen +is built, but trys to reach for a system libz.so. + +diff --git a/build.sh b/build.sh +index e25d2f732..ec8422621 100755 +--- a/build.sh ++++ b/build.sh +@@ -66,6 +66,7 @@ cmake ../src/llvm \ + -DCMAKE_INSTALL_PREFIX=$INSTALLPREFIX \ + -DTAPI_REPOSITORY_STRING=$TAPI_VERSION \ + -DTAPI_FULL_VERSION=$TAPI_VERSION \ ++ -DLLVM_ENABLE_ZLIB=OFF \ + $CMAKE_EXTRA_ARGS + + echo "" diff --git a/doc/build-osx.md b/doc/build-osx.md index f11ed97e098aeb..76fc3579569bdc 100644 --- a/doc/build-osx.md +++ b/doc/build-osx.md @@ -163,14 +163,8 @@ brew install python #### Deploy Dependencies -You can deploy a `.dmg` containing the Bitcoin Core application using `make deploy`. -This command depends on a couple of python packages, so it is required that you have `python` installed. - -Ensuring that `python` is installed, you can install the deploy dependencies by running the following commands in your terminal: - -``` bash -pip3 install ds_store mac_alias -``` +You can deploy a `.zip` containing the Bitcoin Core application using `make deploy`. +It is required that you have `python` installed. ## Building Bitcoin Core @@ -230,7 +224,7 @@ make check # Run tests if Python 3 is available ### 3. Deploy (optional) -You can also create a `.dmg` containing the `.app` bundle by running the following command: +You can also create a `.zip` containing the `.app` bundle by running the following command: ``` bash make deploy diff --git a/doc/build-unix.md b/doc/build-unix.md index 848dd5f71ac804..bf367fc4216b13 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -55,7 +55,7 @@ SQLite is required for the descriptor wallet: sudo apt install libsqlite3-dev Berkeley DB is only required for the legacy wallet. Ubuntu and Debian have their own `libdb-dev` and `libdb++-dev` packages, -but these will install Berkeley DB 5.1 or later. This will break binary wallet compatibility with the distributed +but these will install Berkeley DB 5.3 or later. This will break binary wallet compatibility with the distributed executables, which are based on BerkeleyDB 4.8. If you do not care about wallet compatibility, pass `--with-incompatible-bdb` to configure. Otherwise, you can build Berkeley DB [yourself](#berkeley-db). @@ -111,11 +111,7 @@ SQLite is required for the descriptor wallet: sudo dnf install sqlite-devel -Berkeley DB is required for the legacy wallet: - - sudo dnf install libdb4-devel libdb4-cxx-devel - -Berkeley DB is only required for the legacy wallet. Newer Fedora releases have only `libdb-devel` and `libdb-cxx-devel` packages, but these will install +Berkeley DB is only required for the legacy wallet. Fedora releases have only `libdb-devel` and `libdb-cxx-devel` packages, but these will install Berkeley DB 5.3 or later. This will break binary wallet compatibility with the distributed executables, which are based on Berkeley DB 4.8. If you do not care about wallet compatibility, pass `--with-incompatible-bdb` to configure. Otherwise, you can build Berkeley DB [yourself](#berkeley-db). diff --git a/doc/policy/packages.md b/doc/policy/packages.md index 2a5758318a9b65..399ae945d52db9 100644 --- a/doc/policy/packages.md +++ b/doc/policy/packages.md @@ -18,16 +18,19 @@ tip or some preceding transaction in the package. The following rules are enforced for all packages: -* Packages cannot exceed `MAX_PACKAGE_COUNT=25` count and `MAX_PACKAGE_SIZE=101KvB` total size +* Packages cannot exceed `MAX_PACKAGE_COUNT=25` count and `MAX_PACKAGE_WEIGHT=404000` total weight (#20833) - - *Rationale*: This is already enforced as mempool ancestor/descendant limits. If - transactions in a package are all related, exceeding this limit would mean that the package - can either be split up or it wouldn't pass individual mempool policy. + - *Rationale*: We want package size to be as small as possible to mitigate DoS via package + validation. However, we want to make sure that the limit does not restrict ancestor + packages that would be allowed if submitted individually. - Note that, if these mempool limits change, package limits should be reconsidered. Users may also configure their mempool limits differently. + - Note that the this is transaction weight, not "virtual" size as with other limits to allow + simpler context-less checks. + * Packages must be topologically sorted. (#20833) * Packages cannot have conflicting transactions, i.e. no two transactions in a package can spend diff --git a/doc/reduce-memory.md b/doc/reduce-memory.md index dbe88d45f32ac6..2710797ef3abbd 100644 --- a/doc/reduce-memory.md +++ b/doc/reduce-memory.md @@ -43,7 +43,7 @@ threads take up 8MiB for the thread stack on a 64-bit system, and 4MiB in a ## Linux specific -By default, glibc will create up to two heap arenas per core. This is known to cause excessive memory usage in some scenarios. To avoid this make a script that sets `MALLOC_ARENA_MAX` before starting bitcoind: +By default, glibc's implementation of `malloc` may use more than one arena. This is known to cause excessive memory usage in some scenarios. To avoid this, make a script that sets `MALLOC_ARENA_MAX` before starting bitcoind: ```bash #!/usr/bin/env bash diff --git a/doc/release-notes-27213.md b/doc/release-notes-27213.md new file mode 100644 index 00000000000000..3b478f11b79298 --- /dev/null +++ b/doc/release-notes-27213.md @@ -0,0 +1,8 @@ +P2P and network changes +------ + +- Nodes with multiple reachable networks will actively try to have at least one + outbound connection to each network. This improves individual resistance to + eclipse attacks and network level resistance to partition attacks. Users no + longer need to perform active measures to ensure being connected to multiple + enabled networks. diff --git a/doc/release-notes-28217.md b/doc/release-notes-28217.md new file mode 100644 index 00000000000000..6f771b9f9c7fd5 --- /dev/null +++ b/doc/release-notes-28217.md @@ -0,0 +1,4 @@ +P2P and network changes +----------------------- + +- Changing the default setting of -permitbaremultisig to false. Non-P2SH multisig transactions will no longer be relayed by default. (#28217) \ No newline at end of file diff --git a/doc/release-notes-28354.md b/doc/release-notes-28354.md new file mode 100644 index 00000000000000..03d421eb426d1b --- /dev/null +++ b/doc/release-notes-28354.md @@ -0,0 +1,6 @@ +Tests +----- + +- Non-standard transactions are now disabled by default on testnet + for relay and mempool acceptance. The previous behaviour can be + re-enabled by setting `-acceptnonstdtxn=1`. (#28354) diff --git a/doc/release-notes-28414.md b/doc/release-notes-28414.md new file mode 100644 index 00000000000000..3f83a732525482 --- /dev/null +++ b/doc/release-notes-28414.md @@ -0,0 +1,6 @@ +RPC Wallet +---------- + +- RPC `walletprocesspsbt`, and `descriptorprocesspsbt` return object now includes field `hex` (if the transaction +is complete) containing the serialized transaction suitable for RPC `sendrawtransaction`. + diff --git a/doc/release-notes-28448.md b/doc/release-notes-28448.md new file mode 100644 index 00000000000000..d3ef8cf65ba038 --- /dev/null +++ b/doc/release-notes-28448.md @@ -0,0 +1,6 @@ +RPC +--- + +- Setting `-rpcserialversion=0` is deprecated and will be removed in + a future release. It can currently still be used by also adding + the `-deprecatedrpc=serialversion` option. (#28448) diff --git a/doc/release-process.md b/doc/release-process.md index c7a8998b8bf8e4..bdef57243bffa8 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -123,7 +123,7 @@ git -C ./guix.sigs pull ### Create the macOS SDK tarball (first time, or when SDK version changes) Create the macOS SDK tarball, see the [macdeploy -instructions](/contrib/macdeploy/README.md#deterministic-macos-dmg-notes) for +instructions](/contrib/macdeploy/README.md#deterministic-macos-app-notes) for details. ### Build and attest to build outputs diff --git a/src/.clang-tidy b/src/.clang-tidy index 84c9d5fb3a0040..4deb5a85a5e6f8 100644 --- a/src/.clang-tidy +++ b/src/.clang-tidy @@ -1,5 +1,6 @@ Checks: ' -*, +bitcoin-*, bugprone-argument-comment, bugprone-use-after-move, misc-unused-using-decls, diff --git a/src/Makefile.am b/src/Makefile.am index b48d723bc9a7f0..8905c0ad1cd250 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -24,7 +24,7 @@ check_PROGRAMS = TESTS = BENCHMARKS = -BITCOIN_INCLUDES=-I$(builddir) -I$(srcdir)/$(MINISKETCH_INCLUDE_DIR_INT) -I$(srcdir)/secp256k1/include -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) $(LEVELDB_CPPFLAGS) +BITCOIN_INCLUDES=-I$(builddir) -I$(srcdir)/$(MINISKETCH_INCLUDE_DIR_INT) -I$(srcdir)/secp256k1/include -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) LIBBITCOIN_NODE=libbitcoin_node.a LIBBITCOIN_COMMON=libbitcoin_common.a @@ -117,6 +117,7 @@ endif .PHONY: FORCE check-symbols check-security # bitcoin core # BITCOIN_CORE_H = \ + addresstype.h \ addrdb.h \ addrman.h \ addrman_impl.h \ @@ -124,6 +125,7 @@ BITCOIN_CORE_H = \ banman.h \ base58.h \ bech32.h \ + bip324.h \ blockencodings.h \ blockfilter.h \ chain.h \ @@ -184,10 +186,13 @@ BITCOIN_CORE_H = \ kernel/coinstats.h \ kernel/context.h \ kernel/cs_main.h \ + kernel/disconnected_transactions.h \ kernel/mempool_entry.h \ kernel/mempool_limits.h \ kernel/mempool_options.h \ kernel/mempool_persist.h \ + kernel/mempool_removal_reason.h \ + kernel/messagestartchars.h \ kernel/notifications_interface.h \ kernel/validation_cache_sizes.h \ key.h \ @@ -264,7 +269,7 @@ BITCOIN_CORE_H = \ script/sigcache.h \ script/sign.h \ script/signingprovider.h \ - script/standard.h \ + script/solver.h \ shutdown.h \ signet.h \ streams.h \ @@ -370,12 +375,13 @@ obj/build.h: FORCE libbitcoin_util_a-clientversion.$(OBJEXT): obj/build.h # node # -libbitcoin_node_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) $(MINIUPNPC_CPPFLAGS) $(NATPMP_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) +libbitcoin_node_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(LEVELDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(MINIUPNPC_CPPFLAGS) $(NATPMP_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) libbitcoin_node_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_node_a_SOURCES = \ addrdb.cpp \ addrman.cpp \ banman.cpp \ + bip324.cpp \ blockencodings.cpp \ blockfilter.cpp \ chain.cpp \ @@ -398,6 +404,7 @@ libbitcoin_node_a_SOURCES = \ kernel/context.cpp \ kernel/cs_main.cpp \ kernel/mempool_persist.cpp \ + kernel/mempool_removal_reason.cpp \ mapport.cpp \ net.cpp \ net_processing.cpp \ @@ -546,10 +553,10 @@ crypto_libbitcoin_crypto_base_la_LDFLAGS = $(AM_LDFLAGS) -static crypto_libbitcoin_crypto_base_la_SOURCES = \ crypto/aes.cpp \ crypto/aes.h \ - crypto/chacha_poly_aead.h \ - crypto/chacha_poly_aead.cpp \ crypto/chacha20.h \ crypto/chacha20.cpp \ + crypto/chacha20poly1305.h \ + crypto/chacha20poly1305.cpp \ crypto/common.h \ crypto/hkdf_sha256_32.cpp \ crypto/hkdf_sha256_32.h \ @@ -657,6 +664,7 @@ libbitcoin_consensus_a_SOURCES = \ libbitcoin_common_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) libbitcoin_common_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_common_a_SOURCES = \ + addresstype.cpp \ base58.cpp \ bech32.cpp \ chainparams.cpp \ @@ -697,7 +705,7 @@ libbitcoin_common_a_SOURCES = \ script/miniscript.cpp \ script/sign.cpp \ script/signingprovider.cpp \ - script/standard.cpp \ + script/solver.cpp \ warnings.cpp \ $(BITCOIN_CORE_H) @@ -936,6 +944,7 @@ libbitcoinkernel_la_SOURCES = \ kernel/context.cpp \ kernel/cs_main.cpp \ kernel/mempool_persist.cpp \ + kernel/mempool_removal_reason.cpp \ key.cpp \ logging.cpp \ node/blockstorage.cpp \ @@ -958,7 +967,7 @@ libbitcoinkernel_la_SOURCES = \ script/script.cpp \ script/script_error.cpp \ script/sigcache.cpp \ - script/standard.cpp \ + script/solver.cpp \ signet.cpp \ streams.cpp \ support/cleanse.cpp \ diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index 51bfb1e4593bdf..28b779a5a88e65 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -22,13 +22,13 @@ bench_bench_bitcoin_SOURCES = \ bench/block_assemble.cpp \ bench/ccoins_caching.cpp \ bench/chacha20.cpp \ - bench/chacha_poly_aead.cpp \ bench/checkblock.cpp \ bench/checkqueue.cpp \ bench/crypto_hash.cpp \ bench/data.cpp \ bench/data.h \ bench/descriptors.cpp \ + bench/disconnected_transactions.cpp \ bench/duplicate_inputs.cpp \ bench/ellswift.cpp \ bench/examples.cpp \ diff --git a/src/Makefile.qt_locale.include b/src/Makefile.qt_locale.include index c44cf61a1a827b..d7f586f3dcb356 100644 --- a/src/Makefile.qt_locale.include +++ b/src/Makefile.qt_locale.include @@ -2,15 +2,20 @@ QT_TS = \ qt/locale/bitcoin_am.ts \ qt/locale/bitcoin_ar.ts \ qt/locale/bitcoin_az.ts \ + qt/locale/bitcoin_az@latin.ts \ qt/locale/bitcoin_be.ts \ qt/locale/bitcoin_bg.ts \ qt/locale/bitcoin_bn.ts \ + qt/locale/bitcoin_br.ts \ qt/locale/bitcoin_bs.ts \ qt/locale/bitcoin_ca.ts \ + qt/locale/bitcoin_cmn.ts \ qt/locale/bitcoin_cs.ts \ qt/locale/bitcoin_cy.ts \ qt/locale/bitcoin_da.ts \ qt/locale/bitcoin_de.ts \ + qt/locale/bitcoin_de_AT.ts \ + qt/locale/bitcoin_de_CH.ts \ qt/locale/bitcoin_el.ts \ qt/locale/bitcoin_en.ts \ qt/locale/bitcoin_eo.ts \ @@ -19,6 +24,7 @@ QT_TS = \ qt/locale/bitcoin_es_CO.ts \ qt/locale/bitcoin_es_DO.ts \ qt/locale/bitcoin_es_MX.ts \ + qt/locale/bitcoin_es_SV.ts \ qt/locale/bitcoin_es_VE.ts \ qt/locale/bitcoin_et.ts \ qt/locale/bitcoin_eu.ts \ @@ -26,13 +32,18 @@ QT_TS = \ qt/locale/bitcoin_fi.ts \ qt/locale/bitcoin_fil.ts \ qt/locale/bitcoin_fr.ts \ + qt/locale/bitcoin_fr_CM.ts \ + qt/locale/bitcoin_fr_LU.ts \ qt/locale/bitcoin_ga.ts \ + qt/locale/bitcoin_ga_IE.ts \ qt/locale/bitcoin_gd.ts \ qt/locale/bitcoin_gl.ts \ qt/locale/bitcoin_gl_ES.ts \ qt/locale/bitcoin_gu.ts \ qt/locale/bitcoin_ha.ts \ + qt/locale/bitcoin_hak.ts \ qt/locale/bitcoin_he.ts \ + qt/locale/bitcoin_hi.ts \ qt/locale/bitcoin_hr.ts \ qt/locale/bitcoin_hu.ts \ qt/locale/bitcoin_id.ts \ @@ -43,6 +54,7 @@ QT_TS = \ qt/locale/bitcoin_kk.ts \ qt/locale/bitcoin_kl.ts \ qt/locale/bitcoin_km.ts \ + qt/locale/bitcoin_kn.ts \ qt/locale/bitcoin_ko.ts \ qt/locale/bitcoin_ku.ts \ qt/locale/bitcoin_ku_IQ.ts \ @@ -50,9 +62,11 @@ QT_TS = \ qt/locale/bitcoin_la.ts \ qt/locale/bitcoin_lt.ts \ qt/locale/bitcoin_lv.ts \ + qt/locale/bitcoin_mg.ts \ qt/locale/bitcoin_mk.ts \ qt/locale/bitcoin_ml.ts \ qt/locale/bitcoin_mn.ts \ + qt/locale/bitcoin_mr.ts \ qt/locale/bitcoin_mr_IN.ts \ qt/locale/bitcoin_ms.ts \ qt/locale/bitcoin_my.ts \ @@ -64,6 +78,7 @@ QT_TS = \ qt/locale/bitcoin_pam.ts \ qt/locale/bitcoin_pl.ts \ qt/locale/bitcoin_pt.ts \ + qt/locale/bitcoin_pt@qtfiletype.ts \ qt/locale/bitcoin_pt_BR.ts \ qt/locale/bitcoin_ro.ts \ qt/locale/bitcoin_ru.ts \ @@ -72,8 +87,10 @@ QT_TS = \ qt/locale/bitcoin_sk.ts \ qt/locale/bitcoin_sl.ts \ qt/locale/bitcoin_sn.ts \ + qt/locale/bitcoin_so.ts \ qt/locale/bitcoin_sq.ts \ qt/locale/bitcoin_sr.ts \ + qt/locale/bitcoin_sr@ijekavianlatin.ts \ qt/locale/bitcoin_sr@latin.ts \ qt/locale/bitcoin_sv.ts \ qt/locale/bitcoin_sw.ts \ @@ -92,7 +109,9 @@ QT_TS = \ qt/locale/bitcoin_uz@Latn.ts \ qt/locale/bitcoin_vi.ts \ qt/locale/bitcoin_yo.ts \ + qt/locale/bitcoin_yue.ts \ qt/locale/bitcoin_zh-Hans.ts \ + qt/locale/bitcoin_zh-Hant.ts \ qt/locale/bitcoin_zh.ts \ qt/locale/bitcoin_zh_CN.ts \ qt/locale/bitcoin_zh_HK.ts \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index f2a82ce73b27e7..d66f5bf53a439a 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -74,6 +74,7 @@ BITCOIN_TESTS =\ test/base64_tests.cpp \ test/bech32_tests.cpp \ test/bip32_tests.cpp \ + test/bip324_tests.cpp \ test/blockchain_tests.cpp \ test/blockencodings_tests.cpp \ test/blockfilter_index_tests.cpp \ @@ -246,6 +247,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/banman.cpp \ test/fuzz/base_encode_decode.cpp \ test/fuzz/bech32.cpp \ + test/fuzz/bip324.cpp \ test/fuzz/bitdeque.cpp \ test/fuzz/block.cpp \ test/fuzz/block_header.cpp \ @@ -261,7 +263,6 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/crypto_aes256.cpp \ test/fuzz/crypto_aes256cbc.cpp \ test/fuzz/crypto_chacha20.cpp \ - test/fuzz/crypto_chacha20_poly1305_aead.cpp \ test/fuzz/crypto_common.cpp \ test/fuzz/crypto_diff_fuzz_chacha20.cpp \ test/fuzz/crypto_hkdf_hmac_sha256_l32.cpp \ @@ -299,6 +300,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/netbase_dns_lookup.cpp \ test/fuzz/node_eviction.cpp \ test/fuzz/p2p_transport_serialization.cpp \ + test/fuzz/package_eval.cpp \ test/fuzz/parse_hd_keypath.cpp \ test/fuzz/parse_numbers.cpp \ test/fuzz/parse_script.cpp \ diff --git a/src/addrdb.cpp b/src/addrdb.cpp index 0fcb5ed5c97188..8b85b77e2bfc3e 100644 --- a/src/addrdb.cpp +++ b/src/addrdb.cpp @@ -47,16 +47,16 @@ bool SerializeDB(Stream& stream, const Data& data) } template -bool SerializeFileDB(const std::string& prefix, const fs::path& path, const Data& data, int version) +bool SerializeFileDB(const std::string& prefix, const fs::path& path, const Data& data) { // Generate random temporary filename const uint16_t randv{GetRand()}; std::string tmpfn = strprintf("%s.%04x", prefix, randv); - // open temp output file, and associate with CAutoFile + // open temp output file fs::path pathTmp = gArgs.GetDataDirNet() / fs::u8path(tmpfn); FILE *file = fsbridge::fopen(pathTmp, "wb"); - CAutoFile fileout(file, SER_DISK, version); + AutoFile fileout{file}; if (fileout.IsNull()) { fileout.fclose(); remove(pathTmp); @@ -86,14 +86,14 @@ bool SerializeFileDB(const std::string& prefix, const fs::path& path, const Data } template -void DeserializeDB(Stream& stream, Data& data, bool fCheckSum = true) +void DeserializeDB(Stream& stream, Data&& data, bool fCheckSum = true) { - CHashVerifier verifier(&stream); + HashVerifier verifier{stream}; // de-serialize file header (network specific magic number) and .. - unsigned char pchMsgTmp[4]; + MessageStartChars pchMsgTmp; verifier >> pchMsgTmp; // ... verify the network matches ours - if (memcmp(pchMsgTmp, Params().MessageStart(), sizeof(pchMsgTmp))) { + if (pchMsgTmp != Params().MessageStart()) { throw std::runtime_error{"Invalid network magic number"}; } @@ -111,11 +111,10 @@ void DeserializeDB(Stream& stream, Data& data, bool fCheckSum = true) } template -void DeserializeFileDB(const fs::path& path, Data& data, int version) +void DeserializeFileDB(const fs::path& path, Data&& data) { - // open input file, and associate with CAutoFile FILE* file = fsbridge::fopen(path, "rb"); - CAutoFile filein(file, SER_DISK, version); + AutoFile filein{file}; if (filein.IsNull()) { throw DbNotFoundError{}; } @@ -175,10 +174,10 @@ bool CBanDB::Read(banmap_t& banSet) bool DumpPeerAddresses(const ArgsManager& args, const AddrMan& addr) { const auto pathAddr = args.GetDataDirNet() / "peers.dat"; - return SerializeFileDB("peers", pathAddr, addr, CLIENT_VERSION); + return SerializeFileDB("peers", pathAddr, addr); } -void ReadFromStream(AddrMan& addr, CDataStream& ssPeers) +void ReadFromStream(AddrMan& addr, DataStream& ssPeers) { DeserializeDB(ssPeers, addr, false); } @@ -191,7 +190,7 @@ util::Result> LoadAddrman(const NetGroupManager& netgro const auto start{SteadyClock::now()}; const auto path_addr{args.GetDataDirNet() / "peers.dat"}; try { - DeserializeFileDB(path_addr, *addrman, CLIENT_VERSION); + DeserializeFileDB(path_addr, *addrman); LogPrintf("Loaded %i addresses from peers.dat %dms\n", addrman->Size(), Ticks(SteadyClock::now() - start)); } catch (const DbNotFoundError&) { // Addrman can be in an inconsistent state after failure, reset it @@ -217,14 +216,14 @@ util::Result> LoadAddrman(const NetGroupManager& netgro void DumpAnchors(const fs::path& anchors_db_path, const std::vector& anchors) { LOG_TIME_SECONDS(strprintf("Flush %d outbound block-relay-only peer addresses to anchors.dat", anchors.size())); - SerializeFileDB("anchors", anchors_db_path, anchors, CLIENT_VERSION | ADDRV2_FORMAT); + SerializeFileDB("anchors", anchors_db_path, CAddress::V2_DISK(anchors)); } std::vector ReadAnchors(const fs::path& anchors_db_path) { std::vector anchors; try { - DeserializeFileDB(anchors_db_path, anchors, CLIENT_VERSION | ADDRV2_FORMAT); + DeserializeFileDB(anchors_db_path, CAddress::V2_DISK(anchors)); LogPrintf("Loaded %i addresses from %s\n", anchors.size(), fs::quoted(fs::PathToString(anchors_db_path.filename()))); } catch (const std::exception&) { anchors.clear(); diff --git a/src/addrdb.h b/src/addrdb.h index 0037495d18862d..cc3014dce29b33 100644 --- a/src/addrdb.h +++ b/src/addrdb.h @@ -16,12 +16,13 @@ class ArgsManager; class AddrMan; class CAddress; -class CDataStream; +class DataStream; class NetGroupManager; -bool DumpPeerAddresses(const ArgsManager& args, const AddrMan& addr); /** Only used by tests. */ -void ReadFromStream(AddrMan& addr, CDataStream& ssPeers); +void ReadFromStream(AddrMan& addr, DataStream& ssPeers); + +bool DumpPeerAddresses(const ArgsManager& args, const AddrMan& addr); /** Access to the banlist database (banlist.json) */ class CBanDB diff --git a/src/addresstype.cpp b/src/addresstype.cpp new file mode 100644 index 00000000000000..f199d1b4794464 --- /dev/null +++ b/src/addresstype.cpp @@ -0,0 +1,169 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://www.opensource.org/licenses/mit-license.php. + +#include + +#include +#include +#include +#include