From a6bede532cb44da77874304d9d285e3497930867 Mon Sep 17 00:00:00 2001 From: D33r-Gee Date: Fri, 8 Nov 2024 10:28:58 -0800 Subject: [PATCH] wip: Add minimal support for loading signet UTXO snapshots Adds minimal wiring to connect QML GUI to loading a signet UTXO snapshot via the connection settings. Uses SnapshotSettings.qml to allow user interaction. Modifies src/interfaces/node.h, src/node/interfaces.cpp and chainparams.cpp (temporarily for signet snapshot testing) to implement snapshot loading functionality through the node model. Current limitations: - Not integrated with onboarding process - Requires manual navigation to connection settings after initial startup - Snapshot verification progress is working, could be improved Testing: 1. Start the node 2. Complete onboarding 3. Navigate to connection settings 4. Load snapshot from provided interface --- src/interfaces/node.h | 6 ++ src/kernel/chainparams.cpp | 7 ++ src/node/interfaces.cpp | 89 ++++++++++++++++++- src/qml/components/ConnectionSettings.qml | 18 +++- src/qml/components/SnapshotSettings.qml | 76 +++++++++++++--- src/qml/models/chainmodel.cpp | 22 +++++ src/qml/models/chainmodel.h | 6 +- src/qml/models/nodemodel.cpp | 46 ++++++++++ src/qml/models/nodemodel.h | 18 +++- src/qml/pages/settings/SettingsConnection.qml | 13 +-- src/qml/pages/settings/SettingsSnapshot.qml | 6 +- 11 files changed, 278 insertions(+), 29 deletions(-) diff --git a/src/interfaces/node.h b/src/interfaces/node.h index f6c79f0c1b..cf5033b877 100644 --- a/src/interfaces/node.h +++ b/src/interfaces/node.h @@ -199,6 +199,12 @@ class Node //! List rpc commands. virtual std::vector listRpcCommands() = 0; + //! Load UTXO Snapshot. + virtual bool snapshotLoad(const std::string& path_string) = 0; + + //! Get snapshot progress. + virtual double getSnapshotProgress() = 0; + //! Set RPC timer interface if unset. virtual void rpcSetTimerInterfaceIfUnset(RPCTimerInterface* iface) = 0; diff --git a/src/kernel/chainparams.cpp b/src/kernel/chainparams.cpp index 733a3339b3..635dc40f9f 100644 --- a/src/kernel/chainparams.cpp +++ b/src/kernel/chainparams.cpp @@ -371,6 +371,13 @@ class SigNetParams : public CChainParams { vFixedSeeds.clear(); + m_assumeutxo_data = MapAssumeutxo{ + { + 160000, + {AssumeutxoHash{uint256S("0x5225141cb62dee63ab3be95f9b03d60801f264010b1816d4bd00618b2736e7be")}, 1278002}, + }, + }; + base58Prefixes[PUBKEY_ADDRESS] = std::vector(1,111); base58Prefixes[SCRIPT_ADDRESS] = std::vector(1,196); base58Prefixes[SECRET_KEY] = std::vector(1,239); diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index f1fe42206e..931a651302 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -395,9 +396,95 @@ class NodeImpl : public Node { m_context = context; } + double getSnapshotProgress() override { return m_snapshot_progress.load(); } + bool snapshotLoad(const std::string& path_string) override + { + const fs::path path = fs::u8path(path_string); + if (!fs::exists(path)) { + LogPrintf("[loadsnapshot] Snapshot file %s does not exist\n", path.u8string()); + return false; + } + + AutoFile afile{fsbridge::fopen(path, "rb")}; + if (afile.IsNull()) { + LogPrintf("[loadsnapshot] Failed to open snapshot file %s\n", path.u8string()); + return false; + } + + SnapshotMetadata metadata; + try { + afile >> metadata; + } catch (const std::exception& e) { + LogPrintf("[loadsnapshot] Failed to read snapshot metadata: %s\n", e.what()); + return false; + } + + const uint256& base_blockhash = metadata.m_base_blockhash; + LogPrintf("[loadsnapshot] Waiting for blockheader %s in headers chain before snapshot activation\n", + base_blockhash.ToString()); + + if (!m_context->chainman) { + LogPrintf("[loadsnapshot] Chainman is null\n"); + return false; + } + + ChainstateManager& chainman = *m_context->chainman; + CBlockIndex* snapshot_start_block = nullptr; + + // Wait for the block to appear in the block index + constexpr int max_wait_seconds = 600; // 10 minutes + for (int i = 0; i < max_wait_seconds; ++i) { + snapshot_start_block = WITH_LOCK(::cs_main, return chainman.m_blockman.LookupBlockIndex(base_blockhash)); + if (snapshot_start_block) break; + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + + // Snapshot Progress GUI display + COutPoint outpoint; + Coin coin; + const uint64_t coins_count = metadata.m_coins_count; + uint64_t coins_left = metadata.m_coins_count; + + LogPrintf("[loadsnapshot] Loading %d coins from snapshot %s\n", coins_count, base_blockhash.ToString()); + int64_t coins_processed{0}; + m_snapshot_progress.store(0.0); + + while (coins_left > 0) { + --coins_left; + ++coins_processed; + + if (coins_processed > 0) { + double progress = static_cast(coins_processed) / static_cast(coins_count); + m_snapshot_progress.store(progress); + if (coins_processed % 1000000 == 0) { + LogPrintf("[loadsnapshot] Progress: %.2f%% (%d/%d coins)\n", + progress * 100, coins_processed, coins_count); + } + } + } + m_snapshot_progress.store(1.0); + + if (!snapshot_start_block) { + LogPrintf("[loadsnapshot] Timed out waiting for snapshot start blockheader %s\n", base_blockhash.ToString()); + return false; + } + + // Activate the snapshot + if (!chainman.ActivateSnapshot(afile, metadata, false)) { + LogPrintf("[loadsnapshot] Unable to load UTXO snapshot %s\n", path.u8string()); + return false; + } + + CBlockIndex* new_tip = WITH_LOCK(::cs_main, return chainman.ActiveTip()); + LogPrintf("[loadsnapshot] Loaded %d coins from snapshot %s at height %d\n", + metadata.m_coins_count, new_tip->GetBlockHash().ToString(), new_tip->nHeight); + + return true; + } ArgsManager& args() { return *Assert(Assert(m_context)->args); } ChainstateManager& chainman() { return *Assert(m_context->chainman); } NodeContext* m_context{nullptr}; + std::atomic m_snapshot_progress{0.0}; }; bool FillBlock(const CBlockIndex* index, const FoundBlock& block, UniqueLock& lock, const CChain& active, const BlockManager& blockman) @@ -510,7 +597,7 @@ class RpcHandlerImpl : public Handler class ChainImpl : public Chain { public: - explicit ChainImpl(NodeContext& node) : m_node(node) {} + explicit ChainImpl(node::NodeContext& node) : m_node(node) {} std::optional getHeight() override { const int height{WITH_LOCK(::cs_main, return chainman().ActiveChain().Height())}; diff --git a/src/qml/components/ConnectionSettings.qml b/src/qml/components/ConnectionSettings.qml index e2519d4122..2efd9a2514 100644 --- a/src/qml/components/ConnectionSettings.qml +++ b/src/qml/components/ConnectionSettings.qml @@ -10,13 +10,24 @@ import "../controls" ColumnLayout { id: root signal next - property bool snapshotImported: false + property bool snapshotImported: onboarding ? false : chainModel.isSnapshotActive + property bool onboarding: false + + Component.onCompleted: { + if (!onboarding) { + snapshotImported = chainModel.isSnapshotActive + } else { + snapshotImported = false + } + } + function setSnapshotImported(imported) { snapshotImported = imported } spacing: 4 Setting { id: gotoSnapshot + visible: !root.onboarding Layout.fillWidth: true header: qsTr("Load snapshot") description: qsTr("Instant use with background sync") @@ -40,7 +51,10 @@ ColumnLayout { connectionSwipe.incrementCurrentIndex() } } - Separator { Layout.fillWidth: true } + Separator { + visible: !root.onboarding + Layout.fillWidth: true + } Setting { Layout.fillWidth: true header: qsTr("Enable listening") diff --git a/src/qml/components/SnapshotSettings.qml b/src/qml/components/SnapshotSettings.qml index ebac415b60..cc97d6318e 100644 --- a/src/qml/components/SnapshotSettings.qml +++ b/src/qml/components/SnapshotSettings.qml @@ -5,20 +5,25 @@ import QtQuick 2.15 import QtQuick.Controls 2.15 import QtQuick.Layouts 1.15 +import QtQuick.Dialogs 1.3 import "../controls" ColumnLayout { + property bool snapshotLoading: nodeModel.snapshotLoading signal snapshotImportCompleted() property int snapshotVerificationCycles: 0 property real snapshotVerificationProgress: 0 - property bool snapshotVerified: false + property bool onboarding: false + property bool snapshotVerified: onboarding ? false : chainModel.isSnapshotActive + property string snapshotFileName: "" + property var snapshotInfo: ({}) id: columnLayout width: Math.min(parent.width, 450) anchors.horizontalCenter: parent.horizontalCenter - + // TODO: remove this before release Timer { id: snapshotSimulationTimer interval: 50 // Update every 50ms @@ -29,7 +34,7 @@ ColumnLayout { snapshotVerificationProgress += 0.01 } else { snapshotVerificationCycles++ - if (snapshotVerificationCycles < 1) { + if (snapshotVerificationCycles < 3) { snapshotVerificationProgress = 0 } else { running = false @@ -42,7 +47,7 @@ ColumnLayout { StackLayout { id: settingsStack - currentIndex: 0 + currentIndex: onboarding ? 0 : snapshotVerified ? 2 : snapshotLoading ? 1 : 0 ColumnLayout { Layout.alignment: Qt.AlignHCenter @@ -78,8 +83,24 @@ ColumnLayout { Layout.alignment: Qt.AlignCenter text: qsTr("Choose snapshot file") onClicked: { - settingsStack.currentIndex = 1 - snapshotSimulationTimer.start() + fileDialog.open() + } + } + + FileDialog { + id: fileDialog + folder: shortcuts.home + selectMultiple: false + onAccepted: { + console.log("File chosen:", fileDialog.fileUrls) + snapshotFileName = fileDialog.fileUrl.toString() + console.log("Snapshot file name:", snapshotFileName) + if (snapshotFileName.endsWith(".dat")) { + nodeModel.initializeSnapshot(true, snapshotFileName) + // nodeModel.presyncProgress + } else { + console.error("Snapshot loading failed") + } } } } @@ -102,6 +123,7 @@ ColumnLayout { Layout.leftMargin: 20 Layout.rightMargin: 20 header: qsTr("Loading Snapshot") + description: qsTr("This might take a while...") } ProgressIndicator { @@ -109,10 +131,29 @@ ColumnLayout { Layout.topMargin: 20 width: 200 height: 20 - progress: snapshotVerificationProgress + progress: nodeModel.snapshotProgress Layout.alignment: Qt.AlignCenter progressColor: Theme.color.blue } + + Connections { + target: nodeModel + function onSnapshotProgressChanged() { + progressIndicator.progress = nodeModel.snapshotProgress + } + + function onSnapshotLoaded(success) { + if (success) { + chainModel.isSnapshotActiveChanged() + snapshotVerified = chainModel.isSnapshotActive + snapshotInfo = chainModel.getSnapshotInfo() + settingsStack.currentIndex = 2 // Move to the "Snapshot Loaded" page + } else { + // Handle snapshot loading failure + console.error("Snapshot loading failed") + } + } + } } ColumnLayout { @@ -137,8 +178,11 @@ ColumnLayout { descriptionColor: Theme.color.neutral6 descriptionSize: 17 descriptionLineHeight: 1.1 - description: qsTr("It contains transactions up to January 12, 2024. Newer transactions still need to be downloaded." + - " The data will be verified in the background.") + description: snapshotInfo && snapshotInfo["date"] ? + qsTr("It contains transactions up to %1. Newer transactions still need to be downloaded." + + " The data will be verified in the background.").arg(snapshotInfo["date"]) : + qsTr("It contains transactions up to DEBUG. Newer transactions still need to be downloaded." + + " The data will be verified in the background.") } ContinueButton { @@ -188,16 +232,26 @@ ColumnLayout { font.pixelSize: 14 } CoreText { - text: qsTr("200,000") + text: snapshotInfo && snapshotInfo["height"] ? + snapshotInfo["height"] : qsTr("DEBUG") Layout.alignment: Qt.AlignRight font.pixelSize: 14 } } Separator { Layout.fillWidth: true } CoreText { - text: qsTr("Hash: 0x1234567890abcdef...") + // The CoreText component displays the hash of the loaded snapshot. + text: snapshotInfo && snapshotInfo["hashSerialized"] ? + qsTr("Hash: %1").arg(snapshotInfo["hashSerialized"].substring(0, 13) + "...") : + qsTr("Hash: DEBUG") font.pixelSize: 14 } + + Component.onCompleted: { + if (snapshotVerified) { + snapshotInfo = chainModel.getSnapshotInfo() + } + } } } } diff --git a/src/qml/models/chainmodel.cpp b/src/qml/models/chainmodel.cpp index aeffe99599..d5161df273 100644 --- a/src/qml/models/chainmodel.cpp +++ b/src/qml/models/chainmodel.cpp @@ -9,9 +9,13 @@ #include #include #include +#include +#include +#include ChainModel::ChainModel(interfaces::Chain& chain) : m_chain{chain} + // m_params{Params()} { QTimer* timer = new QTimer(); connect(timer, &QTimer::timeout, this, &ChainModel::setCurrentTimeRatio); @@ -101,3 +105,21 @@ void ChainModel::setCurrentTimeRatio() Q_EMIT timeRatioListChanged(); } + +// Using hardcoded snapshot info to display in SnapshotSettings.qml +QVariantMap ChainModel::getSnapshotInfo() { + QVariantMap snapshot_info; + + const MapAssumeutxo& valid_assumeutxos_map = Params().Assumeutxo(); + if (!valid_assumeutxos_map.empty()) { + const int height = valid_assumeutxos_map.rbegin()->first; + const auto& hash_serialized = valid_assumeutxos_map.rbegin()->second.hash_serialized; + int64_t date = m_chain.getBlockTime(height); + + snapshot_info["height"] = height; + snapshot_info["hashSerialized"] = QString::fromStdString(hash_serialized.ToString()); + snapshot_info["date"] = QDateTime::fromSecsSinceEpoch(date).toString("MMMM d yyyy"); + } + + return snapshot_info; +} diff --git a/src/qml/models/chainmodel.h b/src/qml/models/chainmodel.h index 9318510eda..6a5124be7f 100644 --- a/src/qml/models/chainmodel.h +++ b/src/qml/models/chainmodel.h @@ -27,6 +27,7 @@ class ChainModel : public QObject Q_PROPERTY(quint64 assumedBlockchainSize READ assumedBlockchainSize CONSTANT) Q_PROPERTY(quint64 assumedChainstateSize READ assumedChainstateSize CONSTANT) Q_PROPERTY(QVariantList timeRatioList READ timeRatioList NOTIFY timeRatioListChanged) + Q_PROPERTY(bool isSnapshotActive READ isSnapshotActive NOTIFY isSnapshotActiveChanged) public: explicit ChainModel(interfaces::Chain& chain); @@ -36,11 +37,13 @@ class ChainModel : public QObject quint64 assumedBlockchainSize() const { return m_assumed_blockchain_size; }; quint64 assumedChainstateSize() const { return m_assumed_chainstate_size; }; QVariantList timeRatioList() const { return m_time_ratio_list; }; - + bool isSnapshotActive() const { return m_chain.hasAssumedValidChain(); }; int timestampAtMeridian(); void setCurrentTimeRatio(); + Q_INVOKABLE QVariantMap getSnapshotInfo(); + public Q_SLOTS: void setTimeRatioList(int new_time); void setTimeRatioListInitial(); @@ -48,6 +51,7 @@ public Q_SLOTS: Q_SIGNALS: void timeRatioListChanged(); void currentNetworkNameChanged(); + void isSnapshotActiveChanged(); private: QString m_current_network_name; diff --git a/src/qml/models/nodemodel.cpp b/src/qml/models/nodemodel.cpp index 521e5fa1c5..5429719c86 100644 --- a/src/qml/models/nodemodel.cpp +++ b/src/qml/models/nodemodel.cpp @@ -7,6 +7,9 @@ #include #include #include +#include +#include +#include #include #include @@ -16,6 +19,9 @@ #include #include #include +#include +#include +#include NodeModel::NodeModel(interfaces::Node& node) : m_node{node} @@ -139,6 +145,8 @@ void NodeModel::timerEvent(QTimerEvent* event) if (m_node.shutdownRequested()) { stopShutdownPolling(); Q_EMIT requestedShutdown(); + } else if (event->timerId() == m_snapshot_timer_id) { + setSnapshotProgress(m_node.getSnapshotProgress()); } } @@ -166,3 +174,41 @@ void NodeModel::ConnectToNumConnectionsChangedSignal() setNumOutboundPeers(new_num_peers.outbound_full_relay + new_num_peers.block_relay); }); } + +// Loads a snapshot from a given path using FileDialog +void NodeModel::initializeSnapshot(bool initLoadSnapshot, QString path_file) { + if (initLoadSnapshot) { + // TODO: this is to deal with FileDialog returning a QUrl + path_file = QUrl(path_file).toLocalFile(); + m_snapshot_loading = true; + Q_EMIT snapshotLoadingChanged(); + // TODO: Remove this before release + // qDebug() << "path_file: " << path_file; + QThread* snapshot_thread = new QThread(); + + m_snapshot_timer_id = startTimer(200ms); + + // Capture path_file by value + auto lambda = [this, path_file]() { + bool result = this->snapshotLoad(path_file); + m_snapshot_loading = false; + QMetaObject::invokeMethod(this, [this]() { + killTimer(m_snapshot_timer_id); + }); + Q_EMIT snapshotLoaded(result); + Q_EMIT snapshotLoadingChanged(); + }; + + connect(snapshot_thread, &QThread::started, lambda); + connect(snapshot_thread, &QThread::finished, snapshot_thread, &QThread::deleteLater); + + snapshot_thread->start(); + } +} + +void NodeModel::setSnapshotProgress(double new_progress) { + if (new_progress != m_snapshot_progress) { + m_snapshot_progress = new_progress; + Q_EMIT snapshotProgressChanged(); + } +} diff --git a/src/qml/models/nodemodel.h b/src/qml/models/nodemodel.h index a17f9b0833..d522ffbb11 100644 --- a/src/qml/models/nodemodel.h +++ b/src/qml/models/nodemodel.h @@ -34,6 +34,8 @@ class NodeModel : public QObject Q_PROPERTY(double verificationProgress READ verificationProgress NOTIFY verificationProgressChanged) Q_PROPERTY(bool pause READ pause WRITE setPause NOTIFY pauseChanged) Q_PROPERTY(bool faulted READ errorState WRITE setErrorState NOTIFY errorStateChanged) + Q_PROPERTY(double snapshotProgress READ snapshotProgress WRITE setSnapshotProgress NOTIFY snapshotProgressChanged) + Q_PROPERTY(bool snapshotLoading READ snapshotLoading NOTIFY snapshotLoadingChanged) public: explicit NodeModel(interfaces::Node& node); @@ -52,6 +54,10 @@ class NodeModel : public QObject void setPause(bool new_pause); bool errorState() const { return m_faulted; } void setErrorState(bool new_error); + bool isSnapshotLoaded() const; + double snapshotProgress() const { return m_snapshot_progress; } + void setSnapshotProgress(double new_progress); + bool snapshotLoading() const { return m_snapshot_loading; } Q_INVOKABLE float getTotalBytesReceived() const { return (float)m_node.getTotalBytesRecv(); } Q_INVOKABLE float getTotalBytesSent() const { return (float)m_node.getTotalBytesSent(); } @@ -59,6 +65,9 @@ class NodeModel : public QObject Q_INVOKABLE void startNodeInitializionThread(); Q_INVOKABLE void requestShutdown(); + Q_INVOKABLE void initializeSnapshot(bool initLoadSnapshot, QString path_file); + Q_INVOKABLE bool snapshotLoad(QString path_file) const { return m_node.snapshotLoad(path_file.toStdString()); } + void startShutdownPolling(); void stopShutdownPolling(); @@ -77,7 +86,10 @@ public Q_SLOTS: void setTimeRatioList(int new_time); void setTimeRatioListInitial(); - + void initializationFinished(); + void snapshotLoaded(bool result); + void snapshotProgressChanged(); + void snapshotLoadingChanged(); protected: void timerEvent(QTimerEvent* event) override; @@ -90,8 +102,10 @@ public Q_SLOTS: double m_verification_progress{0.0}; bool m_pause{false}; bool m_faulted{false}; - + double m_snapshot_progress{0.0}; int m_shutdown_polling_timer_id{0}; + int m_snapshot_timer_id{0}; + bool m_snapshot_loading{false}; QVector> m_block_process_time; diff --git a/src/qml/pages/settings/SettingsConnection.qml b/src/qml/pages/settings/SettingsConnection.qml index 1b2d9959d2..5ad0cb4dee 100644 --- a/src/qml/pages/settings/SettingsConnection.qml +++ b/src/qml/pages/settings/SettingsConnection.qml @@ -35,6 +35,7 @@ Item { detailActive: true detailItem: ConnectionSettings { onNext: connectionSwipe.incrementCurrentIndex() + onboarding: root.onboarding } states: [ @@ -89,19 +90,11 @@ Item { } } SettingsSnapshot { + onboarding: root.onboarding onSnapshotImportCompleted: { setSnapshotImported(true) } - onBackClicked: { - connectionSwipe.decrementCurrentIndex() - connectionSwipe.decrementCurrentIndex() - } - } - SettingsSnapshot { - onSnapshotImportCompleted: { - setSnapshotImported(true) - } - onBackClicked: { + onBack: { connectionSwipe.decrementCurrentIndex() connectionSwipe.decrementCurrentIndex() } diff --git a/src/qml/pages/settings/SettingsSnapshot.qml b/src/qml/pages/settings/SettingsSnapshot.qml index e6c557a022..9f2b27b247 100644 --- a/src/qml/pages/settings/SettingsSnapshot.qml +++ b/src/qml/pages/settings/SettingsSnapshot.qml @@ -9,8 +9,9 @@ import "../../controls" import "../../components" Page { - signal backClicked + signal back signal snapshotImportCompleted + property bool onboarding: false id: root @@ -24,12 +25,13 @@ Page { leftItem: NavButton { iconSource: "image://images/caret-left" text: qsTr("Back") - onClicked: root.backClicked() + onClicked: root.back() } } SnapshotSettings { width: Math.min(parent.width, 450) anchors.horizontalCenter: parent.horizontalCenter + onboarding: root.onboarding onSnapshotImportCompleted: { root.snapshotImportCompleted() }