Remove datastore, fetch latest block (#928)

* basic redis setup in the same container * formatting * basic presignature-storage using redis * presignature storage wrapped in arc and ewlock * presignature storage redis functions implemented * redis presignature storage used in presignature manager * serialization for presignature * warnings fixed * presignature bad parameters error check * min refactoring * added redis config file path to dockerfile, added tf to mount /data volume * it fix * rust 2024 requirements * start redis in it * pull redis in it * lock file update * redis unit test * comments 1 * comments 2 * triple manager functions renamed to match presiganture manager * serialization simlified * comments 3 * added start command to mpc-node and added redis URL format * use account id as a part of redis storage key * added full redis config, modified dockerfile to use it * unnecessary port expose removed * remove local redis config * redis config update * redis file ownership change * fix persistent data issue * fix redis unit test * extend redis lifecycle * clippy * datastore triple storage removed * refactor triple storage * use redis to store triples * fix rust typing issue * 2 triple pre-check added * add link to redis in setup-env * use redis connection pool * fmt * change info to debug in storage layer * rename count to lem * develop * Fix locks being acquired incorrectly (#914) * chore: removed Arc and RwLock for triple/presignature storage (#915) * Removed Arc and RwLock for triple/presignature storage * fix conflicts * in progress * remove test output * redis app storage removed * remove starting block parameter * remove redundant datastore setup * add in memory variable for latest processed block * remove datastore users and roles from tf * datastore docker estup removed * fix tf mistake * fetch latest block using RPC * clippy * change log order * revert mpc recovery tf changes * revert mpc recovery tf changes * renamings * ignore RUSTSEC-2024-0399 * merge mistake fix --------- Co-authored-by: kmaus-near <[email protected]> Co-authored-by: Phuong Nguyen <[email protected]> Co-authored-by: Serhii Volovyk <[email protected]>
near · Nov 26, 2024 · 90e1b92 · 90e1b92
1 parent 22fd7b9
commit 90e1b92
Show file tree

Hide file tree

Showing 33 changed files with 205 additions and 1,080 deletions.
diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml
@@ -100,5 +100,6 @@ jobs:
         # RUSTSEC-2024-0344 and RUSTSEC-2022-0093 are both to do with ed25519 signatures in near-sdk, we don't sign things with this library so it's safe
         # RUSTSEC-2022-0054 wee-alloc is unmaintained, it's fine for now because we barely use an allocator and the contracts are short lived, but we should find a replacement/use the default allocator
         # RUSTSEC-2021-0145 atty can do an unallocated read with a custom allocator in windows. We don't run this in windows and we don't use a custom allocator.
+        # RUSTSEC-2024-0399 according to the description, this is not affecting us since we are not using Acceptor
         run: |
-          cargo audit --ignore RUSTSEC-2022-0093 --ignore RUSTSEC-2024-0344 --ignore RUSTSEC-2022-0054 --ignore RUSTSEC-2021-0145
+          cargo audit --ignore RUSTSEC-2022-0093 --ignore RUSTSEC-2024-0344 --ignore RUSTSEC-2022-0054 --ignore RUSTSEC-2021-0145 --ignore RUSTSEC-2024-0399
diff --git a/chain-signatures/node/src/cli.rs b/chain-signatures/node/src/cli.rs
@@ -1,6 +1,7 @@
 use crate::config::{Config, LocalConfig, NetworkConfig, OverrideConfig};
 use crate::gcp::GcpService;
 use crate::protocol::{MpcSignProtocol, SignQueue};
+use crate::storage::app_data_storage;
 use crate::{http_client, indexer, mesh, storage, web};
 use clap::Parser;
 use deadpool_redis::Runtime;
@@ -195,14 +196,6 @@ pub fn run(cmd: Cli) -> anyhow::Result<()> {
                 .build()?;
             let gcp_service =
                 rt.block_on(async { GcpService::init(&account_id, &storage_options).await })?;
-            let (indexer_handle, indexer) = indexer::run(
-                &indexer_options,
-                &mpc_contract_id,
-                &account_id,
-                &sign_queue,
-                &gcp_service,
-                &rt,
-            )?;
 
             let key_storage =
                 storage::secret_storage::init(Some(&gcp_service), &storage_options, &account_id);
@@ -214,6 +207,23 @@ pub fn run(cmd: Cli) -> anyhow::Result<()> {
             let triple_storage = storage::triple_storage::init(&redis_pool, &account_id);
             let presignature_storage =
                 storage::presignature_storage::init(&redis_pool, &account_id);
+            let app_data_storage = app_data_storage::init(&redis_pool, &account_id);
+
+            let mut rpc_client = near_fetch::Client::new(&near_rpc);
+            if let Some(referer_param) = client_header_referer {
+                let client_headers = rpc_client.inner_mut().headers_mut();
+                client_headers.insert(http::header::REFERER, referer_param.parse().unwrap());
+            }
+            tracing::info!(rpc_addr = rpc_client.rpc_addr(), "rpc client initialized");
+
+            let (indexer_handle, indexer) = indexer::run(
+                &indexer_options,
+                &mpc_contract_id,
+                &account_id,
+                &sign_queue,
+                app_data_storage,
+                rpc_client.clone(),
+            )?;
 
             let sign_sk = sign_sk.unwrap_or_else(|| account_sk.clone());
             let my_address = my_address
@@ -229,13 +239,6 @@ pub fn run(cmd: Cli) -> anyhow::Result<()> {
             let (sender, receiver) = mpsc::channel(16384);
 
             tracing::info!(%my_address, "address detected");
-            let mut rpc_client = near_fetch::Client::new(&near_rpc);
-            if let Some(referer_param) = client_header_referer {
-                let client_headers = rpc_client.inner_mut().headers_mut();
-                client_headers.insert(http::header::REFERER, referer_param.parse().unwrap());
-            }
-
-            tracing::info!(rpc_addr = rpc_client.rpc_addr(), "rpc client initialized");
             let signer = InMemorySigner::from_secret_key(account_id.clone(), account_sk);
             let (protocol, protocol_state) = MpcSignProtocol::init(
                 my_address,

diff --git a/chain-signatures/node/src/gcp/error.rs b/chain-signatures/node/src/gcp/error.rs
@@ -19,25 +19,3 @@ pub enum SecretStorageError {
     #[error("(de)serialization error: {0}")]
     SerdeError(#[from] serde_json::Error),
 }
-
-#[derive(thiserror::Error, Debug)]
-pub enum DatastoreStorageError {
-    #[error("GCP error: {0}")]
-    GcpError(#[from] google_datastore1::Error),
-    #[error("IO error: {0}")]
-    IoError(#[from] std::io::Error),
-    #[error("(de)serialization error: {0}")]
-    SerdeError(#[from] serde_json::Error),
-    #[error("datastore value conversion error: {0}")]
-    ConvertError(ConvertError),
-    #[error("fetch_entities error: `{0}`")]
-    FetchEntitiesError(String),
-    #[error("could not find entity: {0}")]
-    EntityNotFound(String),
-}
-
-impl From<ConvertError> for DatastoreStorageError {
-    fn from(err: ConvertError) -> Self {
-        DatastoreStorageError::ConvertError(err)
-    }
-}