[Consensus] add additional block types and store skeleton (MystenLabs…

…#15629)

## Description 

Add additional block types for I/O and data manipulations.
Add skeleton for DagState and Store. I will add more details to them
before the PR is merged.

## Test Plan 

CI

---
If your changes are not user-facing and do not break anything, you can
skip the following section. Otherwise, please briefly describe what has
changed under the Release Notes section.

### Type of Change (Check all that apply)

- [ ] protocol change
- [ ] user-visible impact
- [ ] breaking change for a client SDKs
- [ ] breaking change for FNs (FN binary must upgrade)
- [ ] breaking change for validators or node operators (must upgrade
binaries)
- [ ] breaking change for on-chain data layout
- [ ] necessitate either a data wipe or data migration

### Release notes

.gitignore

@@ -13,6 +13,7 @@
 # Move build directory
 build
 storage
+!consensus/core/src/storage
 !crates/sui-types/src/storage
 !narwhal/storage
 

consensus/config/src/committee.rs

@@ -2,7 +2,10 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use fastcrypto::traits::KeyPair;
-use std::fmt::{Display, Formatter};
+use std::{
+    fmt::{Display, Formatter},
+    ops::{Index, IndexMut},
+};
 
 use multiaddr::Multiaddr;
 use rand::rngs::StdRng;
@@ -109,11 +112,11 @@ impl Committee {
     }
 
     pub fn stake(&self, authority_index: AuthorityIndex) -> Stake {
-        self.authorities[authority_index.value()].stake
+        self.authorities[authority_index].stake
     }
 
     pub fn authority(&self, authority_index: AuthorityIndex) -> &Authority {
-        &self.authorities[authority_index.value()]
+        &self.authorities[authority_index]
     }
 
     pub fn authorities(&self) -> impl Iterator<Item = (AuthorityIndex, &Authority)> {
@@ -173,6 +176,34 @@ impl Display for AuthorityIndex {
     }
 }
 
+impl<T, const N: usize> Index<AuthorityIndex> for [T; N] {
+    type Output = T;
+
+    fn index(&self, index: AuthorityIndex) -> &Self::Output {
+        self.get(index.value()).unwrap()
+    }
+}
+
+impl<T> Index<AuthorityIndex> for Vec<T> {
+    type Output = T;
+
+    fn index(&self, index: AuthorityIndex) -> &Self::Output {
+        self.get(index.value()).unwrap()
+    }
+}
+
+impl<T, const N: usize> IndexMut<AuthorityIndex> for [T; N] {
+    fn index_mut(&mut self, index: AuthorityIndex) -> &mut Self::Output {
+        self.get_mut(index.value()).unwrap()
+    }
+}
+
+impl<T> IndexMut<AuthorityIndex> for Vec<T> {
+    fn index_mut(&mut self, index: AuthorityIndex) -> &mut Self::Output {
+        self.get_mut(index.value()).unwrap()
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use crate::{Committee, Stake};

consensus/config/src/crypto.rs

@@ -21,6 +21,8 @@ use shared_crypto::intent::INTENT_PREFIX_LENGTH;
 pub type NetworkPublicKey = ed25519::Ed25519PublicKey;
 pub type NetworkPrivateKey = ed25519::Ed25519PrivateKey;
 pub type NetworkKeyPair = ed25519::Ed25519KeyPair;
+pub type NetworkKeySignature = ed25519::Ed25519Signature;
+pub type NetworkKeySignatureAsBytes = ed25519::Ed25519SignatureAsBytes;
 
 /// Protocol key is used in random beacon.
 pub type ProtocolPublicKey = bls12381::min_sig::BLS12381PublicKey;

consensus/core/Cargo.toml

@@ -9,9 +9,11 @@ publish = false
 [dependencies]
 base64.workspace = true
 bcs.workspace = true
+bytes.workspace = true
 fastcrypto.workspace = true
 serde.workspace = true
 prometheus.workspace = true
+thiserror.workspace = true
 tokio.workspace = true
 tracing.workspace = true
 async-trait.workspace = true

consensus/core/src/block.rs

@@ -11,7 +11,7 @@ use std::{
 use fastcrypto::hash::{Digest, HashFunction};
 use serde::{Deserialize, Serialize};
 
-use consensus_config::{AuthorityIndex, DefaultHashFunction, DIGEST_LENGTH};
+use consensus_config::{AuthorityIndex, DefaultHashFunction, NetworkKeySignature, DIGEST_LENGTH};
 
 /// Round number of a block.
 pub type Round = u32;
@@ -45,7 +45,7 @@ pub trait BlockAPI {
     fn digest(&self) -> BlockDigest;
     fn round(&self) -> Round;
     fn author(&self) -> AuthorityIndex;
-    fn timestamp(&self) -> BlockTimestampMs;
+    fn timestamp_ms(&self) -> BlockTimestampMs;
     fn ancestors(&self) -> &[BlockRef];
     // TODO: add accessor for transactions.
 }
@@ -54,7 +54,7 @@ pub trait BlockAPI {
 pub struct BlockV1 {
     round: Round,
     author: AuthorityIndex,
-    timestamp: BlockTimestampMs,
+    timestamp_ms: BlockTimestampMs,
     ancestors: Vec<BlockRef>,
 
     #[serde(skip)]
@@ -86,8 +86,8 @@ impl BlockAPI for BlockV1 {
         self.author
     }
 
-    fn timestamp(&self) -> BlockTimestampMs {
-        self.timestamp
+    fn timestamp_ms(&self) -> BlockTimestampMs {
+        self.timestamp_ms
     }
 
     fn ancestors(&self) -> &[BlockRef] {
@@ -120,14 +120,6 @@ impl Hash for BlockRef {
     }
 }
 
-impl fastcrypto::hash::Hash<{ DIGEST_LENGTH }> for BlockRef {
-    type TypedDigest = BlockDigest;
-
-    fn digest(&self) -> BlockDigest {
-        self.digest
-    }
-}
-
 /// Hash of a block, covers all fields except signature.
 #[derive(Clone, Copy, Serialize, Deserialize, Default, PartialEq, Eq, PartialOrd, Ord)]
 pub struct BlockDigest([u8; consensus_config::DIGEST_LENGTH]);
@@ -154,4 +146,34 @@ impl fmt::Debug for BlockDigest {
     }
 }
 
+/// Signature of block digest by its author.
+#[allow(unused)]
+pub(crate) type BlockSignature = NetworkKeySignature;
+
+/// Unverified block only allows limited access to its content.
+#[allow(unused)]
+#[derive(Deserialize)]
+pub(crate) struct SignedBlock {
+    block: Block,
+    signature: bytes::Bytes,
+
+    #[serde(skip)]
+    serialized: bytes::Bytes,
+}
+
+impl SignedBlock {
+    // TODO: add deserialization and verification.
+}
+
+/// Verifiied block allows access to its content.
+#[allow(unused)]
+#[derive(Deserialize, Serialize)]
+pub(crate) struct VerifiedBlock {
+    pub block: Block,
+    pub signature: bytes::Bytes,
+
+    #[serde(skip)]
+    serialized: bytes::Bytes,
+}
+
 // TODO: add basic verification for BlockRef and BlockDigest computations.

consensus/core/src/commit.rs

@@ -0,0 +1,20 @@
+// Copyright (c) Mysten Labs, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+use serde::{Deserialize, Serialize};
+
+use crate::block::BlockRef;
+
+/// Specifies one consensus commit.
+/// It is stored on disk, so it does not contain blocks which are stored individually.
+#[allow(unused)]
+#[derive(Deserialize, Serialize)]
+pub(crate) struct Commit {
+    /// Index of the commit.
+    /// First commit after genesis has an index of 1, then every next commit has an index incremented by 1.
+    pub index: u64,
+    /// A reference to the the commit leader.
+    pub leader: BlockRef,
+    /// Refs to committed blocks, in the commit order.
+    pub blocks: Vec<BlockRef>,
+}

consensus/core/src/dag_state.rs

@@ -0,0 +1,69 @@
+// Copyright (c) Mysten Labs, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+use std::{
+    collections::{BTreeMap, BTreeSet},
+    sync::Arc,
+};
+
+use crate::{
+    block::{BlockAPI as _, BlockRef, Round, VerifiedBlock},
+    context::Context,
+    storage::Store,
+};
+
+/// Recent rounds of blocks to cached in memory, counted from the last committed leader round.
+#[allow(unused)]
+const BLOCK_CACHED_ROUNDS: Round = 100;
+
+/// DagState provides the API to write and read accepted blocks from the DAG.
+/// Only uncommited and last committed blocks are cached in memory.
+/// The rest of blocks are stored on disk.
+/// Refs to cached blocks and additional refs are cached as well, to speed up existence checks.
+///
+/// Note: DagState should be wrapped with Arc<RwMutex<_>>, to allow concurrent access from
+/// multiple components.
+#[allow(unused)]
+pub(crate) struct DagState {
+    context: Arc<Context>,
+
+    // Caches uncommitted blocks, and recent blocks within BLOCK_CACHED_ROUNDS from the
+    // last committed leader round.
+    recent_blocks: BTreeMap<BlockRef, VerifiedBlock>,
+
+    // All accepted blocks have their refs cached. Cached refs are never removed for now.
+    // Each element in the vector contains refs for the authority corresponding to its index.
+    cached_refs: Vec<BTreeSet<BlockRef>>,
+
+    // Persistent storage for blocks, commits and other consensus data.
+    store: Arc<dyn Store>,
+}
+
+#[allow(unused)]
+impl DagState {
+    pub(crate) fn new(
+        context: Arc<Context>,
+        blocks: Vec<VerifiedBlock>,
+        store: Arc<dyn Store>,
+    ) -> Self {
+        let num_authorities = context.committee.size();
+        let mut state = Self {
+            context,
+            recent_blocks: BTreeMap::new(),
+            cached_refs: vec![BTreeSet::new(); num_authorities],
+            store,
+        };
+
+        for block in blocks {
+            state.add_block(block);
+        }
+
+        state
+    }
+
+    pub(crate) fn add_block(&mut self, block: VerifiedBlock) {
+        let block_ref = block.block.reference();
+        self.recent_blocks.insert(block_ref, block);
+        self.cached_refs[block_ref.author].insert(block_ref);
+    }
+}

consensus/core/src/error.rs

@@ -0,0 +1,15 @@
+// Copyright (c) Mysten Labs, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+use thiserror::Error;
+
+/// Errors that can occur when processing blocks, reading from storage, or encountering shutdown.
+#[allow(unused)]
+#[derive(Clone, Debug, Error)]
+pub enum ConsensusError {
+    #[error("Error deserializing block")]
+    MalformattedBlock,
+}
+
+#[allow(unused)]
+pub type ConsensusResult<T> = Result<T, ConsensusError>;

consensus/core/src/lib.rs

@@ -3,9 +3,13 @@
 
 mod block;
 mod block_verifier;
+mod commit;
 mod context;
 mod core;
+mod dag_state;
+mod error;
 mod metrics;
 mod stake_aggregator;
+mod storage;
 mod threshold_clock;
 mod validator;

consensus/core/src/storage/mod.rs

@@ -0,0 +1,17 @@
+// Copyright (c) Mysten Labs, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+mod rocksdb;
+
+use crate::{block::VerifiedBlock, commit::Commit, error::ConsensusResult};
+
+/// A common interface for consensus storage.
+pub(crate) trait Store {
+    /// Loads last committed blocks, all uncommitted blocks and last commit from store.
+    fn recover(&self) -> ConsensusResult<(Vec<VerifiedBlock>, Commit)>;
+
+    /// Writes blocks and consensus commits to store.
+    fn write(&self, blocks: Vec<VerifiedBlock>, commits: Vec<Commit>) -> ConsensusResult<()>;
+
+    // TODO: add methods to read and scan blocks.
+}

consensus/core/src/storage/rocksdb.rs

@@ -0,0 +1,20 @@
+// Copyright (c) Mysten Labs, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::{block::VerifiedBlock, commit::Commit, error::ConsensusResult};
+
+use super::Store;
+
+/// Storage implementation using RocksDB.
+pub(crate) struct RocksDB {}
+
+#[allow(unused)]
+impl Store for RocksDB {
+    fn recover(&self) -> ConsensusResult<(Vec<VerifiedBlock>, Commit)> {
+        unimplemented!()
+    }
+
+    fn write(&self, blocks: Vec<VerifiedBlock>, commits: Vec<Commit>) -> ConsensusResult<()> {
+        unimplemented!()
+    }
+}