From fc81ab76c60e6f5636cc6754024272fcf5cb3d99 Mon Sep 17 00:00:00 2001 From: mwtian <81660174+mwtian@users.noreply.github.com> Date: Wed, 10 Jan 2024 10:52:10 -0800 Subject: [PATCH] [Consensus] add additional block types and store skeleton (#15629) ## Description Add additional block types for I/O and data manipulations. Add skeleton for DagState and Store. I will add more details to them before the PR is merged. ## Test Plan CI --- If your changes are not user-facing and do not break anything, you can skip the following section. Otherwise, please briefly describe what has changed under the Release Notes section. ### Type of Change (Check all that apply) - [ ] protocol change - [ ] user-visible impact - [ ] breaking change for a client SDKs - [ ] breaking change for FNs (FN binary must upgrade) - [ ] breaking change for validators or node operators (must upgrade binaries) - [ ] breaking change for on-chain data layout - [ ] necessitate either a data wipe or data migration ### Release notes --- .gitignore | 1 + Cargo.lock | 2 + consensus/config/src/committee.rs | 37 ++++++++++++-- consensus/config/src/crypto.rs | 2 + consensus/core/Cargo.toml | 2 + consensus/core/src/block.rs | 48 ++++++++++++++----- consensus/core/src/commit.rs | 20 ++++++++ consensus/core/src/dag_state.rs | 69 +++++++++++++++++++++++++++ consensus/core/src/error.rs | 15 ++++++ consensus/core/src/lib.rs | 4 ++ consensus/core/src/storage/mod.rs | 17 +++++++ consensus/core/src/storage/rocksdb.rs | 20 ++++++++ 12 files changed, 221 insertions(+), 16 deletions(-) create mode 100644 consensus/core/src/commit.rs create mode 100644 consensus/core/src/dag_state.rs create mode 100644 consensus/core/src/error.rs create mode 100644 consensus/core/src/storage/mod.rs create mode 100644 consensus/core/src/storage/rocksdb.rs diff --git a/.gitignore b/.gitignore index b95fb9f872e05..1d8fabd6045d8 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ # Move build directory build storage +!consensus/core/src/storage !crates/sui-types/src/storage !narwhal/storage diff --git a/Cargo.lock b/Cargo.lock index 896662bcd76ff..b47bf648ddea0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2467,6 +2467,7 @@ dependencies = [ "async-trait", "base64 0.21.2", "bcs", + "bytes", "consensus-config", "enum_dispatch", "fastcrypto", @@ -2474,6 +2475,7 @@ dependencies = [ "prometheus", "serde", "sui-protocol-config", + "thiserror", "tokio", "tracing", "workspace-hack", diff --git a/consensus/config/src/committee.rs b/consensus/config/src/committee.rs index 5385c9cf03c8b..27c9d30e6fdb1 100644 --- a/consensus/config/src/committee.rs +++ b/consensus/config/src/committee.rs @@ -2,7 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 use fastcrypto::traits::KeyPair; -use std::fmt::{Display, Formatter}; +use std::{ + fmt::{Display, Formatter}, + ops::{Index, IndexMut}, +}; use multiaddr::Multiaddr; use rand::rngs::StdRng; @@ -109,11 +112,11 @@ impl Committee { } pub fn stake(&self, authority_index: AuthorityIndex) -> Stake { - self.authorities[authority_index.value()].stake + self.authorities[authority_index].stake } pub fn authority(&self, authority_index: AuthorityIndex) -> &Authority { - &self.authorities[authority_index.value()] + &self.authorities[authority_index] } pub fn authorities(&self) -> impl Iterator { @@ -173,6 +176,34 @@ impl Display for AuthorityIndex { } } +impl Index for [T; N] { + type Output = T; + + fn index(&self, index: AuthorityIndex) -> &Self::Output { + self.get(index.value()).unwrap() + } +} + +impl Index for Vec { + type Output = T; + + fn index(&self, index: AuthorityIndex) -> &Self::Output { + self.get(index.value()).unwrap() + } +} + +impl IndexMut for [T; N] { + fn index_mut(&mut self, index: AuthorityIndex) -> &mut Self::Output { + self.get_mut(index.value()).unwrap() + } +} + +impl IndexMut for Vec { + fn index_mut(&mut self, index: AuthorityIndex) -> &mut Self::Output { + self.get_mut(index.value()).unwrap() + } +} + #[cfg(test)] mod tests { use crate::{Committee, Stake}; diff --git a/consensus/config/src/crypto.rs b/consensus/config/src/crypto.rs index 3129addb2cdf2..bf65f94a5baf7 100644 --- a/consensus/config/src/crypto.rs +++ b/consensus/config/src/crypto.rs @@ -21,6 +21,8 @@ use shared_crypto::intent::INTENT_PREFIX_LENGTH; pub type NetworkPublicKey = ed25519::Ed25519PublicKey; pub type NetworkPrivateKey = ed25519::Ed25519PrivateKey; pub type NetworkKeyPair = ed25519::Ed25519KeyPair; +pub type NetworkKeySignature = ed25519::Ed25519Signature; +pub type NetworkKeySignatureAsBytes = ed25519::Ed25519SignatureAsBytes; /// Protocol key is used in random beacon. pub type ProtocolPublicKey = bls12381::min_sig::BLS12381PublicKey; diff --git a/consensus/core/Cargo.toml b/consensus/core/Cargo.toml index 20e8f5ac6a509..1f988edfa792e 100644 --- a/consensus/core/Cargo.toml +++ b/consensus/core/Cargo.toml @@ -9,9 +9,11 @@ publish = false [dependencies] base64.workspace = true bcs.workspace = true +bytes.workspace = true fastcrypto.workspace = true serde.workspace = true prometheus.workspace = true +thiserror.workspace = true tokio.workspace = true tracing.workspace = true async-trait.workspace = true diff --git a/consensus/core/src/block.rs b/consensus/core/src/block.rs index 96779700f8c04..3bcf83c640e83 100644 --- a/consensus/core/src/block.rs +++ b/consensus/core/src/block.rs @@ -11,7 +11,7 @@ use std::{ use fastcrypto::hash::{Digest, HashFunction}; use serde::{Deserialize, Serialize}; -use consensus_config::{AuthorityIndex, DefaultHashFunction, DIGEST_LENGTH}; +use consensus_config::{AuthorityIndex, DefaultHashFunction, NetworkKeySignature, DIGEST_LENGTH}; /// Round number of a block. pub type Round = u32; @@ -45,7 +45,7 @@ pub trait BlockAPI { fn digest(&self) -> BlockDigest; fn round(&self) -> Round; fn author(&self) -> AuthorityIndex; - fn timestamp(&self) -> BlockTimestampMs; + fn timestamp_ms(&self) -> BlockTimestampMs; fn ancestors(&self) -> &[BlockRef]; // TODO: add accessor for transactions. } @@ -54,7 +54,7 @@ pub trait BlockAPI { pub struct BlockV1 { round: Round, author: AuthorityIndex, - timestamp: BlockTimestampMs, + timestamp_ms: BlockTimestampMs, ancestors: Vec, #[serde(skip)] @@ -86,8 +86,8 @@ impl BlockAPI for BlockV1 { self.author } - fn timestamp(&self) -> BlockTimestampMs { - self.timestamp + fn timestamp_ms(&self) -> BlockTimestampMs { + self.timestamp_ms } fn ancestors(&self) -> &[BlockRef] { @@ -120,14 +120,6 @@ impl Hash for BlockRef { } } -impl fastcrypto::hash::Hash<{ DIGEST_LENGTH }> for BlockRef { - type TypedDigest = BlockDigest; - - fn digest(&self) -> BlockDigest { - self.digest - } -} - /// Hash of a block, covers all fields except signature. #[derive(Clone, Copy, Serialize, Deserialize, Default, PartialEq, Eq, PartialOrd, Ord)] pub struct BlockDigest([u8; consensus_config::DIGEST_LENGTH]); @@ -154,4 +146,34 @@ impl fmt::Debug for BlockDigest { } } +/// Signature of block digest by its author. +#[allow(unused)] +pub(crate) type BlockSignature = NetworkKeySignature; + +/// Unverified block only allows limited access to its content. +#[allow(unused)] +#[derive(Deserialize)] +pub(crate) struct SignedBlock { + block: Block, + signature: bytes::Bytes, + + #[serde(skip)] + serialized: bytes::Bytes, +} + +impl SignedBlock { + // TODO: add deserialization and verification. +} + +/// Verifiied block allows access to its content. +#[allow(unused)] +#[derive(Deserialize, Serialize)] +pub(crate) struct VerifiedBlock { + pub block: Block, + pub signature: bytes::Bytes, + + #[serde(skip)] + serialized: bytes::Bytes, +} + // TODO: add basic verification for BlockRef and BlockDigest computations. diff --git a/consensus/core/src/commit.rs b/consensus/core/src/commit.rs new file mode 100644 index 0000000000000..0487851e03511 --- /dev/null +++ b/consensus/core/src/commit.rs @@ -0,0 +1,20 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +use serde::{Deserialize, Serialize}; + +use crate::block::BlockRef; + +/// Specifies one consensus commit. +/// It is stored on disk, so it does not contain blocks which are stored individually. +#[allow(unused)] +#[derive(Deserialize, Serialize)] +pub(crate) struct Commit { + /// Index of the commit. + /// First commit after genesis has an index of 1, then every next commit has an index incremented by 1. + pub index: u64, + /// A reference to the the commit leader. + pub leader: BlockRef, + /// Refs to committed blocks, in the commit order. + pub blocks: Vec, +} diff --git a/consensus/core/src/dag_state.rs b/consensus/core/src/dag_state.rs new file mode 100644 index 0000000000000..480b21a13bdf5 --- /dev/null +++ b/consensus/core/src/dag_state.rs @@ -0,0 +1,69 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +use std::{ + collections::{BTreeMap, BTreeSet}, + sync::Arc, +}; + +use crate::{ + block::{BlockAPI as _, BlockRef, Round, VerifiedBlock}, + context::Context, + storage::Store, +}; + +/// Recent rounds of blocks to cached in memory, counted from the last committed leader round. +#[allow(unused)] +const BLOCK_CACHED_ROUNDS: Round = 100; + +/// DagState provides the API to write and read accepted blocks from the DAG. +/// Only uncommited and last committed blocks are cached in memory. +/// The rest of blocks are stored on disk. +/// Refs to cached blocks and additional refs are cached as well, to speed up existence checks. +/// +/// Note: DagState should be wrapped with Arc>, to allow concurrent access from +/// multiple components. +#[allow(unused)] +pub(crate) struct DagState { + context: Arc, + + // Caches uncommitted blocks, and recent blocks within BLOCK_CACHED_ROUNDS from the + // last committed leader round. + recent_blocks: BTreeMap, + + // All accepted blocks have their refs cached. Cached refs are never removed for now. + // Each element in the vector contains refs for the authority corresponding to its index. + cached_refs: Vec>, + + // Persistent storage for blocks, commits and other consensus data. + store: Arc, +} + +#[allow(unused)] +impl DagState { + pub(crate) fn new( + context: Arc, + blocks: Vec, + store: Arc, + ) -> Self { + let num_authorities = context.committee.size(); + let mut state = Self { + context, + recent_blocks: BTreeMap::new(), + cached_refs: vec![BTreeSet::new(); num_authorities], + store, + }; + + for block in blocks { + state.add_block(block); + } + + state + } + + pub(crate) fn add_block(&mut self, block: VerifiedBlock) { + let block_ref = block.block.reference(); + self.recent_blocks.insert(block_ref, block); + self.cached_refs[block_ref.author].insert(block_ref); + } +} diff --git a/consensus/core/src/error.rs b/consensus/core/src/error.rs new file mode 100644 index 0000000000000..1460dc3f4ccbf --- /dev/null +++ b/consensus/core/src/error.rs @@ -0,0 +1,15 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +use thiserror::Error; + +/// Errors that can occur when processing blocks, reading from storage, or encountering shutdown. +#[allow(unused)] +#[derive(Clone, Debug, Error)] +pub enum ConsensusError { + #[error("Error deserializing block")] + MalformattedBlock, +} + +#[allow(unused)] +pub type ConsensusResult = Result; diff --git a/consensus/core/src/lib.rs b/consensus/core/src/lib.rs index e3e59244580c7..06fecc3d77d4a 100644 --- a/consensus/core/src/lib.rs +++ b/consensus/core/src/lib.rs @@ -3,9 +3,13 @@ mod block; mod block_verifier; +mod commit; mod context; mod core; +mod dag_state; +mod error; mod metrics; mod stake_aggregator; +mod storage; mod threshold_clock; mod validator; diff --git a/consensus/core/src/storage/mod.rs b/consensus/core/src/storage/mod.rs new file mode 100644 index 0000000000000..99a6e97028c96 --- /dev/null +++ b/consensus/core/src/storage/mod.rs @@ -0,0 +1,17 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +mod rocksdb; + +use crate::{block::VerifiedBlock, commit::Commit, error::ConsensusResult}; + +/// A common interface for consensus storage. +pub(crate) trait Store { + /// Loads last committed blocks, all uncommitted blocks and last commit from store. + fn recover(&self) -> ConsensusResult<(Vec, Commit)>; + + /// Writes blocks and consensus commits to store. + fn write(&self, blocks: Vec, commits: Vec) -> ConsensusResult<()>; + + // TODO: add methods to read and scan blocks. +} diff --git a/consensus/core/src/storage/rocksdb.rs b/consensus/core/src/storage/rocksdb.rs new file mode 100644 index 0000000000000..cd8abcfa003e2 --- /dev/null +++ b/consensus/core/src/storage/rocksdb.rs @@ -0,0 +1,20 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +use crate::{block::VerifiedBlock, commit::Commit, error::ConsensusResult}; + +use super::Store; + +/// Storage implementation using RocksDB. +pub(crate) struct RocksDB {} + +#[allow(unused)] +impl Store for RocksDB { + fn recover(&self) -> ConsensusResult<(Vec, Commit)> { + unimplemented!() + } + + fn write(&self, blocks: Vec, commits: Vec) -> ConsensusResult<()> { + unimplemented!() + } +}