diff --git a/Cargo.lock b/Cargo.lock index 893297b114e42..66b6c69343092 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12060,6 +12060,7 @@ dependencies = [ "bcs", "bimap", "clap", + "eyre", "fastcrypto", "move-binary-format", "move-bytecode-utils", @@ -12073,7 +12074,9 @@ dependencies = [ "msim", "once_cell", "rand 0.8.5", + "rocksdb", "simulacrum", + "sui-config", "sui-core", "sui-framework", "sui-framework-snapshot", @@ -12086,6 +12089,8 @@ dependencies = [ "sui-types", "tempfile", "tokio", + "typed-store", + "typed-store-derive", "workspace-hack", ] diff --git a/crates/simulacrum/src/epoch_state.rs b/crates/simulacrum/src/epoch_state.rs index 112cab6713b26..2c4399ee6ebd7 100644 --- a/crates/simulacrum/src/epoch_state.rs +++ b/crates/simulacrum/src/epoch_state.rs @@ -20,7 +20,7 @@ use sui_types::{ transaction::{TransactionDataAPI, VerifiedTransaction}, }; -use crate::store::InMemoryStore; +use crate::SimulatorStore; pub struct EpochState { epoch_start_state: EpochStartSystemState, @@ -84,7 +84,7 @@ impl EpochState { pub fn execute_transaction( &self, - store: &InMemoryStore, + store: &dyn SimulatorStore, deny_config: &TransactionDenyConfig, transaction: &VerifiedTransaction, ) -> Result<( @@ -103,7 +103,7 @@ impl EpochState { &input_object_kinds, &receiving_object_refs, deny_config, - store, + &store, )?; let (input_objects, receiving_objects) = store.read_objects_for_synchronous_execution( @@ -126,7 +126,7 @@ impl EpochState { let transaction_data = transaction.data().transaction_data(); let (kind, signer, gas) = transaction_data.execution_parts(); Ok(self.executor.execute_transaction_to_effects( - store, + store.backing_store(), &self.protocol_config, self.limits_metrics.clone(), false, // enable_expensive_checks diff --git a/crates/simulacrum/src/lib.rs b/crates/simulacrum/src/lib.rs index cdca5b2dff332..9b0cb5533aee4 100644 --- a/crates/simulacrum/src/lib.rs +++ b/crates/simulacrum/src/lib.rs @@ -38,12 +38,13 @@ use sui_types::{ }; use self::epoch_state::EpochState; -use self::store::KeyStore; -pub use self::store::{InMemoryStore, SimulatorStore}; +pub use self::store::in_mem_store::InMemoryStore; +use self::store::in_mem_store::KeyStore; +pub use self::store::SimulatorStore; use sui_types::mock_checkpoint_builder::{MockCheckpointBuilder, ValidatorKeypairProvider}; mod epoch_state; -mod store; +pub mod store; /// A `Simulacrum` of Sui. /// @@ -101,7 +102,7 @@ where .with_chain_start_timestamp_ms(1) .deterministic_committee_size(NonZeroUsize::new(1).unwrap()) .build(); - Self::new_with_network_config(&config, rng) + Self::new_with_network_config_in_mem(&config, rng) } pub fn new_with_protocol_version_and_accounts( @@ -117,12 +118,18 @@ where .with_protocol_version(protocol_version) .with_accounts(account_configs) .build(); - Self::new_with_network_config(&config, rng) + Self::new_with_network_config_in_mem(&config, rng) } - fn new_with_network_config(config: &NetworkConfig, rng: R) -> Self { - let keystore = KeyStore::from_network_config(config); + fn new_with_network_config_in_mem(config: &NetworkConfig, rng: R) -> Self { let store = InMemoryStore::new(&config.genesis); + Self::new_with_network_config_store(config, rng, store) + } +} + +impl Simulacrum { + pub fn new_with_network_config_store(config: &NetworkConfig, rng: R, store: S) -> Self { + let keystore = KeyStore::from_network_config(config); let checkpoint_builder = MockCheckpointBuilder::new(config.genesis.checkpoint()); let genesis = &config.genesis; @@ -138,9 +145,7 @@ where deny_config: TransactionDenyConfig::default(), } } -} -impl Simulacrum { /// Attempts to execute the provided Transaction. /// /// The provided Transaction undergoes the same types of checks that a Validator does prior to @@ -254,7 +259,7 @@ impl Simulacrum { self.epoch_state = new_epoch_state; } - pub fn store(&self) -> &InMemoryStore { + pub fn store(&self) -> &dyn SimulatorStore { &self.store } @@ -395,7 +400,7 @@ mod tests { fn deterministic_genesis() { let rng = StdRng::from_seed([9; 32]); let chain1 = Simulacrum::new_with_rng(rng); - let genesis_checkpoint_digest1 = chain1 + let genesis_checkpoint_digest1 = *chain1 .store() .get_checkpoint_by_sequence_number(0) .unwrap() @@ -403,7 +408,7 @@ mod tests { let rng = StdRng::from_seed([9; 32]); let chain2 = Simulacrum::new_with_rng(rng); - let genesis_checkpoint_digest2 = chain2 + let genesis_checkpoint_digest2 = *chain2 .store() .get_checkpoint_by_sequence_number(0) .unwrap() @@ -469,7 +474,7 @@ mod tests { .owned_objects(sender) .find(|object| object.is_gas_coin()) .unwrap(); - let gas_coin = GasCoin::try_from(object).unwrap(); + let gas_coin = GasCoin::try_from(&object).unwrap(); let gas_id = object.id(); let transfer_amount = gas_coin.value() / 2; @@ -496,9 +501,8 @@ mod tests { assert_eq!( (transfer_amount as i64 - gas_paid) as u64, - sim.store() - .get_object(&gas_id) - .and_then(|object| GasCoin::try_from(object).ok()) + store::SimulatorStore::get_object(sim.store(), &gas_id) + .and_then(|object| GasCoin::try_from(&object).ok()) .unwrap() .value() ); @@ -508,7 +512,7 @@ mod tests { sim.store() .owned_objects(recipient) .next() - .and_then(|object| GasCoin::try_from(object).ok()) + .and_then(|object| GasCoin::try_from(&object).ok()) .unwrap() .value() ); diff --git a/crates/simulacrum/src/store.rs b/crates/simulacrum/src/store/in_mem_store.rs similarity index 70% rename from crates/simulacrum/src/store.rs rename to crates/simulacrum/src/store/in_mem_store.rs index 2c1acb8f02bfc..33cf81b42cedb 100644 --- a/crates/simulacrum/src/store.rs +++ b/crates/simulacrum/src/store/in_mem_store.rs @@ -8,24 +8,23 @@ use std::collections::{BTreeMap, HashMap}; use sui_config::genesis; use sui_types::storage::{get_module, load_package_object_from_object_store, PackageObjectArc}; use sui_types::{ - base_types::{AuthorityName, ObjectID, ObjectRef, SequenceNumber, SuiAddress}, + base_types::{AuthorityName, ObjectID, SequenceNumber, SuiAddress}, committee::{Committee, EpochId}, crypto::{AccountKeyPair, AuthorityKeyPair}, digests::{ObjectDigest, TransactionDigest, TransactionEventsDigest}, effects::{TransactionEffects, TransactionEffectsAPI, TransactionEvents}, - error::{SuiError, SuiResult, UserInputError}, + error::SuiError, messages_checkpoint::{ CheckpointContents, CheckpointContentsDigest, CheckpointDigest, CheckpointSequenceNumber, VerifiedCheckpoint, }, object::{Object, Owner}, storage::{BackingPackageStore, ChildObjectResolver, ObjectStore, ParentSync}, - transaction::{ - InputObjectKind, InputObjects, ObjectReadResult, ReceivingObjectReadResult, - ReceivingObjects, VerifiedTransaction, - }, + transaction::VerifiedTransaction, }; +use super::SimulatorStore; + #[derive(Debug, Default)] pub struct InMemoryStore { // Checkpoint data @@ -51,30 +50,7 @@ pub struct InMemoryStore { impl InMemoryStore { pub fn new(genesis: &genesis::Genesis) -> Self { let mut store = Self::default(); - - store.insert_checkpoint(genesis.checkpoint()); - store.insert_checkpoint_contents(genesis.checkpoint_contents().clone()); - store.insert_committee(genesis.committee().unwrap()); - store.insert_transaction(VerifiedTransaction::new_unchecked( - genesis.transaction().clone(), - )); - store.insert_transaction_effects(genesis.effects().clone()); - store.insert_events( - genesis.effects().transaction_digest(), - genesis.events().clone(), - ); - - for object in genesis.objects() { - let object_id = object.id(); - let version = object.version(); - store.live_objects.insert(object_id, version); - store - .objects - .entry(object_id) - .or_default() - .insert(version, object.clone()); - } - + store.init_with_genesis(genesis); store } @@ -110,7 +86,6 @@ impl InMemoryStore { pub fn get_committee_by_epoch(&self, epoch: EpochId) -> Option<&Committee> { self.epoch_to_committee.get(epoch as usize) } - pub fn get_transaction(&self, digest: &TransactionDigest) -> Option<&VerifiedTransaction> { self.transactions.get(digest) } @@ -129,15 +104,6 @@ impl InMemoryStore { self.events.get(digest) } - pub fn get_transaction_events_by_tx_digest( - &self, - tx_digest: &TransactionDigest, - ) -> Option<&TransactionEvents> { - self.events_tx_digest_index - .get(tx_digest) - .and_then(|x| self.events.get(x)) - } - pub fn get_object(&self, id: &ObjectID) -> Option<&Object> { let version = self.live_objects.get(id)?; self.get_object_at_version(id, *version) @@ -272,7 +238,7 @@ impl ChildObjectResolver for InMemoryStore { child: &ObjectID, child_version_upper_bound: SequenceNumber, ) -> sui_types::error::SuiResult> { - let child_object = match self.get_object(child).cloned() { + let child_object = match crate::store::SimulatorStore::get_object(self, child) { None => return Ok(None), Some(obj) => obj, }; @@ -303,7 +269,8 @@ impl ChildObjectResolver for InMemoryStore { receive_object_at_version: SequenceNumber, _epoch_id: EpochId, ) -> sui_types::error::SuiResult> { - let recv_object = match self.get_object(receiving_object_id).cloned() { + let recv_object = match crate::store::SimulatorStore::get_object(self, receiving_object_id) + { None => return Ok(None), Some(obj) => obj, }; @@ -407,159 +374,65 @@ impl KeyStore { } } -// TODO: After we abstract object storage into the ExecutionCache trait, we can replace this with -// sui_core::TransactionInputLoad using an appropriate cache implementation. -impl InMemoryStore { - pub fn read_objects_for_synchronous_execution( - &self, - _tx_digest: &TransactionDigest, - input_object_kinds: &[InputObjectKind], - receiving_object_refs: &[ObjectRef], - ) -> SuiResult<(InputObjects, ReceivingObjects)> { - let mut input_objects = Vec::new(); - for kind in input_object_kinds { - let obj = match kind { - InputObjectKind::MovePackage(id) => self.get_object(id).cloned(), - InputObjectKind::ImmOrOwnedMoveObject(objref) => { - self.get_object_by_key(&objref.0, objref.1)? - } - - InputObjectKind::SharedMoveObject { id, .. } => self.get_object(id).cloned(), - }; - - input_objects.push(ObjectReadResult::new( - *kind, - obj.ok_or_else(|| kind.object_not_found_error())?.into(), - )); - } - - let mut receiving_objects = Vec::new(); - for objref in receiving_object_refs { - // no need for marker table check in simulacrum - let Some(obj) = self.get_object(&objref.0).cloned() else { - return Err(UserInputError::ObjectNotFound { - object_id: objref.0, - version: Some(objref.1), - } - .into()); - }; - receiving_objects.push(ReceivingObjectReadResult::new(*objref, obj.into())); - } - - Ok((input_objects.into(), receiving_objects.into())) - } -} - -pub trait SimulatorStore: - sui_types::storage::BackingPackageStore + sui_types::storage::ObjectStore -{ - fn get_checkpoint_by_sequence_number( - &self, - sequence_number: CheckpointSequenceNumber, - ) -> Option<&VerifiedCheckpoint>; - - fn get_checkpoint_by_digest(&self, digest: &CheckpointDigest) -> Option<&VerifiedCheckpoint>; - - fn get_highest_checkpint(&self) -> Option<&VerifiedCheckpoint>; - fn get_checkpoint_contents( - &self, - digest: &CheckpointContentsDigest, - ) -> Option<&CheckpointContents>; - - fn get_committee_by_epoch(&self, epoch: EpochId) -> Option<&Committee>; - - fn get_transaction(&self, digest: &TransactionDigest) -> Option<&VerifiedTransaction>; - - fn get_transaction_effects(&self, digest: &TransactionDigest) -> Option<&TransactionEffects>; - fn get_transaction_events( - &self, - digest: &TransactionEventsDigest, - ) -> Option<&TransactionEvents>; - - fn get_object(&self, id: &ObjectID) -> Option<&Object>; - fn get_object_at_version(&self, id: &ObjectID, version: SequenceNumber) -> Option<&Object>; - - fn get_system_state(&self) -> sui_types::sui_system_state::SuiSystemState; - - fn get_clock(&self) -> sui_types::clock::Clock; - - fn owned_objects(&self, owner: SuiAddress) -> Box + '_>; - - fn insert_checkpoint(&mut self, checkpoint: VerifiedCheckpoint); - - fn insert_checkpoint_contents(&mut self, contents: CheckpointContents); - - fn insert_committee(&mut self, committee: Committee); - - fn insert_executed_transaction( - &mut self, - transaction: VerifiedTransaction, - effects: TransactionEffects, - events: TransactionEvents, - written_objects: BTreeMap, - ); - - fn insert_transaction(&mut self, transaction: VerifiedTransaction); - - fn insert_transaction_effects(&mut self, effects: TransactionEffects); - - fn insert_events(&mut self, tx_digest: &TransactionDigest, events: TransactionEvents); - - fn update_objects( - &mut self, - written_objects: BTreeMap, - deleted_objects: Vec<(ObjectID, SequenceNumber, ObjectDigest)>, - ); -} - impl SimulatorStore for InMemoryStore { fn get_checkpoint_by_sequence_number( &self, sequence_number: CheckpointSequenceNumber, - ) -> Option<&VerifiedCheckpoint> { + ) -> Option { self.get_checkpoint_by_sequence_number(sequence_number) + .cloned() } - fn get_checkpoint_by_digest(&self, digest: &CheckpointDigest) -> Option<&VerifiedCheckpoint> { - self.get_checkpoint_by_digest(digest) + fn get_checkpoint_by_digest(&self, digest: &CheckpointDigest) -> Option { + self.get_checkpoint_by_digest(digest).cloned() } - fn get_highest_checkpint(&self) -> Option<&VerifiedCheckpoint> { - self.get_highest_checkpint() + fn get_highest_checkpint(&self) -> Option { + self.get_highest_checkpint().cloned() } fn get_checkpoint_contents( &self, digest: &CheckpointContentsDigest, - ) -> Option<&CheckpointContents> { - self.get_checkpoint_contents(digest) + ) -> Option { + self.get_checkpoint_contents(digest).cloned() } - fn get_committee_by_epoch(&self, epoch: EpochId) -> Option<&Committee> { - self.get_committee_by_epoch(epoch) + fn get_committee_by_epoch(&self, epoch: EpochId) -> Option { + self.get_committee_by_epoch(epoch).cloned() } - fn get_transaction(&self, digest: &TransactionDigest) -> Option<&VerifiedTransaction> { - self.get_transaction(digest) + fn get_transaction(&self, digest: &TransactionDigest) -> Option { + self.get_transaction(digest).cloned() } - fn get_transaction_effects(&self, digest: &TransactionDigest) -> Option<&TransactionEffects> { - self.get_transaction_effects(digest) + fn get_transaction_effects(&self, digest: &TransactionDigest) -> Option { + self.get_transaction_effects(digest).cloned() } fn get_transaction_events( &self, digest: &TransactionEventsDigest, - ) -> Option<&TransactionEvents> { - self.get_transaction_events(digest) + ) -> Option { + self.get_transaction_events(digest).cloned() } - fn get_object(&self, id: &ObjectID) -> Option<&Object> { - self.get_object(id) + fn get_transaction_events_by_tx_digest( + &self, + tx_digest: &TransactionDigest, + ) -> Option { + self.events_tx_digest_index + .get(tx_digest) + .and_then(|x| self.events.get(x)) + .cloned() } - fn get_object_at_version(&self, id: &ObjectID, version: SequenceNumber) -> Option<&Object> { - self.get_object_at_version(id, version) + fn get_object(&self, id: &ObjectID) -> Option { + self.get_object(id).cloned() + } + + fn get_object_at_version(&self, id: &ObjectID, version: SequenceNumber) -> Option { + self.get_object_at_version(id, version).cloned() } fn get_system_state(&self) -> sui_types::sui_system_state::SuiSystemState { @@ -570,8 +443,8 @@ impl SimulatorStore for InMemoryStore { self.get_clock() } - fn owned_objects(&self, owner: SuiAddress) -> Box + '_> { - Box::new(self.owned_objects(owner)) + fn owned_objects(&self, owner: SuiAddress) -> Box + '_> { + Box::new(self.owned_objects(owner).cloned()) } fn insert_checkpoint(&mut self, checkpoint: VerifiedCheckpoint) { @@ -615,4 +488,8 @@ impl SimulatorStore for InMemoryStore { ) { self.update_objects(written_objects, deleted_objects) } + + fn backing_store(&self) -> &dyn sui_types::storage::BackingStore { + self + } } diff --git a/crates/simulacrum/src/store/mod.rs b/crates/simulacrum/src/store/mod.rs new file mode 100644 index 0000000000000..e91ad4aaec0e9 --- /dev/null +++ b/crates/simulacrum/src/store/mod.rs @@ -0,0 +1,167 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::BTreeMap; +use sui_config::genesis; +use sui_types::base_types::ObjectRef; +use sui_types::error::UserInputError; +use sui_types::transaction::InputObjects; +use sui_types::transaction::ObjectReadResult; +use sui_types::transaction::ReceivingObjectReadResult; +use sui_types::transaction::ReceivingObjects; +use sui_types::{ + base_types::{ObjectID, SequenceNumber, SuiAddress}, + committee::{Committee, EpochId}, + digests::{ObjectDigest, TransactionDigest, TransactionEventsDigest}, + effects::{TransactionEffects, TransactionEffectsAPI, TransactionEvents}, + error::SuiResult, + messages_checkpoint::{ + CheckpointContents, CheckpointContentsDigest, CheckpointDigest, CheckpointSequenceNumber, + VerifiedCheckpoint, + }, + object::Object, + storage::{BackingStore, ChildObjectResolver, ParentSync}, + transaction::{InputObjectKind, VerifiedTransaction}, +}; +pub mod in_mem_store; + +pub trait SimulatorStore: + sui_types::storage::BackingPackageStore + + sui_types::storage::ObjectStore + + ParentSync + + ChildObjectResolver +{ + fn init_with_genesis(&mut self, genesis: &genesis::Genesis) { + self.insert_checkpoint(genesis.checkpoint()); + self.insert_checkpoint_contents(genesis.checkpoint_contents().clone()); + self.insert_committee(genesis.committee().unwrap()); + self.insert_transaction(VerifiedTransaction::new_unchecked( + genesis.transaction().clone(), + )); + self.insert_transaction_effects(genesis.effects().clone()); + self.insert_events( + genesis.effects().transaction_digest(), + genesis.events().clone(), + ); + + self.update_objects( + genesis + .objects() + .iter() + .map(|o| (o.id(), o.clone())) + .collect(), + vec![], + ); + } + + fn get_checkpoint_by_sequence_number( + &self, + sequence_number: CheckpointSequenceNumber, + ) -> Option; + + fn get_checkpoint_by_digest(&self, digest: &CheckpointDigest) -> Option; + + fn get_highest_checkpint(&self) -> Option; + + fn get_checkpoint_contents( + &self, + digest: &CheckpointContentsDigest, + ) -> Option; + + fn get_committee_by_epoch(&self, epoch: EpochId) -> Option; + + fn get_transaction(&self, digest: &TransactionDigest) -> Option; + + fn get_transaction_effects(&self, digest: &TransactionDigest) -> Option; + + fn get_transaction_events(&self, digest: &TransactionEventsDigest) + -> Option; + + fn get_transaction_events_by_tx_digest( + &self, + tx_digest: &TransactionDigest, + ) -> Option; + + fn get_object(&self, id: &ObjectID) -> Option; + + fn get_object_at_version(&self, id: &ObjectID, version: SequenceNumber) -> Option; + + fn get_system_state(&self) -> sui_types::sui_system_state::SuiSystemState; + + fn get_clock(&self) -> sui_types::clock::Clock; + + fn owned_objects(&self, owner: SuiAddress) -> Box + '_>; + + fn insert_checkpoint(&mut self, checkpoint: VerifiedCheckpoint); + + fn insert_checkpoint_contents(&mut self, contents: CheckpointContents); + + fn insert_committee(&mut self, committee: Committee); + + fn insert_executed_transaction( + &mut self, + transaction: VerifiedTransaction, + effects: TransactionEffects, + events: TransactionEvents, + written_objects: BTreeMap, + ); + + fn insert_transaction(&mut self, transaction: VerifiedTransaction); + + fn insert_transaction_effects(&mut self, effects: TransactionEffects); + + fn insert_events(&mut self, tx_digest: &TransactionDigest, events: TransactionEvents); + + fn update_objects( + &mut self, + written_objects: BTreeMap, + deleted_objects: Vec<(ObjectID, SequenceNumber, ObjectDigest)>, + ); + + fn backing_store(&self) -> &dyn BackingStore; + + // TODO: After we abstract object storage into the ExecutionCache trait, we can replace this with + // sui_core::TransactionInputLoad using an appropriate cache implementation. + fn read_objects_for_synchronous_execution( + &self, + _tx_digest: &TransactionDigest, + input_object_kinds: &[InputObjectKind], + receiving_object_refs: &[ObjectRef], + ) -> SuiResult<(InputObjects, ReceivingObjects)> { + let mut input_objects = Vec::new(); + for kind in input_object_kinds { + let obj = match kind { + InputObjectKind::MovePackage(id) => { + crate::store::SimulatorStore::get_object(self, id) + } + InputObjectKind::ImmOrOwnedMoveObject(objref) => { + self.get_object_by_key(&objref.0, objref.1)? + } + + InputObjectKind::SharedMoveObject { id, .. } => { + crate::store::SimulatorStore::get_object(self, id) + } + }; + + input_objects.push(ObjectReadResult::new( + *kind, + obj.ok_or_else(|| kind.object_not_found_error())?.into(), + )); + } + + let mut receiving_objects = Vec::new(); + for objref in receiving_object_refs { + // no need for marker table check in simulacrum + let Some(obj) = crate::store::SimulatorStore::get_object(self, &objref.0) else { + return Err(UserInputError::ObjectNotFound { + object_id: objref.0, + version: Some(objref.1), + } + .into()); + }; + receiving_objects.push(ReceivingObjectReadResult::new(*objref, obj.into())); + } + + Ok((input_objects.into(), receiving_objects.into())) + } +} diff --git a/crates/sui-graphql-rpc/tests/e2e_tests.rs b/crates/sui-graphql-rpc/tests/e2e_tests.rs index 9acbf96e04fe8..f3d368b677dae 100644 --- a/crates/sui-graphql-rpc/tests/e2e_tests.rs +++ b/crates/sui-graphql-rpc/tests/e2e_tests.rs @@ -78,13 +78,13 @@ mod tests { sim.create_checkpoint(); sim.create_checkpoint(); - let genesis_checkpoint_digest1 = sim + let genesis_checkpoint_digest1 = *sim .store() .get_checkpoint_by_sequence_number(0) .unwrap() .digest(); - let chain_id_actual = format!("{}", ChainIdentifier::from(*genesis_checkpoint_digest1)); + let chain_id_actual = format!("{}", ChainIdentifier::from(genesis_checkpoint_digest1)); let exp = format!( "{{\"data\":{{\"chainIdentifier\":\"{}\"}}}}", chain_id_actual diff --git a/crates/sui-rest-api/src/node_state_getter.rs b/crates/sui-rest-api/src/node_state_getter.rs index 065c88dda0b2f..f78825d5a637a 100644 --- a/crates/sui-rest-api/src/node_state_getter.rs +++ b/crates/sui-rest-api/src/node_state_getter.rs @@ -127,7 +127,6 @@ impl NodeStateGetter for simulacrum::Simulacrum { ) -> SuiResult { self.store() .get_checkpoint_by_sequence_number(sequence_number) - .cloned() .ok_or(SuiError::UserInputError { error: UserInputError::VerifiedCheckpointNotFound(sequence_number), }) @@ -147,7 +146,6 @@ impl NodeStateGetter for simulacrum::Simulacrum { ) -> SuiResult { self.store() .get_checkpoint_contents(&content_digest) - .cloned() .ok_or(SuiError::UserInputError { error: UserInputError::CheckpointContentsNotFound(content_digest), }) @@ -159,7 +157,7 @@ impl NodeStateGetter for simulacrum::Simulacrum { ) -> SuiResult>> { Ok(tx_digests .iter() - .map(|digest| self.store().get_transaction(digest).cloned()) + .map(|digest| self.store().get_transaction(digest)) .collect()) } @@ -169,7 +167,7 @@ impl NodeStateGetter for simulacrum::Simulacrum { ) -> SuiResult>> { Ok(digests .iter() - .map(|digest| self.store().get_transaction_effects(digest).cloned()) + .map(|digest| self.store().get_transaction_effects(digest)) .collect()) } @@ -179,7 +177,7 @@ impl NodeStateGetter for simulacrum::Simulacrum { ) -> SuiResult>> { Ok(event_digests .iter() - .map(|digest| self.store().get_transaction_events(digest).cloned()) + .map(|digest| self.store().get_transaction_events(digest)) .collect()) } @@ -189,7 +187,7 @@ impl NodeStateGetter for simulacrum::Simulacrum { ) -> Result>, SuiError> { object_keys .iter() - .map(|key| ObjectStore::get_object_by_key(&self.store(), &key.0, key.1)) + .map(|key| self.store().get_object_by_key(&key.0, key.1)) .collect::, SuiError>>() } @@ -198,13 +196,13 @@ impl NodeStateGetter for simulacrum::Simulacrum { object_id: &ObjectID, version: VersionNumber, ) -> Result, SuiError> { - Ok(self - .store() - .get_object_at_version(object_id, version) - .cloned()) + Ok(self.store().get_object_at_version(object_id, version)) } fn get_object(&self, object_id: &ObjectID) -> Result, SuiError> { - ObjectStore::get_object(&self.store(), object_id) + Ok(simulacrum::SimulatorStore::get_object( + self.store(), + object_id, + )) } } diff --git a/crates/sui-transaction-checks/src/deny.rs b/crates/sui-transaction-checks/src/deny.rs index 659cbbb56d645..fd10c19489c07 100644 --- a/crates/sui-transaction-checks/src/deny.rs +++ b/crates/sui-transaction-checks/src/deny.rs @@ -30,7 +30,7 @@ pub fn check_transaction_for_signing( input_object_kinds: &[InputObjectKind], receiving_objects: &[ObjectRef], filter_config: &TransactionDenyConfig, - package_store: &impl BackingPackageStore, + package_store: &dyn BackingPackageStore, ) -> SuiResult { check_disabled_features(filter_config, tx_data, tx_signatures)?; @@ -151,7 +151,7 @@ fn check_input_objects( fn check_package_dependencies( filter_config: &TransactionDenyConfig, tx_data: &TransactionData, - package_store: &impl BackingPackageStore, + package_store: &dyn BackingPackageStore, ) -> SuiResult { let deny_map = filter_config.get_package_deny_set(); if deny_map.is_empty() { diff --git a/crates/sui-transactional-test-runner/Cargo.toml b/crates/sui-transactional-test-runner/Cargo.toml index 39d8461407629..11f5922b35621 100644 --- a/crates/sui-transactional-test-runner/Cargo.toml +++ b/crates/sui-transactional-test-runner/Cargo.toml @@ -12,6 +12,7 @@ anyhow.workspace = true bcs.workspace = true bimap.workspace = true clap.workspace = true +eyre.workspace = true once_cell.workspace = true rand.workspace = true tempfile.workspace = true @@ -30,9 +31,11 @@ move-transactional-test-runner.workspace = true move-stdlib = { path = "../../external-crates/move/crates/move-stdlib" } move-vm-runtime = { path = "../../external-crates/move/crates/move-vm-runtime" } +rocksdb.workspace = true simulacrum.workspace = true sui-rest-api.workspace = true sui-swarm-config.workspace = true +sui-config.workspace = true sui-core = { workspace = true, features = ["test-utils"] } sui-framework.workspace = true sui-protocol-config.workspace = true @@ -41,6 +44,8 @@ sui-json-rpc-types.workspace = true sui-json-rpc.workspace = true sui-framework-snapshot.workspace = true sui-storage.workspace = true +typed-store.workspace = true +typed-store-derive.workspace = true workspace-hack.workspace = true [target.'cfg(msim)'.dependencies] diff --git a/crates/sui-transactional-test-runner/src/lib.rs b/crates/sui-transactional-test-runner/src/lib.rs index 2e330b2cbab64..76b19fc261881 100644 --- a/crates/sui-transactional-test-runner/src/lib.rs +++ b/crates/sui-transactional-test-runner/src/lib.rs @@ -5,40 +5,41 @@ pub mod args; pub mod programmable_transaction_test_parser; +mod simulator_persisted_store; pub mod test_adapter; use move_transactional_test_runner::framework::run_test_impl; use rand::rngs::StdRng; use simulacrum::Simulacrum; +use simulacrum::SimulatorStore; use std::path::Path; -use sui_rest_api::node_state_getter::NodeStateGetter; -use sui_types::digests::TransactionDigest; -use sui_types::digests::TransactionEventsDigest; -use sui_types::effects::TransactionEvents; -use sui_types::event::Event; -use sui_types::messages_checkpoint::CheckpointContentsDigest; -use sui_types::storage::ObjectKey; -use sui_types::storage::ObjectStore; -use test_adapter::{SuiTestAdapter, PRE_COMPILED}; - use std::sync::Arc; use sui_core::authority::authority_test_utils::send_and_confirm_transaction_with_execution_error; use sui_core::authority::AuthorityState; use sui_json_rpc_types::DevInspectResults; use sui_json_rpc_types::EventFilter; +use sui_rest_api::node_state_getter::NodeStateGetter; use sui_storage::key_value_store::TransactionKeyValueStore; use sui_types::base_types::ObjectID; use sui_types::base_types::SuiAddress; use sui_types::base_types::VersionNumber; +use sui_types::digests::TransactionDigest; +use sui_types::digests::TransactionEventsDigest; use sui_types::effects::TransactionEffects; +use sui_types::effects::TransactionEvents; use sui_types::error::ExecutionError; use sui_types::error::SuiError; use sui_types::error::SuiResult; +use sui_types::event::Event; +use sui_types::messages_checkpoint::CheckpointContentsDigest; use sui_types::messages_checkpoint::VerifiedCheckpoint; use sui_types::object::Object; +use sui_types::storage::ObjectKey; +use sui_types::storage::ObjectStore; use sui_types::transaction::Transaction; use sui_types::transaction::TransactionDataAPI; use sui_types::transaction::TransactionKind; +use test_adapter::{SuiTestAdapter, PRE_COMPILED}; #[cfg_attr(not(msim), tokio::main)] #[cfg_attr(msim, msim::main)] @@ -272,7 +273,7 @@ impl TransactionalAdapter for Simulacrum { Ok(self .store() .get_transaction_events_by_tx_digest(tx_digest) - .map(|x| x.data.clone()) + .map(|x| x.data) .unwrap_or_default()) } diff --git a/crates/sui-transactional-test-runner/src/simulator_persisted_store.rs b/crates/sui-transactional-test-runner/src/simulator_persisted_store.rs new file mode 100644 index 0000000000000..0aa010062b966 --- /dev/null +++ b/crates/sui-transactional-test-runner/src/simulator_persisted_store.rs @@ -0,0 +1,485 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +use std::{collections::BTreeMap, path::PathBuf, time::Duration}; + +use move_binary_format::CompiledModule; +use move_bytecode_utils::module_cache::GetModule; +use move_core_types::{language_storage::ModuleId, resolver::ModuleResolver}; +use simulacrum::Simulacrum; +use std::num::NonZeroUsize; +use sui_config::genesis; +use sui_protocol_config::ProtocolVersion; +use sui_swarm_config::genesis_config::AccountConfig; +use sui_swarm_config::network_config_builder::ConfigBuilder; +use sui_types::{ + base_types::{ObjectID, SequenceNumber, SuiAddress}, + committee::{Committee, EpochId}, + digests::{ObjectDigest, TransactionDigest, TransactionEventsDigest}, + effects::{TransactionEffects, TransactionEffectsAPI, TransactionEvents}, + error::SuiError, + messages_checkpoint::{ + CheckpointContents, CheckpointContentsDigest, CheckpointDigest, CheckpointSequenceNumber, + VerifiedCheckpoint, + }, + object::{Object, Owner}, + storage::{ + load_package_object_from_object_store, BackingPackageStore, ChildObjectResolver, + ObjectStore, PackageObjectArc, ParentSync, + }, + transaction::VerifiedTransaction, +}; +use tempfile::tempdir; +use typed_store::traits::TableSummary; +use typed_store::traits::TypedStoreDebug; +use typed_store::Map; +use typed_store::{ + metrics::SamplingInterval, + rocks::{DBMap, MetricConf}, +}; +use typed_store_derive::DBMapUtils; + +use super::SimulatorStore; + +#[derive(Debug, DBMapUtils)] +pub struct PersistedStore { + // Checkpoint data + checkpoints: DBMap, + checkpoint_digest_to_sequence_number: DBMap, + checkpoint_contents: DBMap, + + // Transaction data + transactions: DBMap, + effects: DBMap, + events: DBMap, + events_tx_digest_index: DBMap, + + // Committee data + epoch_to_committee: DBMap<(), Vec>, + + // Object data + live_objects: DBMap, + objects: DBMap>, +} + +impl PersistedStore { + pub fn _new(genesis: &genesis::Genesis, path: Option) -> Self { + let path = path.unwrap_or(tempdir().unwrap().into_path()); + + let mut store = Self::open_tables_read_write( + path, + MetricConf::with_sampling(SamplingInterval::new(Duration::from_secs(60), 0)), + None, + None, + ); + + store.init_with_genesis(genesis); + store + } + + pub fn _new_sim_with_protocol_version_and_accounts( + mut rng: R, + chain_start_timestamp_ms: u64, + protocol_version: ProtocolVersion, + account_configs: Vec, + path: Option, + ) -> Simulacrum + where + R: rand::RngCore + rand::CryptoRng, + { + let config = ConfigBuilder::new_with_temp_dir() + .rng(&mut rng) + .with_chain_start_timestamp_ms(chain_start_timestamp_ms) + .deterministic_committee_size(NonZeroUsize::new(1).unwrap()) + .with_protocol_version(protocol_version) + .with_accounts(account_configs) + .build(); + let genesis = &config.genesis; + let store = PersistedStore::_new(genesis, path); + + Simulacrum::new_with_network_config_store(&config, rng, store) + } +} + +impl SimulatorStore for PersistedStore { + fn get_checkpoint_by_sequence_number( + &self, + sequence_number: CheckpointSequenceNumber, + ) -> Option { + self.checkpoints + .get(&sequence_number) + .expect("Fatal: DB read failed") + .map(|checkpoint| checkpoint.into()) + } + + fn get_checkpoint_by_digest(&self, digest: &CheckpointDigest) -> Option { + self.checkpoint_digest_to_sequence_number + .get(digest) + .expect("Fatal: DB read failed") + .and_then(|sequence_number| self.get_checkpoint_by_sequence_number(sequence_number)) + } + + fn get_highest_checkpint(&self) -> Option { + self.checkpoints + .unbounded_iter() + .skip_to_last() + .next() + .map(|(_, checkpoint)| checkpoint.into()) + } + + fn get_checkpoint_contents( + &self, + digest: &CheckpointContentsDigest, + ) -> Option { + self.checkpoint_contents + .get(digest) + .expect("Fatal: DB read failed") + } + + fn get_committee_by_epoch(&self, epoch: EpochId) -> Option { + self.epoch_to_committee + .get(&()) + .expect("Fatal: DB read failed") + .and_then(|committees| committees.get(epoch as usize).cloned()) + } + + fn get_transaction(&self, digest: &TransactionDigest) -> Option { + self.transactions + .get(digest) + .expect("Fatal: DB read failed") + .map(|transaction| transaction.into()) + } + + fn get_transaction_effects(&self, digest: &TransactionDigest) -> Option { + self.effects.get(digest).expect("Fatal: DB read failed") + } + + fn get_transaction_events( + &self, + digest: &TransactionEventsDigest, + ) -> Option { + self.events.get(digest).expect("Fatal: DB read failed") + } + + fn get_transaction_events_by_tx_digest( + &self, + tx_digest: &TransactionDigest, + ) -> Option { + self.events_tx_digest_index + .get(tx_digest) + .expect("Fatal: DB read failed") + .and_then(|x| self.events.get(&x).expect("Fatal: DB read failed")) + } + + fn get_object(&self, id: &ObjectID) -> Option { + let version = self.live_objects.get(id).expect("Fatal: DB read failed")?; + self.get_object_at_version(id, version) + } + + fn get_object_at_version(&self, id: &ObjectID, version: SequenceNumber) -> Option { + self.objects + .get(id) + .expect("Fatal: DB read failed") + .and_then(|versions| versions.get(&version).cloned()) + } + + fn get_system_state(&self) -> sui_types::sui_system_state::SuiSystemState { + sui_types::sui_system_state::get_sui_system_state(self).expect("system state must exist") + } + + fn get_clock(&self) -> sui_types::clock::Clock { + SimulatorStore::get_object(self, &sui_types::SUI_CLOCK_OBJECT_ID) + .expect("clock should exist") + .to_rust() + .expect("clock object should deserialize") + } + + fn owned_objects(&self, owner: SuiAddress) -> Box + '_> { + Box::new(self.live_objects + .unbounded_iter() + .flat_map(|(id, version)| self.get_object_at_version(&id, version)) + .filter( + move |object| matches!(object.owner, Owner::AddressOwner(addr) if addr == owner), + )) + } + + fn insert_checkpoint(&mut self, checkpoint: VerifiedCheckpoint) { + self.checkpoint_digest_to_sequence_number + .insert(checkpoint.digest(), checkpoint.sequence_number()) + .expect("Fatal: DB write failed"); + self.checkpoints + .insert(checkpoint.sequence_number(), checkpoint.serializable_ref()) + .expect("Fatal: DB write failed"); + } + + fn insert_checkpoint_contents(&mut self, contents: CheckpointContents) { + self.checkpoint_contents + .insert(contents.digest(), &contents) + .expect("Fatal: DB write failed"); + } + + fn insert_committee(&mut self, committee: Committee) { + let epoch = committee.epoch as usize; + + let mut committees = if let Some(c) = self + .epoch_to_committee + .get(&()) + .expect("Fatal: DB read failed") + { + c + } else { + vec![] + }; + + if committees.get(epoch).is_some() { + return; + } + + if committees.len() == epoch { + committees.push(committee); + } else { + panic!("committee was inserted into EpochCommitteeMap out of order"); + } + self.epoch_to_committee + .insert(&(), &committees) + .expect("Fatal: DB write failed"); + } + + fn insert_executed_transaction( + &mut self, + transaction: VerifiedTransaction, + effects: TransactionEffects, + events: TransactionEvents, + written_objects: BTreeMap, + ) { + let deleted_objects = effects.deleted(); + let tx_digest = *effects.transaction_digest(); + self.insert_transaction(transaction); + self.insert_transaction_effects(effects); + self.insert_events(&tx_digest, events); + self.update_objects(written_objects, deleted_objects); + } + + fn insert_transaction(&mut self, transaction: VerifiedTransaction) { + self.transactions + .insert(transaction.digest(), transaction.serializable_ref()) + .expect("Fatal: DB write failed"); + } + + fn insert_transaction_effects(&mut self, effects: TransactionEffects) { + self.effects + .insert(effects.transaction_digest(), &effects) + .expect("Fatal: DB write failed"); + } + + fn insert_events(&mut self, tx_digest: &TransactionDigest, events: TransactionEvents) { + self.events_tx_digest_index + .insert(tx_digest, &events.digest()) + .expect("Fatal: DB write failed"); + self.events + .insert(&events.digest(), &events) + .expect("Fatal: DB write failed"); + } + + fn update_objects( + &mut self, + written_objects: BTreeMap, + deleted_objects: Vec<(ObjectID, SequenceNumber, ObjectDigest)>, + ) { + for (object_id, _, _) in deleted_objects { + self.live_objects + .remove(&object_id) + .expect("Fatal: DB write failed"); + } + + for (object_id, object) in written_objects { + let version = object.version(); + self.live_objects + .insert(&object_id, &version) + .expect("Fatal: DB write failed"); + let mut q = + if let Some(x) = self.objects.get(&object_id).expect("Fatal: DB read failed") { + x + } else { + BTreeMap::new() + }; + q.insert(version, object); + self.objects + .insert(&object_id, &q) + .expect("Fatal: DB write failed"); + } + } + + fn backing_store(&self) -> &dyn sui_types::storage::BackingStore { + self + } +} + +impl BackingPackageStore for PersistedStore { + fn get_package_object( + &self, + package_id: &ObjectID, + ) -> sui_types::error::SuiResult> { + load_package_object_from_object_store(self, package_id) + } +} + +impl ChildObjectResolver for PersistedStore { + fn read_child_object( + &self, + parent: &ObjectID, + child: &ObjectID, + child_version_upper_bound: SequenceNumber, + ) -> sui_types::error::SuiResult> { + let child_object = match SimulatorStore::get_object(self, child) { + None => return Ok(None), + Some(obj) => obj, + }; + + let parent = *parent; + if child_object.owner != Owner::ObjectOwner(parent.into()) { + return Err(SuiError::InvalidChildObjectAccess { + object: *child, + given_parent: parent, + actual_owner: child_object.owner, + }); + } + + if child_object.version() > child_version_upper_bound { + return Err(SuiError::UnsupportedFeatureError { + error: "TODO InMemoryStorage::read_child_object does not yet support bounded reads" + .to_owned(), + }); + } + + Ok(Some(child_object)) + } + + fn get_object_received_at_version( + &self, + owner: &ObjectID, + receiving_object_id: &ObjectID, + receive_object_at_version: SequenceNumber, + _epoch_id: EpochId, + ) -> sui_types::error::SuiResult> { + let recv_object = match SimulatorStore::get_object(self, receiving_object_id) { + None => return Ok(None), + Some(obj) => obj, + }; + if recv_object.owner != Owner::AddressOwner((*owner).into()) { + return Ok(None); + } + + if recv_object.version() != receive_object_at_version { + return Ok(None); + } + Ok(Some(recv_object)) + } +} + +impl GetModule for PersistedStore { + type Error = SuiError; + type Item = CompiledModule; + + fn get_module_by_id(&self, id: &ModuleId) -> Result, Self::Error> { + Ok(self + .get_module(id)? + .map(|bytes| CompiledModule::deserialize_with_defaults(&bytes).unwrap())) + } +} + +impl ModuleResolver for PersistedStore { + type Error = SuiError; + + fn get_module(&self, module_id: &ModuleId) -> Result>, Self::Error> { + Ok(self + .get_package_object(&ObjectID::from(*module_id.address()))? + .and_then(|package| { + package + .move_package() + .serialized_module_map() + .get(module_id.name().as_str()) + .cloned() + })) + } +} + +impl ObjectStore for PersistedStore { + fn get_object( + &self, + object_id: &ObjectID, + ) -> Result, sui_types::error::SuiError> { + Ok(SimulatorStore::get_object(self, object_id)) + } + + fn get_object_by_key( + &self, + object_id: &ObjectID, + version: sui_types::base_types::VersionNumber, + ) -> Result, sui_types::error::SuiError> { + Ok(self.get_object_at_version(object_id, version)) + } +} + +impl ParentSync for PersistedStore { + fn get_latest_parent_entry_ref_deprecated( + &self, + _object_id: ObjectID, + ) -> sui_types::error::SuiResult> { + panic!("Never called in newer protocol versions") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rand::{rngs::StdRng, SeedableRng}; + + #[tokio::test] + async fn deterministic_genesis() { + let rng = StdRng::from_seed([9; 32]); + let chain1 = PersistedStore::_new_sim_with_protocol_version_and_accounts( + rng, + 0, + ProtocolVersion::MAX, + vec![], + None, + ); + let genesis_checkpoint_digest1 = *chain1 + .store() + .get_checkpoint_by_sequence_number(0) + .unwrap() + .digest(); + + let rng = StdRng::from_seed([9; 32]); + let chain2 = PersistedStore::_new_sim_with_protocol_version_and_accounts( + rng, + 0, + ProtocolVersion::MAX, + vec![], + None, + ); + let genesis_checkpoint_digest2 = *chain2 + .store() + .get_checkpoint_by_sequence_number(0) + .unwrap() + .digest(); + + assert_eq!(genesis_checkpoint_digest1, genesis_checkpoint_digest2); + + // Ensure the committees are different when using different seeds + let rng = StdRng::from_seed([0; 32]); + let chain3 = PersistedStore::_new_sim_with_protocol_version_and_accounts( + rng, + 0, + ProtocolVersion::MAX, + vec![], + None, + ); + + assert_ne!( + chain1.store().get_committee_by_epoch(0), + chain3.store().get_committee_by_epoch(0), + ); + } +}