Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[GraphQL] Add a new multiGetObjects query on Query. #20300

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

[GraphQL] Add a new multiGetObjects query on Query. #20300

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion crates/sui-graphql-rpc/schema.graphql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2899,7 +2899,9 @@ input ObjectFilter {
"""
objectIds: [SuiAddress!]
"""
Filter for live or potentially historical objects by their ID and version.
Filter for live objects by their ID and version. NOTE: this input filter has been
deprecated in favor of `multiGetObjects` query as it does not make sense to query for live
objects by their versions. This filter will be removed with v1.42.0 release.
"""
objectKeys: [ObjectKey!]
}
Expand Down Expand Up @@ -3325,6 +3327,10 @@ type Query {
"""
transactionBlock(digest: String!): TransactionBlock
"""
Fetch a list of objects by their IDs and versions.
"""
multiGetObjects(keys: [ObjectKey!]!): [Object!]!
"""
The coin objects that exist in the network.

The type field is a string of the inner type of the coin by which to filter (e.g.
Expand Down Expand Up @@ -3599,6 +3605,10 @@ type ServiceConfig {
"""
maxTransactionIds: Int!
"""
Maximum number of keys that can be passed to a `multiGetObjects` query.
"""
maxMultiGetObjectsKeys: Int!
"""
Maximum number of candidates to scan when gathering a page of results.
"""
maxScanLimit: Int!
Expand Down
14 changes: 13 additions & 1 deletion crates/sui-graphql-rpc/src/config.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,10 +118,12 @@ pub struct Limits {
pub max_type_argument_width: u32,
/// Maximum size of a fully qualified type.
pub max_type_nodes: u32,
/// Maximum deph of a move value.
/// Maximum depth of a move value.
pub max_move_value_depth: u32,
/// Maximum number of transaction ids that can be passed to a `TransactionBlockFilter`.
pub max_transaction_ids: u32,
stefan-mysten marked this conversation as resolved.
Show resolved Hide resolved
/// Maximum number of keys that can be passed to a `multiGetObjects` query.
pub max_multi_get_objects_keys: u32,
/// Maximum number of candidates to scan when gathering a page of results.
pub max_scan_limit: u32,
}
Expand Down Expand Up @@ -343,6 +345,11 @@ impl ServiceConfig {
self.limits.max_transaction_ids
}

/// Maximum number of keys that can be passed to a `multiGetObjects` query.
async fn max_multi_get_objects_keys(&self) -> u32 {
self.limits.max_multi_get_objects_keys
}

/// Maximum number of candidates to scan when gathering a page of results.
async fn max_scan_limit(&self) -> u32 {
self.limits.max_scan_limit
Expand Down Expand Up @@ -510,6 +517,7 @@ impl Default for Limits {
// Filter-specific limits, such as the number of transaction ids that can be specified
// for the `TransactionBlockFilter`.
max_transaction_ids: 1000,
max_multi_get_objects_keys: 500,
max_scan_limit: 100_000_000,
// This value is set to be the size of the max transaction bytes allowed + base64
// overhead (roughly 1/3 of the original string). This is rounded up.
Expand Down Expand Up @@ -594,6 +602,7 @@ mod tests {
max-type-nodes = 128
max-move-value-depth = 256
max-transaction-ids = 11
max-multi-get-objects-keys = 11
max-scan-limit = 50
"#,
)
Expand All @@ -616,6 +625,7 @@ mod tests {
max_type_nodes: 128,
max_move_value_depth: 256,
max_transaction_ids: 11,
max_multi_get_objects_keys: 11,
max_scan_limit: 50,
},
..Default::default()
Expand Down Expand Up @@ -682,6 +692,7 @@ mod tests {
max-type-nodes = 128
max-move-value-depth = 256
max-transaction-ids = 42
max-multi-get-objects-keys = 42
max-scan-limit = 420

[experiments]
Expand All @@ -707,6 +718,7 @@ mod tests {
max_type_nodes: 128,
max_move_value_depth: 256,
max_transaction_ids: 42,
max_multi_get_objects_keys: 42,
max_scan_limit: 420,
},
disabled_features: BTreeSet::from([FunctionalGroup::Analytics]),
Expand Down
47 changes: 42 additions & 5 deletions crates/sui-graphql-rpc/src/extensions/query_limits_checker.rs
View file
Open in desktop
stefan-mysten marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ use uuid::Uuid;
pub(crate) const CONNECTION_FIELDS: [&str; 2] = ["edges", "nodes"];
const DRY_RUN_TX_BLOCK: &str = "dryRunTransactionBlock";
const EXECUTE_TX_BLOCK: &str = "executeTransactionBlock";
const MULTI_GET_PREFIX: &str = "multiGet";
const MULTI_GET_OBJECT_KEYS: &str = "keys";
const VERIFY_ZKLOGIN: &str = "verifyZkloginSignature";

/// The size of the query payload in bytes, as it comes from the request header: `Content-Length`.
Expand Down Expand Up @@ -228,6 +230,7 @@ impl<'a> LimitsTraversal<'a> {
match &item.node {
Selection::Field(f) => {
let name = &f.node.name.node;

if name == DRY_RUN_TX_BLOCK || name == EXECUTE_TX_BLOCK {
for (_name, value) in &f.node.arguments {
self.check_tx_arg(value)?;
Expand Down Expand Up @@ -415,20 +418,22 @@ impl<'a> LimitsTraversal<'a> {
self.output_budget -= multiplicity;
}

// If the field being traversed is a connection field, increase multiplicity by a
// factor of page size. This operation can fail due to overflow, which will be
// treated as a limits check failure, even if the resulting value does not get used
// for anything.
let name = &f.node.name.node;

// Handle regular connection fields and multiGet queries
let multiplicity = 'm: {
// check if it is a multiGet query and return the number of keys
if let Some(page_size) = self.multi_get_page_size(f)? {
break 'm multiplicity * page_size;
}

if !CONNECTION_FIELDS.contains(&name.as_str()) {
break 'm multiplicity;
}

let Some(page_size) = page_size else {
break 'm multiplicity;
};

multiplicity
.checked_mul(page_size)
.ok_or_else(|| self.output_node_error())?
Expand Down Expand Up @@ -484,6 +489,23 @@ impl<'a> LimitsTraversal<'a> {
))
}

// If the field `f` is a multiGet query, extract the number of keys, otherwise return `None`.
// Returns an error if the number of keys cannot be represented as a `u32`.
fn multi_get_page_size(&mut self, f: &Positioned<Field>) -> ServerResult<Option<u32>> {
if !f.node.name.node.starts_with(MULTI_GET_PREFIX) {
return Ok(None);
}

let keys = f.node.get_argument(MULTI_GET_OBJECT_KEYS);
let Some(page_size) = self.resolve_list_size(keys) else {
return Ok(None);
};

Ok(Some(
page_size.try_into().map_err(|_| self.output_node_error())?,
))
}

/// Checks if the given field corresponds to a connection based on whether it contains a
/// selection for `edges` or `nodes`. That selection could be immediately in that field's
/// selection set, or nested within a fragment or inline fragment spread.
Expand Down Expand Up @@ -548,6 +570,21 @@ impl<'a> LimitsTraversal<'a> {
.as_u64()
}

/// Find the size of a list, resolving variables if necessary.
fn resolve_list_size(&self, value: Option<&Positioned<Value>>) -> Option<usize> {
match &value?.node {
Value::List(list) => Some(list.len()),
Value::Variable(var) => {
if let ConstValue::List(list) = self.variables.get(var)? {
Some(list.len())
} else {
None
}
}
_ => None,
}
}

/// Error returned if transaction payloads exceed limit. Also sets the transaction payload
/// budget to zero to indicate it has been spent (This is done to prevent future checks for
/// smaller arguments from succeeding even though a previous larger argument has already
Expand Down
60 changes: 60 additions & 0 deletions crates/sui-graphql-rpc/src/server/builder.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2067,4 +2067,64 @@ pub mod tests {
bytes or fewer."
);
}

#[tokio::test]
async fn test_multi_get_objects_query_limits() {
let cluster = prep_executor_cluster().await;
let db_url = cluster.graphql_connection_config.db_url.clone();
assert_eq!(
execute_for_error(
&db_url,
Limits {
max_output_nodes: 5,
..Default::default()
}, // the query will have 6 output nodes: 2 keys * 3 fields, thus exceeding the
// limit
r#"
query {
multiGetObjects(
keys: [
{objectId: "0x01dcb4674affb04e68d8088895e951f4ea335ef1695e9e50c166618f6789d808", version: 2},
{objectId: "0x23e340e97fb41249278c85b1f067dc88576f750670c6dc56572e90971f857c8c", version: 2},
]
) {
address
status
version
}
}
"#
.into(),
)
.await,
"Estimated output nodes exceeds 5"
);
assert_eq!(
execute_for_error(
&db_url,
Limits {
max_output_nodes: 4,
..Default::default()
}, // the query will have 5 output nodes, thus exceeding the limit
r#"
query {
multiGetObjects(
keys: [
{objectId: "0x01dcb4674affb04e68d8088895e951f4ea335ef1695e9e50c166618f6789d808", version: 2},
{objectId: "0x23e340e97fb41249278c85b1f067dc88576f750670c6dc56572e90971f857c8c", version: 2},
{objectId: "0x23e340e97fb41249278c85b1f067dc88576f750670c6dc56572e90971f857c8c", version: 2},
{objectId: "0x33032e0706337632361f2607b79df8c9d1079e8069259b27b1fa5c0394e79893", version: 2},
{objectId: "0x388295e3ecad53986ebf9a7a1e5854b7df94c3f1f0bba934c5396a2a9eb4550b", version: 2},
]
) {
address
}
}
"#
.into(),
)
.await,
"Estimated output nodes exceeds 4"
);
}
}
38 changes: 37 additions & 1 deletion crates/sui-graphql-rpc/src/types/object.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,9 @@ pub(crate) struct ObjectFilter {
/// Filter for live objects by their IDs.
pub object_ids: Option<Vec<SuiAddress>>,

/// Filter for live or potentially historical objects by their ID and version.
/// Filter for live objects by their ID and version. NOTE: this input filter has been
/// deprecated in favor of `multiGetObjects` query as it does not make sense to query for live
/// objects by their versions. This filter will be removed with v1.42.0 release.
pub object_keys: Option<Vec<ObjectKey>>,
}

Expand Down Expand Up @@ -801,6 +803,40 @@ impl Object {
self.root_version
}

/// Fetch objects by their id and version. If you need to query for live objects, use the
/// `objects` field.
pub(crate) async fn query_many(
ctx: &Context<'_>,
keys: Vec<ObjectKey>,
checkpoint_viewed_at: u64,
) -> Result<Vec<Self>, Error> {
let DataLoader(loader) = &ctx.data_unchecked();

let keys: Vec<PointLookupKey> = keys
.into_iter()
.map(|key| PointLookupKey {
id: key.object_id,
version: key.version.into(),
})
.collect();

let data = loader.load_many(keys).await?;
let objects = data
.into_iter()
.filter_map(|(lookup_key, bcs)| {
Object::new_serialized(
lookup_key.id,
lookup_key.version,
bcs,
checkpoint_viewed_at,
lookup_key.version,
)
})
.collect::<Vec<_>>();
Comment on lines +824 to +835
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
let objects = data
.into_iter()
.filter_map(|(lookup_key, bcs)| {
Object::new_serialized(
lookup_key.id,
lookup_key.version,
bcs,
checkpoint_viewed_at,
lookup_key.version,
)
})
.collect::<Vec<_>>();
let objects: Vec<_> = data
.into_iter()
.filter_map(|(lookup_key, bcs)| {
Object::new_serialized(
lookup_key.id,
lookup_key.version,
bcs,
checkpoint_viewed_at,
lookup_key.version,
)
})
.collect();


Ok(objects)
}

/// Query the database for a `page` of objects, optionally `filter`-ed.
///
/// `checkpoint_viewed_at` represents the checkpoint sequence number at which this page was
Expand Down
21 changes: 21 additions & 0 deletions crates/sui-graphql-rpc/src/types/query.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ use super::move_package::{
};
use super::move_registry::named_move_package::NamedMovePackage;
use super::move_registry::named_type::NamedType;
use super::object::ObjectKey;
use super::suins_registration::NameService;
use super::uint53::UInt53;
use super::{
Expand Down Expand Up @@ -327,6 +328,26 @@ impl Query {
TransactionBlock::query(ctx, lookup).await.extend()
}

/// Fetch a list of objects by their IDs and versions.
async fn multi_get_objects(
&self,
ctx: &Context<'_>,
keys: Vec<ObjectKey>,
) -> Result<Vec<Object>> {
let cfg: &ServiceConfig = ctx.data_unchecked();
if keys.len() > cfg.limits.max_multi_get_objects_keys as usize {
return Err(Error::Client(format!(
"Number of keys exceeds max limit of '{}'",
cfg.limits.max_multi_get_objects_keys
))
.into());
}

let Watermark { hi_cp, .. } = *ctx.data()?;

Object::query_many(ctx, keys, hi_cp).await.extend()
}

/// The coin objects that exist in the network.
///
/// The type field is a string of the inner type of the coin by which to filter (e.g.
Expand Down
12 changes: 11 additions & 1 deletion crates/sui-graphql-rpc/staging.graphql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2899,7 +2899,9 @@ input ObjectFilter {
"""
objectIds: [SuiAddress!]
"""
Filter for live or potentially historical objects by their ID and version.
Filter for live objects by their ID and version. NOTE: this input filter has been
deprecated in favor of `multiGetObjects` query as it does not make sense to query for live
objects by their versions. This filter will be removed with v1.42.0 release.
"""
objectKeys: [ObjectKey!]
}
Expand Down Expand Up @@ -3325,6 +3327,10 @@ type Query {
"""
transactionBlock(digest: String!): TransactionBlock
"""
Fetch a list of objects by their IDs and versions.
"""
multiGetObjects(keys: [ObjectKey!]!): [Object!]!
"""
The coin objects that exist in the network.

The type field is a string of the inner type of the coin by which to filter (e.g.
Expand Down Expand Up @@ -3599,6 +3605,10 @@ type ServiceConfig {
"""
maxTransactionIds: Int!
"""
Maximum number of keys that can be passed to a `multiGetObjects` query.
"""
maxMultiGetObjectsKeys: Int!
"""
Maximum number of candidates to scan when gathering a page of results.
"""
maxScanLimit: Int!
Expand Down
Loading
Loading