From 6941e362a7d49c8dc97ccdcfc62c24e471b5c0a5 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Thu, 21 Nov 2024 13:40:00 +0800 Subject: [PATCH 01/18] refactor: use temp table to refactor materialized cte --- src/query/settings/src/settings_default.rs | 6 ++ .../settings/src/settings_getter_setter.rs | 4 + .../sql/src/planner/binder/bind_context.rs | 52 --------- .../bind_mutation/mutation_expression.rs | 5 +- .../sql/src/planner/binder/bind_query/bind.rs | 45 +++++++- .../binder/bind_table_reference/bind_table.rs | 7 +- src/query/sql/src/planner/binder/binder.rs | 2 - src/query/sql/src/planner/binder/table.rs | 100 ------------------ .../standalone/explain/materialized_cte.test | 92 ++++++++-------- 9 files changed, 102 insertions(+), 211 deletions(-) diff --git a/src/query/settings/src/settings_default.rs b/src/query/settings/src/settings_default.rs index e87b519c1fae..6c6d39123687 100644 --- a/src/query/settings/src/settings_default.rs +++ b/src/query/settings/src/settings_default.rs @@ -952,6 +952,12 @@ impl DefaultSettings { mode: SettingMode::Both, range: Some(SettingRange::Numeric(0..=1)), }), + ("persist_materialized_cte", DefaultSettingValue { + value: UserSettingValue::UInt64(0), // 0 for in-memory, 1 for disk + desc: "Decides if materialized CTEs should be persisted to disk.", + mode: SettingMode::Both, + range: Some(SettingRange::Numeric(0..=1)), + }), ]); Ok(Arc::new(DefaultSettings { diff --git a/src/query/settings/src/settings_getter_setter.rs b/src/query/settings/src/settings_getter_setter.rs index 12d4746ab8e9..5dc160133268 100644 --- a/src/query/settings/src/settings_getter_setter.rs +++ b/src/query/settings/src/settings_getter_setter.rs @@ -821,4 +821,8 @@ impl Settings { pub fn get_enable_distributed_pruning(&self) -> Result { Ok(self.try_get_u64("enable_distributed_pruning")? == 1) } + + pub fn get_persist_materialized_cte(&self) -> Result { + Ok(self.try_get_u64("persist_materialized_cte")? != 0) + } } diff --git a/src/query/sql/src/planner/binder/bind_context.rs b/src/query/sql/src/planner/binder/bind_context.rs index 5d8e54e5509f..b52c3a733aa8 100644 --- a/src/query/sql/src/planner/binder/bind_context.rs +++ b/src/query/sql/src/planner/binder/bind_context.rs @@ -17,7 +17,6 @@ use std::collections::BTreeMap; use std::collections::HashMap; use std::collections::HashSet; use std::hash::Hash; -use std::sync::Arc; use dashmap::DashMap; use databend_common_ast::ast::Identifier; @@ -45,9 +44,6 @@ use crate::binder::project_set::SetReturningInfo; use crate::binder::window::WindowInfo; use crate::binder::ColumnBindingBuilder; use crate::normalize_identifier; -use crate::optimizer::SExpr; -use crate::plans::MaterializedCte; -use crate::plans::RelOperator; use crate::plans::ScalarExpr; use crate::ColumnSet; use crate::IndexType; @@ -182,28 +178,10 @@ pub struct BindContext { pub struct CteContext { /// If the `BindContext` is created from a CTE, record the cte name pub cte_name: Option, - /// Use `IndexMap` because need to keep the insertion order - /// Then wrap materialized ctes to main plan. pub cte_map: Box>, - /// Record the bound s_expr of materialized cte - pub m_cte_bound_s_expr: HashMap, - pub m_cte_materialized_indexes: HashMap, } impl CteContext { - pub fn set_m_cte_bound_s_expr(&mut self, cte_idx: IndexType, s_expr: SExpr) { - self.m_cte_bound_s_expr.insert(cte_idx, s_expr); - } - - pub fn set_m_cte_materialized_indexes(&mut self, cte_idx: IndexType, index: IndexType) { - self.m_cte_materialized_indexes.insert(cte_idx, index); - } - - // Check if the materialized cte has been bound. - pub fn has_bound(&self, cte_idx: IndexType) -> bool { - self.m_cte_bound_s_expr.contains_key(&cte_idx) - } - // Merge two `CteContext` into one. pub fn merge(&mut self, other: CteContext) { let mut merged_cte_map = IndexMap::new(); @@ -217,41 +195,11 @@ impl CteContext { } } self.cte_map = Box::new(merged_cte_map); - self.m_cte_bound_s_expr.extend(other.m_cte_bound_s_expr); - self.m_cte_materialized_indexes - .extend(other.m_cte_materialized_indexes); - } - - // Wrap materialized cte to main plan. - // It will be called at the end of binding. - pub fn wrap_m_cte(&self, mut s_expr: SExpr) -> SExpr { - for (_, cte_info) in self.cte_map.iter().rev() { - if !cte_info.materialized { - continue; - } - if let Some(cte_s_expr) = self.m_cte_bound_s_expr.get(&cte_info.cte_idx) { - let materialized_output_columns = cte_info.columns.clone(); - s_expr = SExpr::create_binary( - Arc::new(RelOperator::MaterializedCte(MaterializedCte { - cte_idx: cte_info.cte_idx, - materialized_output_columns, - materialized_indexes: self.m_cte_materialized_indexes.clone(), - })), - Arc::new(s_expr), - Arc::new(cte_s_expr.clone()), - ); - } - } - s_expr } // Set cte context to current `BindContext`. - // To make sure the last `BindContext` of the whole binding phase contains `cte context` - // Then we can wrap materialized cte to main plan. pub fn set_cte_context(&mut self, cte_context: CteContext) { self.cte_map = cte_context.cte_map; - self.m_cte_bound_s_expr = cte_context.m_cte_bound_s_expr; - self.m_cte_materialized_indexes = cte_context.m_cte_materialized_indexes; } } diff --git a/src/query/sql/src/planner/binder/bind_mutation/mutation_expression.rs b/src/query/sql/src/planner/binder/bind_mutation/mutation_expression.rs index 66dddfe8b131..406761815f1e 100644 --- a/src/query/sql/src/planner/binder/bind_mutation/mutation_expression.rs +++ b/src/query/sql/src/planner/binder/bind_mutation/mutation_expression.rs @@ -92,7 +92,7 @@ impl MutationExpression { mutation_strategy, } => { // Bind source table reference. - let (mut source_s_expr, mut source_context) = + let (source_s_expr, mut source_context) = binder.bind_table_reference(bind_context, source)?; // Bind target table reference. @@ -119,9 +119,6 @@ impl MutationExpression { required_columns.insert(*column_index); } - // Wrap `LogicalMaterializedCte` to `source_expr`. - source_s_expr = source_context.cte_context.wrap_m_cte(source_s_expr); - // When there is "update *" or "insert *", prepare all source columns. let all_source_columns = Self::all_source_columns( *has_star_clause, diff --git a/src/query/sql/src/planner/binder/bind_query/bind.rs b/src/query/sql/src/planner/binder/bind_query/bind.rs index 7c5a1314f869..a903a9d8c810 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind.rs @@ -14,10 +14,15 @@ use std::sync::Arc; +use databend_common_ast::ast::CreateOption; +use databend_common_ast::ast::CreateTableStmt; +use databend_common_ast::ast::Engine; use databend_common_ast::ast::Expr; use databend_common_ast::ast::Query; use databend_common_ast::ast::SetExpr; +use databend_common_ast::ast::TableType; use databend_common_ast::ast::With; +use databend_common_ast::ast::CTE; use databend_common_exception::ErrorCode; use databend_common_exception::Result; @@ -84,15 +89,19 @@ impl Binder { let cte_info = CteInfo { columns_alias: column_name, query: *cte.query.clone(), - materialized: cte.materialized, recursive: with.recursive, cte_idx: idx, columns: vec![], + materialized: cte.materialized, }; bind_context .cte_context .cte_map .insert(table_name, cte_info); + // If the CTE is materialized, we'll construct a temp table for it. + if cte.materialized { + self.m_cte_to_temp_table(cte)?; + } } Ok(()) @@ -165,4 +174,38 @@ impl Binder { Arc::new(child), )) } + + fn m_cte_to_temp_table(&self, cte: &CTE) -> Result<()> { + let engine = if self.ctx.get_settings().get_persist_materialized_cte()? { + Engine::Fuse + } else { + Engine::Memory + }; + + let create_table_stmt = CreateTableStmt { + create_option: CreateOption::CreateOrReplace, + catalog: None, + database: None, + table: cte.alias.name.clone(), + source: None, + engine: Some(engine), + uri_location: None, + cluster_by: None, + table_options: Default::default(), + as_query: Some(cte.query.clone()), + table_type: TableType::Temporary, + }; + + let create_table_sql = create_table_stmt.to_string(); + if let Some(subquery_executor) = &self.subquery_executor { + let _ = databend_common_base::runtime::block_on(async move { + subquery_executor + .execute_query_with_sql_string(&create_table_sql) + .await + })?; + } else { + return Err(ErrorCode::Internal("Binder's Subquery executor is not set")); + }; + Ok(()) + } } diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs index cfc8bd99a80c..3d5ea59fbd0b 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs @@ -71,7 +71,9 @@ impl Binder { // Check and bind common table expression let cte_map = bind_context.cte_context.cte_map.clone(); - if let Some(cte_info) = cte_map.get(&table_name) { + if let Some(cte_info) = cte_map.get(&table_name) + && !cte_info.materialized + { if self .metadata .read() @@ -186,9 +188,6 @@ impl Binder { unreachable!() }; let (s_expr, mut new_bind_context) = self.bind_query(&mut new_bind_context, query)?; - bind_context - .cte_context - .set_cte_context(new_bind_context.cte_context.clone()); let cols = table_meta .schema() diff --git a/src/query/sql/src/planner/binder/binder.rs b/src/query/sql/src/planner/binder/binder.rs index d99b52b9c9f8..9c1d566647f0 100644 --- a/src/query/sql/src/planner/binder/binder.rs +++ b/src/query/sql/src/planner/binder/binder.rs @@ -179,8 +179,6 @@ impl<'a> Binder { Statement::Query(query) => { let (mut s_expr, bind_context) = self.bind_query(bind_context, query)?; - // Wrap `LogicalMaterializedCte` to `s_expr` - s_expr = bind_context.cte_context.wrap_m_cte(s_expr); // Remove unused cache columns and join conditions and construct ExpressionScan's child. (s_expr, _) = self.construct_expression_scan(&s_expr, self.metadata.clone())?; let formatted_ast = if self.ctx.get_settings().get_enable_query_result_cache()? { diff --git a/src/query/sql/src/planner/binder/table.rs b/src/query/sql/src/planner/binder/table.rs index a7560c519ab5..fb2443ee870e 100644 --- a/src/query/sql/src/planner/binder/table.rs +++ b/src/query/sql/src/planner/binder/table.rs @@ -55,7 +55,6 @@ use databend_common_storage::StageFileInfo; use databend_common_storage::StageFilesInfo; use databend_storages_common_table_meta::table::ChangeType; use log::info; -use parking_lot::RwLock; use crate::binder::Binder; use crate::binder::ColumnBindingBuilder; @@ -63,10 +62,8 @@ use crate::binder::CteInfo; use crate::binder::ExprContext; use crate::binder::Visibility; use crate::optimizer::SExpr; -use crate::optimizer::StatInfo; use crate::planner::semantic::normalize_identifier; use crate::planner::semantic::TypeChecker; -use crate::plans::CteScan; use crate::plans::DummyTableScan; use crate::plans::RecursiveCteScan; use crate::plans::RelOperator; @@ -156,56 +153,6 @@ impl Binder { Ok((s_expr, bind_context)) } - fn bind_cte_scan( - &mut self, - cte_info: &CteInfo, - mut bind_context: BindContext, - ) -> Result<(SExpr, BindContext)> { - let blocks = Arc::new(RwLock::new(vec![])); - let cte_distinct_number = 1 + self - .ctx - .get_materialized_ctes() - .read() - .iter() - .filter(|((idx, _), _)| idx == &cte_info.cte_idx) - .count(); - self.ctx - .set_materialized_cte((cte_info.cte_idx, cte_distinct_number), blocks)?; - // Get the fields in the cte - let mut fields = vec![]; - let mut offsets = vec![]; - let mut materialized_indexes = vec![]; - for (idx, column) in bind_context.columns.iter_mut().enumerate() { - let materialized_index = column.index; - materialized_indexes.push(materialized_index); - column.index = self.metadata.write().add_derived_column( - column.column_name.clone(), - *column.data_type.clone(), - None, - ); - bind_context - .cte_context - .m_cte_materialized_indexes - .insert(column.index, materialized_index); - fields.push(DataField::new( - column.index.to_string().as_str(), - *column.data_type.clone(), - )); - offsets.push(idx); - } - let cte_scan = SExpr::create_leaf(Arc::new( - CteScan { - cte_idx: (cte_info.cte_idx, cte_distinct_number), - fields, - materialized_indexes, - offsets, - stat: Arc::new(StatInfo::default()), - } - .into(), - )); - Ok((cte_scan, bind_context)) - } - pub(crate) fn bind_cte( &mut self, span: Span, @@ -282,53 +229,6 @@ impl Binder { Ok((s_expr, res_bind_context)) } - // Bind materialized cte - pub(crate) fn bind_m_cte( - &mut self, - bind_context: &mut BindContext, - cte_info: &CteInfo, - table_name: &String, - alias: &Option, - span: &Span, - ) -> Result<(SExpr, BindContext)> { - let new_bind_context = if !bind_context.cte_context.has_bound(cte_info.cte_idx) { - let (cte_s_expr, mut cte_bind_ctx) = - self.bind_cte(*span, bind_context, table_name, alias, cte_info)?; - cte_bind_ctx - .cte_context - .cte_map - .entry(table_name.clone()) - .and_modify(|cte_info| { - cte_info.columns = cte_bind_ctx.columns.clone(); - }); - cte_bind_ctx - .cte_context - .set_m_cte_bound_s_expr(cte_info.cte_idx, cte_s_expr); - cte_bind_ctx - } else { - let mut bound_ctx = BindContext::with_parent(Box::new(bind_context.clone())); - // Resolve the alias name for the bound cte. - let alias_table_name = alias - .as_ref() - .map(|alias| normalize_identifier(&alias.name, &self.name_resolution_ctx).name) - .unwrap_or_else(|| table_name.to_string()); - for column in cte_info.columns.iter() { - let mut column = column.clone(); - column.database_name = None; - column.table_name = Some(alias_table_name.clone()); - bound_ctx.columns.push(column); - } - bound_ctx - }; - let cte_info = new_bind_context - .cte_context - .cte_map - .get(table_name) - .unwrap() - .clone(); - self.bind_cte_scan(&cte_info, new_bind_context) - } - pub(crate) fn bind_r_cte_scan( &mut self, bind_context: &mut BindContext, diff --git a/tests/sqllogictests/suites/mode/standalone/explain/materialized_cte.test b/tests/sqllogictests/suites/mode/standalone/explain/materialized_cte.test index c7ee3e19f1df..6638183e384b 100644 --- a/tests/sqllogictests/suites/mode/standalone/explain/materialized_cte.test +++ b/tests/sqllogictests/suites/mode/standalone/explain/materialized_cte.test @@ -1,61 +1,57 @@ -# Nest materialized cte query T explain with t1 as materialized (select number as a from numbers(10)), t2 as (select a as b from t1) select t1.a from t1 join t2 on t1.a = t2.b; ---- -MaterializedCTE -├── output columns: [a (#1)] -├── TableScan -│ ├── table: default.system.numbers -│ ├── output columns: [number (#0)] +HashJoin +├── output columns: [t1.a (#0)] +├── join type: INNER +├── build keys: [t2.b (#1)] +├── probe keys: [t1.a (#0)] +├── filters: [] +├── estimated rows: 0.00 +├── TableScan(Build) +│ ├── table: default.default.t1 +│ ├── output columns: [a (#1)] │ ├── read rows: 10 │ ├── read size: < 1 KiB -│ ├── partitions total: 1 -│ ├── partitions scanned: 1 +│ ├── partitions total: 0 +│ ├── partitions scanned: 0 │ ├── push downs: [filters: [], limit: NONE] -│ └── estimated rows: 10.00 -└── HashJoin - ├── output columns: [a (#1)] - ├── join type: INNER - ├── build keys: [t2.b (#2)] - ├── probe keys: [t1.a (#1)] - ├── filters: [] - ├── estimated rows: 100.00 - ├── CTEScan(Build) - │ ├── CTE index: 0, sub index: 2 - │ └── estimated rows: 10.00 - └── CTEScan(Probe) - ├── CTE index: 0, sub index: 1 - └── estimated rows: 10.00 +│ └── estimated rows: 0.00 +└── TableScan(Probe) + ├── table: default.default.t1 + ├── output columns: [a (#0)] + ├── read rows: 10 + ├── read size: < 1 KiB + ├── partitions total: 0 + ├── partitions scanned: 0 + ├── push downs: [filters: [], limit: NONE] + └── estimated rows: 0.00 query T explain with t1 as materialized (select number as a from numbers(10)), t2 as materialized (select a as b from t1) select t1.a from t1 join t2 on t1.a = t2.b; ---- -MaterializedCTE -├── output columns: [a (#1)] -├── TableScan -│ ├── table: default.system.numbers -│ ├── output columns: [number (#0)] +HashJoin +├── output columns: [t1.a (#0)] +├── join type: INNER +├── build keys: [t2.b (#1)] +├── probe keys: [t1.a (#0)] +├── filters: [] +├── estimated rows: 0.00 +├── TableScan(Build) +│ ├── table: default.default.t2 +│ ├── output columns: [b (#1)] │ ├── read rows: 10 │ ├── read size: < 1 KiB -│ ├── partitions total: 1 -│ ├── partitions scanned: 1 +│ ├── partitions total: 0 +│ ├── partitions scanned: 0 │ ├── push downs: [filters: [], limit: NONE] -│ └── estimated rows: 10.00 -└── MaterializedCTE - ├── output columns: [a (#1)] - ├── CTEScan - │ ├── CTE index: 0, sub index: 2 - │ └── estimated rows: 10.00 - └── HashJoin - ├── output columns: [a (#1)] - ├── join type: INNER - ├── build keys: [t2.b (#3)] - ├── probe keys: [t1.a (#1)] - ├── filters: [] - ├── estimated rows: 100.00 - ├── CTEScan(Build) - │ ├── CTE index: 1, sub index: 1 - │ └── estimated rows: 10.00 - └── CTEScan(Probe) - ├── CTE index: 0, sub index: 1 - └── estimated rows: 10.00 +│ └── estimated rows: 0.00 +└── TableScan(Probe) + ├── table: default.default.t1 + ├── output columns: [a (#0)] + ├── read rows: 10 + ├── read size: < 1 KiB + ├── partitions total: 0 + ├── partitions scanned: 0 + ├── push downs: [filters: [], limit: NONE] + └── estimated rows: 0.00 From 2dcf94326eab2cecddb6944f5066cb66e43914db Mon Sep 17 00:00:00 2001 From: xudong963 Date: Thu, 21 Nov 2024 14:54:11 +0800 Subject: [PATCH 02/18] remove useless code --- src/query/catalog/src/table_context.rs | 16 -- .../pipeline/sources/src/async_source.rs | 4 +- .../sources/src/prefetch_async_source.rs | 4 +- .../src/pipelines/builders/builder_join.rs | 73 ------- .../src/pipelines/builders/builder_scan.rs | 22 -- .../pipelines/builders/builder_union_all.rs | 2 - .../service/src/pipelines/pipeline_builder.rs | 13 -- .../pipelines/processors/transforms/mod.rs | 4 - .../transforms/transform_materialized_cte.rs | 193 ------------------ .../transform_recursive_cte_source.rs | 2 - src/query/service/src/sessions/query_ctx.rs | 24 --- .../service/src/sessions/query_ctx_shared.rs | 4 - .../tests/it/sql/exec/get_table_bind_test.rs | 21 -- .../it/storages/fuse/operations/commit.rs | 20 -- src/query/sql/src/executor/format.rs | 53 ----- src/query/sql/src/executor/physical_plan.rs | 27 --- .../sql/src/executor/physical_plan_builder.rs | 14 -- .../sql/src/executor/physical_plan_visitor.rs | 27 --- .../sql/src/executor/physical_plans/mod.rs | 4 - .../physical_plans/physical_cte_scan.rs | 91 --------- .../physical_materialized_cte.rs | 85 -------- src/query/sql/src/planner/binder/util.rs | 3 +- .../src/planner/optimizer/cost/cost_model.rs | 9 +- .../decorrelate/subquery_rewriter.rs | 13 +- .../dynamic_sample/dynamic_sample.rs | 9 - .../optimizer/filter/pull_up_filter.rs | 1 - src/query/sql/src/planner/optimizer/format.rs | 2 - .../src/planner/optimizer/hyper_dp/dphyp.rs | 2 - .../rule/rewrite/rule_semi_to_inner_join.rs | 2 - src/query/sql/src/planner/optimizer/s_expr.rs | 4 - .../statistics/collect_statistics.rs | 24 --- src/query/sql/src/planner/plans/cte_scan.rs | 110 ---------- .../sql/src/planner/plans/materialized_cte.rs | 114 ----------- src/query/sql/src/planner/plans/mod.rs | 4 - src/query/sql/src/planner/plans/operator.rs | 70 ------- .../push_down_filter_self_join.test | 63 +++--- .../sqllogictests/suites/tpch/join_order.test | 26 +-- 37 files changed, 43 insertions(+), 1116 deletions(-) delete mode 100644 src/query/service/src/pipelines/processors/transforms/transform_materialized_cte.rs delete mode 100644 src/query/sql/src/executor/physical_plans/physical_cte_scan.rs delete mode 100644 src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs delete mode 100644 src/query/sql/src/planner/plans/cte_scan.rs delete mode 100644 src/query/sql/src/planner/plans/materialized_cte.rs diff --git a/src/query/catalog/src/table_context.rs b/src/query/catalog/src/table_context.rs index c52ea832a4e6..0b6ed28fbc70 100644 --- a/src/query/catalog/src/table_context.rs +++ b/src/query/catalog/src/table_context.rs @@ -29,7 +29,6 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_exception::ResultExt; use databend_common_expression::BlockThresholds; -use databend_common_expression::DataBlock; use databend_common_expression::Expr; use databend_common_expression::FunctionContext; use databend_common_expression::Scalar; @@ -78,8 +77,6 @@ use crate::runtime_filter_info::RuntimeFilterReady; use crate::statistics::data_cache_statistics::DataCacheMetrics; use crate::table::Table; -pub type MaterializedCtesBlocks = Arc>>>>>; - pub struct ContextError; #[derive(Debug)] @@ -281,19 +278,6 @@ pub trait TableContext: Send + Sync { max_files: Option, ) -> Result; - fn set_materialized_cte( - &self, - idx: (usize, usize), - mem_table: Arc>>, - ) -> Result<()>; - - fn get_materialized_cte( - &self, - idx: (usize, usize), - ) -> Result>>>>; - - fn get_materialized_ctes(&self) -> MaterializedCtesBlocks; - fn add_segment_location(&self, segment_loc: Location) -> Result<()>; fn clear_segment_locations(&self) -> Result<()>; diff --git a/src/query/pipeline/sources/src/async_source.rs b/src/query/pipeline/sources/src/async_source.rs index 0c552f745670..84bf331d0df4 100644 --- a/src/query/pipeline/sources/src/async_source.rs +++ b/src/query/pipeline/sources/src/async_source.rs @@ -115,9 +115,7 @@ impl Processor for AsyncSourcer { match self.inner.generate().await? { None => self.is_finish = true, Some(data_block) => { - // Don't need to record the scan progress of `MaterializedCteSource` - // Because it reads data from memory. - if !data_block.is_empty() && self.name() != "MaterializedCteSource" { + if !data_block.is_empty() { let progress_values = ProgressValues { rows: data_block.num_rows(), bytes: data_block.memory_size(), diff --git a/src/query/pipeline/sources/src/prefetch_async_source.rs b/src/query/pipeline/sources/src/prefetch_async_source.rs index e04c03a2d7f8..878b57190f98 100644 --- a/src/query/pipeline/sources/src/prefetch_async_source.rs +++ b/src/query/pipeline/sources/src/prefetch_async_source.rs @@ -116,9 +116,7 @@ impl Processor for PrefetchAsyncSourcer { match self.inner.generate().await? { None => self.is_inner_finish = true, Some(data_block) => { - // Don't need to record the scan progress of `MaterializedCteSource` - // Because it reads data from memory. - if !data_block.is_empty() && self.name() != "MaterializedCteSource" { + if !data_block.is_empty() { let progress_values = ProgressValues { rows: data_block.num_rows(), bytes: data_block.memory_size(), diff --git a/src/query/service/src/pipelines/builders/builder_join.rs b/src/query/service/src/pipelines/builders/builder_join.rs index 07289a4ae51d..08fe01e8eb6e 100644 --- a/src/query/service/src/pipelines/builders/builder_join.rs +++ b/src/query/service/src/pipelines/builders/builder_join.rs @@ -19,19 +19,14 @@ use databend_common_exception::Result; use databend_common_pipeline_core::processors::ProcessorPtr; use databend_common_pipeline_sinks::Sinker; use databend_common_sql::executor::physical_plans::HashJoin; -use databend_common_sql::executor::physical_plans::MaterializedCte; use databend_common_sql::executor::physical_plans::RangeJoin; use databend_common_sql::executor::PhysicalPlan; -use databend_common_sql::ColumnBinding; -use databend_common_sql::IndexType; use crate::pipelines::processors::transforms::range_join::RangeJoinState; use crate::pipelines::processors::transforms::range_join::TransformRangeJoinLeft; use crate::pipelines::processors::transforms::range_join::TransformRangeJoinRight; use crate::pipelines::processors::transforms::HashJoinBuildState; use crate::pipelines::processors::transforms::HashJoinProbeState; -use crate::pipelines::processors::transforms::MaterializedCteSink; -use crate::pipelines::processors::transforms::MaterializedCteState; use crate::pipelines::processors::transforms::TransformHashJoinBuild; use crate::pipelines::processors::transforms::TransformHashJoinProbe; use crate::pipelines::processors::HashJoinDesc; @@ -77,8 +72,6 @@ impl PipelineBuilder { right_side_context, self.main_pipeline.get_scopes(), ); - right_side_builder.cte_state = self.cte_state.clone(); - right_side_builder.cte_scan_offsets = self.cte_scan_offsets.clone(); right_side_builder.hash_join_states = self.hash_join_states.clone(); let mut right_res = right_side_builder.finalize(&range_join.right)?; @@ -148,8 +141,6 @@ impl PipelineBuilder { build_side_context, self.main_pipeline.get_scopes(), ); - build_side_builder.cte_state = self.cte_state.clone(); - build_side_builder.cte_scan_offsets = self.cte_scan_offsets.clone(); build_side_builder.hash_join_states = self.hash_join_states.clone(); let mut build_res = build_side_builder.finalize(build)?; @@ -224,68 +215,4 @@ impl PipelineBuilder { Ok(()) } - - pub(crate) fn build_materialized_cte( - &mut self, - materialized_cte: &MaterializedCte, - ) -> Result<()> { - self.cte_scan_offsets.insert( - materialized_cte.cte_idx, - materialized_cte.cte_scan_offset.clone(), - ); - self.expand_materialized_side_pipeline( - &materialized_cte.right, - materialized_cte.cte_idx, - &materialized_cte.materialized_output_columns, - )?; - self.build_pipeline(&materialized_cte.left) - } - - fn expand_materialized_side_pipeline( - &mut self, - materialized_side: &PhysicalPlan, - cte_idx: IndexType, - materialized_output_columns: &[ColumnBinding], - ) -> Result<()> { - let materialized_side_ctx = QueryContext::create_from(self.ctx.clone()); - let state = Arc::new(MaterializedCteState::new(self.ctx.clone())); - self.cte_state.insert(cte_idx, state.clone()); - let mut materialized_side_builder = PipelineBuilder::create( - self.func_ctx.clone(), - self.settings.clone(), - materialized_side_ctx, - self.main_pipeline.get_scopes(), - ); - materialized_side_builder.cte_state = self.cte_state.clone(); - materialized_side_builder.cte_scan_offsets = self.cte_scan_offsets.clone(); - materialized_side_builder.hash_join_states = self.hash_join_states.clone(); - let mut materialized_side_pipeline = - materialized_side_builder.finalize(materialized_side)?; - assert!( - materialized_side_pipeline - .main_pipeline - .is_pulling_pipeline()? - ); - - PipelineBuilder::build_result_projection( - &self.func_ctx, - materialized_side.output_schema()?, - materialized_output_columns, - &mut materialized_side_pipeline.main_pipeline, - false, - )?; - - materialized_side_pipeline.main_pipeline.add_sink(|input| { - let transform = Sinker::::create( - input, - MaterializedCteSink::create(self.ctx.clone(), cte_idx, state.clone())?, - ); - Ok(ProcessorPtr::create(transform)) - })?; - self.pipelines - .push(materialized_side_pipeline.main_pipeline.finalize()); - self.pipelines - .extend(materialized_side_pipeline.sources_pipelines); - Ok(()) - } } diff --git a/src/query/service/src/pipelines/builders/builder_scan.rs b/src/query/service/src/pipelines/builders/builder_scan.rs index 4e85b4e6542e..c80f3331bb32 100644 --- a/src/query/service/src/pipelines/builders/builder_scan.rs +++ b/src/query/service/src/pipelines/builders/builder_scan.rs @@ -24,14 +24,12 @@ use databend_common_sql::evaluator::BlockOperator; use databend_common_sql::evaluator::CompoundBlockOperator; use databend_common_sql::executor::physical_plans::CacheScan; use databend_common_sql::executor::physical_plans::ConstantTableScan; -use databend_common_sql::executor::physical_plans::CteScan; use databend_common_sql::executor::physical_plans::ExpressionScan; use databend_common_sql::executor::physical_plans::TableScan; use databend_common_sql::plans::CacheSource; use crate::pipelines::processors::transforms::CacheSourceState; use crate::pipelines::processors::transforms::HashJoinCacheState; -use crate::pipelines::processors::transforms::MaterializedCteSource; use crate::pipelines::processors::transforms::TransformAddInternalColumns; use crate::pipelines::processors::transforms::TransformCacheScan; use crate::pipelines::processors::transforms::TransformExpressionScan; @@ -76,26 +74,6 @@ impl PipelineBuilder { Ok(()) } - pub(crate) fn build_cte_scan(&mut self, cte_scan: &CteScan) -> Result<()> { - let max_threads = self.settings.get_max_threads()?; - self.main_pipeline.add_source( - |output| { - MaterializedCteSource::create( - self.ctx.clone(), - output, - cte_scan.cte_idx, - self.cte_state.get(&cte_scan.cte_idx.0).unwrap().clone(), - cte_scan.offsets.clone(), - self.cte_scan_offsets - .get(&cte_scan.cte_idx.0) - .unwrap() - .clone(), - ) - }, - max_threads as usize, - ) - } - pub(crate) fn build_constant_table_scan(&mut self, scan: &ConstantTableScan) -> Result<()> { self.main_pipeline.add_source( |output| { diff --git a/src/query/service/src/pipelines/builders/builder_union_all.rs b/src/query/service/src/pipelines/builders/builder_union_all.rs index d0584e311f4f..ab4b572519ba 100644 --- a/src/query/service/src/pipelines/builders/builder_union_all.rs +++ b/src/query/service/src/pipelines/builders/builder_union_all.rs @@ -78,8 +78,6 @@ impl PipelineBuilder { union_ctx, self.main_pipeline.get_scopes(), ); - pipeline_builder.cte_state = self.cte_state.clone(); - pipeline_builder.cte_scan_offsets = self.cte_scan_offsets.clone(); pipeline_builder.hash_join_states = self.hash_join_states.clone(); let mut build_res = pipeline_builder.finalize(input)?; diff --git a/src/query/service/src/pipelines/pipeline_builder.rs b/src/query/service/src/pipelines/pipeline_builder.rs index 12e8463ee028..59b4f57e156f 100644 --- a/src/query/service/src/pipelines/pipeline_builder.rs +++ b/src/query/service/src/pipelines/pipeline_builder.rs @@ -25,12 +25,10 @@ use databend_common_pipeline_core::processors::PlanScopeGuard; use databend_common_pipeline_core::Pipeline; use databend_common_settings::Settings; use databend_common_sql::executor::PhysicalPlan; -use databend_common_sql::IndexType; use super::PipelineBuilderData; use crate::interpreters::CreateTableInterpreter; use crate::pipelines::processors::transforms::HashJoinBuildState; -use crate::pipelines::processors::transforms::MaterializedCteState; use crate::pipelines::processors::HashJoinState; use crate::pipelines::PipelineBuildResult; use crate::servers::flight::v1::exchange::DefaultExchangeInjector; @@ -49,11 +47,6 @@ pub struct PipelineBuilder { pub merge_into_probe_data_fields: Option>, pub join_state: Option>, - // The cte state of each materialized cte. - pub cte_state: HashMap>, - // The column offsets used by cte scan - pub cte_scan_offsets: HashMap>, - pub(crate) exchange_injector: Arc, pub hash_join_states: HashMap>, @@ -78,8 +71,6 @@ impl PipelineBuilder { pipelines: vec![], main_pipeline: Pipeline::with_scopes(scopes), exchange_injector: DefaultExchangeInjector::create(), - cte_state: HashMap::new(), - cte_scan_offsets: HashMap::new(), merge_into_probe_data_fields: None, join_state: None, hash_join_states: HashMap::new(), @@ -162,7 +153,6 @@ impl PipelineBuilder { match plan { PhysicalPlan::TableScan(scan) => self.build_table_scan(scan), - PhysicalPlan::CteScan(scan) => self.build_cte_scan(scan), PhysicalPlan::ConstantTableScan(scan) => self.build_constant_table_scan(scan), PhysicalPlan::Filter(filter) => self.build_filter(filter), PhysicalPlan::EvalScalar(eval_scalar) => self.build_eval_scalar(eval_scalar), @@ -189,9 +179,6 @@ impl PipelineBuilder { "Invalid physical plan with PhysicalPlan::Exchange", )), PhysicalPlan::RangeJoin(range_join) => self.build_range_join(range_join), - PhysicalPlan::MaterializedCte(materialized_cte) => { - self.build_materialized_cte(materialized_cte) - } PhysicalPlan::CacheScan(cache_scan) => self.build_cache_scan(cache_scan), PhysicalPlan::ExpressionScan(expression_scan) => { self.build_expression_scan(expression_scan) diff --git a/src/query/service/src/pipelines/processors/transforms/mod.rs b/src/query/service/src/pipelines/processors/transforms/mod.rs index 903ee416bacb..706135c9b079 100644 --- a/src/query/service/src/pipelines/processors/transforms/mod.rs +++ b/src/query/service/src/pipelines/processors/transforms/mod.rs @@ -28,7 +28,6 @@ mod transform_dictionary; mod transform_expression_scan; mod transform_filter; mod transform_limit; -mod transform_materialized_cte; mod transform_merge_block; mod transform_null_if; mod transform_recursive_cte_scan; @@ -55,9 +54,6 @@ pub use transform_create_sets::TransformCreateSets; pub use transform_expression_scan::TransformExpressionScan; pub use transform_filter::TransformFilter; pub use transform_limit::TransformLimit; -pub use transform_materialized_cte::MaterializedCteSink; -pub use transform_materialized_cte::MaterializedCteSource; -pub use transform_materialized_cte::MaterializedCteState; pub use transform_merge_block::TransformMergeBlock; pub use transform_null_if::TransformNullIf; pub use transform_recursive_cte_scan::TransformRecursiveCteScan; diff --git a/src/query/service/src/pipelines/processors/transforms/transform_materialized_cte.rs b/src/query/service/src/pipelines/processors/transforms/transform_materialized_cte.rs deleted file mode 100644 index bb371eaf64c8..000000000000 --- a/src/query/service/src/pipelines/processors/transforms/transform_materialized_cte.rs +++ /dev/null @@ -1,193 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::sync::Arc; - -use databend_common_base::base::tokio::sync::Notify; -use databend_common_catalog::table_context::TableContext; -use databend_common_exception::Result; -use databend_common_expression::BlockEntry; -use databend_common_expression::DataBlock; -use databend_common_pipeline_core::processors::OutputPort; -use databend_common_pipeline_core::processors::ProcessorPtr; -use databend_common_pipeline_sinks::Sink; -use databend_common_pipeline_sources::AsyncSource; -use databend_common_pipeline_sources::AsyncSourcer; -use databend_common_sql::IndexType; -use parking_lot::Mutex; -use parking_lot::RwLock; - -use crate::sessions::QueryContext; - -pub struct MaterializedCteState { - pub ctx: Arc, - pub left_sinker_count: Arc>, - pub sink_finished_notifier: Arc, - pub sink_finished: Arc>, -} - -impl MaterializedCteState { - pub fn new(ctx: Arc) -> Self { - MaterializedCteState { - ctx, - left_sinker_count: Arc::new(RwLock::new(0)), - sink_finished_notifier: Arc::new(Default::default()), - sink_finished: Arc::new(Mutex::new(false)), - } - } - - pub fn attach_sinker(&self) -> Result<()> { - let mut left_sinker_count = self.left_sinker_count.write(); - *left_sinker_count += 1; - Ok(()) - } - - pub fn detach_sinker(&self, cte_idx: IndexType) -> Result<()> { - let mut left_sinker_count = self.left_sinker_count.write(); - *left_sinker_count -= 1; - if *left_sinker_count == 0 { - // Sink finished. Clone materialized blocks to all materialized cte in ctx with same cte_idx - let blocks = self.ctx.get_materialized_cte((cte_idx, 1usize))?; - if let Some(blocks) = blocks { - let blocks = (*blocks).read(); - let ctes = self.ctx.get_materialized_ctes(); - for (idx, cte) in ctes.write().iter() { - if idx.0 == cte_idx && idx.1 != 1 { - let mut cte = cte.write(); - *cte = (*blocks).clone(); - } - } - } - let mut sink_finished = self.sink_finished.lock(); - *sink_finished = true; - self.sink_finished_notifier.notify_waiters(); - } - Ok(()) - } - - pub async fn wait_sink_finished(&self) -> Result<()> { - let notified = { - let sink_finished = self.sink_finished.lock(); - match *sink_finished { - true => None, - false => Some(self.sink_finished_notifier.notified()), - } - }; - - if let Some(notified) = notified { - notified.await; - } - Ok(()) - } -} - -pub struct MaterializedCteSink { - cte_idx: IndexType, - ctx: Arc, - blocks: Vec, - state: Arc, -} - -impl MaterializedCteSink { - pub fn create( - ctx: Arc, - cte_idx: IndexType, - state: Arc, - ) -> Result { - state.attach_sinker()?; - Ok(MaterializedCteSink { - cte_idx, - ctx, - blocks: vec![], - state, - }) - } -} - -impl Sink for MaterializedCteSink { - const NAME: &'static str = "MaterializedCteSink"; - - fn on_finish(&mut self) -> Result<()> { - let materialized_cte = self.ctx.get_materialized_cte((self.cte_idx, 1usize))?; - if let Some(blocks) = materialized_cte { - let mut blocks = blocks.write(); - blocks.extend(self.blocks.clone()); - } - self.state.detach_sinker(self.cte_idx) - } - - fn consume(&mut self, data_block: DataBlock) -> Result<()> { - self.blocks.push(data_block); - Ok(()) - } -} - -pub struct MaterializedCteSource { - cte_idx: (IndexType, IndexType), - ctx: Arc, - cte_state: Arc, - column_offsets: Vec, - scan_offsets: Vec, -} - -impl MaterializedCteSource { - pub fn create( - ctx: Arc, - output_port: Arc, - cte_idx: (IndexType, IndexType), - cte_state: Arc, - column_offsets: Vec, - scan_offsets: Vec, - ) -> Result { - AsyncSourcer::create(ctx.clone(), output_port, MaterializedCteSource { - ctx, - cte_idx, - cte_state, - column_offsets, - scan_offsets, - }) - } -} - -#[async_trait::async_trait] -impl AsyncSource for MaterializedCteSource { - const NAME: &'static str = "MaterializedCteSource"; - - #[async_backtrace::framed] - async fn generate(&mut self) -> Result> { - self.cte_state.wait_sink_finished().await?; - let materialized_cte = self.ctx.get_materialized_cte(self.cte_idx)?; - if let Some(blocks) = materialized_cte { - let mut blocks_guard = blocks.write(); - let data_block = blocks_guard.pop(); - if let Some(data_block) = data_block { - if self.column_offsets.len() == data_block.num_columns() { - return Ok(Some(data_block)); - } - let num_rows = data_block.num_rows(); - let pruned_columns = self - .column_offsets - .iter() - .map(|offset| data_block.get_by_offset(self.scan_offsets[*offset]).clone()) - .collect::>(); - - Ok(Some(DataBlock::new(pruned_columns, num_rows))) - } else { - Ok(None) - } - } else { - Ok(None) - } - } -} diff --git a/src/query/service/src/pipelines/processors/transforms/transform_recursive_cte_source.rs b/src/query/service/src/pipelines/processors/transforms/transform_recursive_cte_source.rs index e86b7b10c3b6..4fd297786eae 100644 --- a/src/query/service/src/pipelines/processors/transforms/transform_recursive_cte_source.rs +++ b/src/query/service/src/pipelines/processors/transforms/transform_recursive_cte_source.rs @@ -318,8 +318,6 @@ async fn create_memory_table_for_cte_scan( create_memory_table_for_cte_scan(ctx, plan.input.as_ref()).await?; } PhysicalPlan::TableScan(_) - | PhysicalPlan::CteScan(_) - | PhysicalPlan::MaterializedCte(_) | PhysicalPlan::ConstantTableScan(_) | PhysicalPlan::ExpressionScan(_) | PhysicalPlan::CacheScan(_) diff --git a/src/query/service/src/sessions/query_ctx.rs b/src/query/service/src/sessions/query_ctx.rs index 1a3b66d1e217..3ce989de87bd 100644 --- a/src/query/service/src/sessions/query_ctx.rs +++ b/src/query/service/src/sessions/query_ctx.rs @@ -54,14 +54,12 @@ use databend_common_catalog::statistics::data_cache_statistics::DataCacheMetrics use databend_common_catalog::table_args::TableArgs; use databend_common_catalog::table_context::ContextError; use databend_common_catalog::table_context::FilteredCopyFiles; -use databend_common_catalog::table_context::MaterializedCtesBlocks; use databend_common_catalog::table_context::StageAttachment; use databend_common_config::GlobalConfig; use databend_common_config::DATABEND_COMMIT_VERSION; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::BlockThresholds; -use databend_common_expression::DataBlock; use databend_common_expression::Expr; use databend_common_expression::FunctionContext; use databend_common_expression::Scalar; @@ -1063,28 +1061,6 @@ impl TableContext for QueryContext { }) } - fn set_materialized_cte( - &self, - idx: (IndexType, IndexType), - blocks: Arc>>, - ) -> Result<()> { - let mut ctes = self.shared.materialized_cte_tables.write(); - ctes.insert(idx, blocks); - Ok(()) - } - - fn get_materialized_cte( - &self, - idx: (IndexType, IndexType), - ) -> Result>>>> { - let ctes = self.shared.materialized_cte_tables.read(); - Ok(ctes.get(&idx).cloned()) - } - - fn get_materialized_ctes(&self) -> MaterializedCtesBlocks { - self.shared.materialized_cte_tables.clone() - } - fn add_segment_location(&self, segment_loc: Location) -> Result<()> { let mut segment_locations = self.inserted_segment_locs.write(); segment_locations.insert(segment_loc); diff --git a/src/query/service/src/sessions/query_ctx_shared.rs b/src/query/service/src/sessions/query_ctx_shared.rs index 05a672448721..7864f495d39f 100644 --- a/src/query/service/src/sessions/query_ctx_shared.rs +++ b/src/query/service/src/sessions/query_ctx_shared.rs @@ -34,7 +34,6 @@ use databend_common_catalog::runtime_filter_info::RuntimeFilterInfo; use databend_common_catalog::runtime_filter_info::RuntimeFilterReady; use databend_common_catalog::statistics::data_cache_statistics::DataCacheMetrics; use databend_common_catalog::table_context::ContextError; -use databend_common_catalog::table_context::MaterializedCtesBlocks; use databend_common_catalog::table_context::StageAttachment; use databend_common_exception::ErrorCode; use databend_common_exception::Result; @@ -126,8 +125,6 @@ pub struct QueryContextShared { // Client User-Agent pub(in crate::sessions) user_agent: Arc>, - /// Key is (cte index, used_count), value contains cte's materialized blocks - pub(in crate::sessions) materialized_cte_tables: MaterializedCtesBlocks, pub(in crate::sessions) query_profiles: Arc, PlanProfile>>>, @@ -187,7 +184,6 @@ impl QueryContextShared { enable_sort_spill: Arc::new(AtomicBool::new(true)), status: Arc::new(RwLock::new("null".to_string())), user_agent: Arc::new(RwLock::new("null".to_string())), - materialized_cte_tables: Arc::new(Default::default()), join_spill_progress: Arc::new(Progress::create()), agg_spill_progress: Arc::new(Progress::create()), group_by_spill_progress: Arc::new(Progress::create()), diff --git a/src/query/service/tests/it/sql/exec/get_table_bind_test.rs b/src/query/service/tests/it/sql/exec/get_table_bind_test.rs index c2b2fa719e94..89ff1fc3aa23 100644 --- a/src/query/service/tests/it/sql/exec/get_table_bind_test.rs +++ b/src/query/service/tests/it/sql/exec/get_table_bind_test.rs @@ -38,14 +38,12 @@ use databend_common_catalog::statistics::data_cache_statistics::DataCacheMetrics use databend_common_catalog::table::Table; use databend_common_catalog::table_context::ContextError; use databend_common_catalog::table_context::FilteredCopyFiles; -use databend_common_catalog::table_context::MaterializedCtesBlocks; use databend_common_catalog::table_context::ProcessInfo; use databend_common_catalog::table_context::StageAttachment; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::BlockThresholds; -use databend_common_expression::DataBlock; use databend_common_expression::Expr; use databend_common_expression::FunctionContext; use databend_common_expression::Scalar; @@ -841,25 +839,6 @@ impl TableContext for CtxDelegation { todo!() } - fn set_materialized_cte( - &self, - _idx: (usize, usize), - _blocks: Arc>>, - ) -> Result<()> { - todo!() - } - - fn get_materialized_cte( - &self, - _idx: (usize, usize), - ) -> Result>>>> { - todo!() - } - - fn get_materialized_ctes(&self) -> MaterializedCtesBlocks { - todo!() - } - fn add_segment_location(&self, _segment_loc: Location) -> Result<()> { todo!() } diff --git a/src/query/service/tests/it/storages/fuse/operations/commit.rs b/src/query/service/tests/it/storages/fuse/operations/commit.rs index d034eb65df53..00d6ddf6bd8f 100644 --- a/src/query/service/tests/it/storages/fuse/operations/commit.rs +++ b/src/query/service/tests/it/storages/fuse/operations/commit.rs @@ -37,7 +37,6 @@ use databend_common_catalog::statistics::data_cache_statistics::DataCacheMetrics use databend_common_catalog::table::Table; use databend_common_catalog::table_context::ContextError; use databend_common_catalog::table_context::FilteredCopyFiles; -use databend_common_catalog::table_context::MaterializedCtesBlocks; use databend_common_catalog::table_context::ProcessInfo; use databend_common_catalog::table_context::StageAttachment; use databend_common_catalog::table_context::TableContext; @@ -729,25 +728,6 @@ impl TableContext for CtxDelegation { HashMap::new() } - fn set_materialized_cte( - &self, - _idx: (usize, usize), - _blocks: Arc>>, - ) -> Result<()> { - todo!() - } - - fn get_materialized_cte( - &self, - _idx: (usize, usize), - ) -> Result>>>> { - todo!() - } - - fn get_materialized_ctes(&self) -> MaterializedCtesBlocks { - todo!() - } - fn add_segment_location(&self, _segment_loc: Location) -> Result<()> { todo!() } diff --git a/src/query/sql/src/executor/format.rs b/src/query/sql/src/executor/format.rs index e0463d9aa8a3..a671c196405a 100644 --- a/src/query/sql/src/executor/format.rs +++ b/src/query/sql/src/executor/format.rs @@ -38,7 +38,6 @@ use crate::executor::physical_plans::CommitSink; use crate::executor::physical_plans::ConstantTableScan; use crate::executor::physical_plans::CopyIntoLocation; use crate::executor::physical_plans::CopyIntoTable; -use crate::executor::physical_plans::CteScan; use crate::executor::physical_plans::DistributedInsertSelect; use crate::executor::physical_plans::EvalScalar; use crate::executor::physical_plans::Exchange; @@ -49,7 +48,6 @@ use crate::executor::physical_plans::Filter; use crate::executor::physical_plans::FragmentKind; use crate::executor::physical_plans::HashJoin; use crate::executor::physical_plans::Limit; -use crate::executor::physical_plans::MaterializedCte; use crate::executor::physical_plans::Mutation; use crate::executor::physical_plans::MutationManipulate; use crate::executor::physical_plans::MutationOrganize; @@ -157,19 +155,6 @@ impl PhysicalPlan { children, )) } - PhysicalPlan::CteScan(cte_scan) => cte_scan_to_format_tree(cte_scan), - PhysicalPlan::MaterializedCte(materialized_cte) => { - let right_child = materialized_cte.right.format_join(metadata)?; - let left_child = materialized_cte.left.format_join(metadata)?; - let children = vec![ - FormatTreeNode::with_children("Right".to_string(), vec![right_child]), - FormatTreeNode::with_children("Left".to_string(), vec![left_child]), - ]; - Ok(FormatTreeNode::with_children( - format!("MaterializedCte: {}", materialized_cte.cte_idx), - children, - )) - } PhysicalPlan::UnionAll(union_all) => { let left_child = union_all.left.format_join(metadata)?; let right_child = union_all.right.format_join(metadata)?; @@ -295,7 +280,6 @@ pub fn format_partial_tree( children, )) } - PhysicalPlan::CteScan(cte_scan) => cte_scan_to_format_tree(cte_scan), PhysicalPlan::UnionAll(union_all) => { let left_child = format_partial_tree(&union_all.left, metadata, profs)?; let right_child = format_partial_tree(&union_all.right, metadata, profs)?; @@ -389,13 +373,9 @@ fn to_format_tree( } PhysicalPlan::MutationOrganize(plan) => format_merge_into_organize(plan, metadata, profs), PhysicalPlan::AddStreamColumn(plan) => format_add_stream_column(plan, metadata, profs), - PhysicalPlan::CteScan(plan) => cte_scan_to_format_tree(plan), PhysicalPlan::RecursiveCteScan(_) => { Ok(FormatTreeNode::new("RecursiveCTEScan".to_string())) } - PhysicalPlan::MaterializedCte(plan) => { - materialized_cte_to_format_tree(plan, metadata, profs) - } PhysicalPlan::ConstantTableScan(plan) => constant_table_scan_to_format_tree(plan, metadata), PhysicalPlan::ExpressionScan(plan) => expression_scan_to_format_tree(plan, metadata, profs), PhysicalPlan::CacheScan(plan) => cache_scan_to_format_tree(plan, metadata), @@ -846,20 +826,6 @@ fn table_scan_to_format_tree( )) } -fn cte_scan_to_format_tree(plan: &CteScan) -> Result> { - let mut children = vec![FormatTreeNode::new(format!( - "CTE index: {}, sub index: {}", - plan.cte_idx.0, plan.cte_idx.1 - ))]; - let items = plan_stats_info_to_format_tree(&plan.stat); - children.extend(items); - - Ok(FormatTreeNode::with_children( - "CTEScan".to_string(), - children, - )) -} - fn constant_table_scan_to_format_tree( plan: &ConstantTableScan, metadata: &Metadata, @@ -1779,25 +1745,6 @@ fn udf_to_format_tree( Ok(FormatTreeNode::with_children("Udf".to_string(), children)) } -fn materialized_cte_to_format_tree( - plan: &MaterializedCte, - metadata: &Metadata, - profs: &HashMap, -) -> Result> { - let children = vec![ - FormatTreeNode::new(format!( - "output columns: [{}]", - format_output_columns(plan.output_schema()?, metadata, true) - )), - to_format_tree(&plan.right, metadata, profs)?, - to_format_tree(&plan.left, metadata, profs)?, - ]; - Ok(FormatTreeNode::with_children( - "MaterializedCTE".to_string(), - children, - )) -} - fn format_output_columns( output_schema: DataSchemaRef, metadata: &Metadata, diff --git a/src/query/sql/src/executor/physical_plan.rs b/src/query/sql/src/executor/physical_plan.rs index 4bbe18f59e0b..76b83e84557a 100644 --- a/src/query/sql/src/executor/physical_plan.rs +++ b/src/query/sql/src/executor/physical_plan.rs @@ -46,7 +46,6 @@ use crate::executor::physical_plans::ConstantTableScan; use crate::executor::physical_plans::CopyIntoLocation; use crate::executor::physical_plans::CopyIntoTable; use crate::executor::physical_plans::CopyIntoTableSource; -use crate::executor::physical_plans::CteScan; use crate::executor::physical_plans::DistributedInsertSelect; use crate::executor::physical_plans::Duplicate; use crate::executor::physical_plans::EvalScalar; @@ -57,7 +56,6 @@ use crate::executor::physical_plans::ExpressionScan; use crate::executor::physical_plans::Filter; use crate::executor::physical_plans::HashJoin; use crate::executor::physical_plans::Limit; -use crate::executor::physical_plans::MaterializedCte; use crate::executor::physical_plans::Mutation; use crate::executor::physical_plans::ProjectSet; use crate::executor::physical_plans::RangeJoin; @@ -94,8 +92,6 @@ pub enum PhysicalPlan { RangeJoin(RangeJoin), Exchange(Exchange), UnionAll(UnionAll), - CteScan(CteScan), - MaterializedCte(MaterializedCte), ConstantTableScan(ConstantTableScan), ExpressionScan(ExpressionScan), CacheScan(CacheScan), @@ -244,18 +240,10 @@ impl PhysicalPlan { plan.left.adjust_plan_id(next_id); plan.right.adjust_plan_id(next_id); } - PhysicalPlan::CteScan(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - } PhysicalPlan::RecursiveCteScan(plan) => { plan.plan_id = *next_id; *next_id += 1; } - PhysicalPlan::MaterializedCte(plan) => { - plan.plan_id = *next_id; - *next_id += 1; - } PhysicalPlan::ConstantTableScan(plan) => { plan.plan_id = *next_id; *next_id += 1; @@ -431,8 +419,6 @@ impl PhysicalPlan { PhysicalPlan::DistributedInsertSelect(v) => v.plan_id, PhysicalPlan::ExchangeSource(v) => v.plan_id, PhysicalPlan::ExchangeSink(v) => v.plan_id, - PhysicalPlan::CteScan(v) => v.plan_id, - PhysicalPlan::MaterializedCte(v) => v.plan_id, PhysicalPlan::ConstantTableScan(v) => v.plan_id, PhysicalPlan::ExpressionScan(v) => v.plan_id, PhysicalPlan::CacheScan(v) => v.plan_id, @@ -488,8 +474,6 @@ impl PhysicalPlan { PhysicalPlan::RangeJoin(plan) => plan.output_schema(), PhysicalPlan::CopyIntoTable(plan) => plan.output_schema(), PhysicalPlan::CopyIntoLocation(plan) => plan.output_schema(), - PhysicalPlan::CteScan(plan) => plan.output_schema(), - PhysicalPlan::MaterializedCte(plan) => plan.output_schema(), PhysicalPlan::ConstantTableScan(plan) => plan.output_schema(), PhysicalPlan::ExpressionScan(plan) => plan.output_schema(), PhysicalPlan::CacheScan(plan) => plan.output_schema(), @@ -563,9 +547,7 @@ impl PhysicalPlan { PhysicalPlan::MutationManipulate(_) => "MutationManipulate".to_string(), PhysicalPlan::MutationOrganize(_) => "MutationOrganize".to_string(), PhysicalPlan::AddStreamColumn(_) => "AddStreamColumn".to_string(), - PhysicalPlan::CteScan(_) => "PhysicalCteScan".to_string(), PhysicalPlan::RecursiveCteScan(_) => "RecursiveCteScan".to_string(), - PhysicalPlan::MaterializedCte(_) => "PhysicalMaterializedCte".to_string(), PhysicalPlan::ConstantTableScan(_) => "PhysicalConstantTableScan".to_string(), PhysicalPlan::ExpressionScan(_) => "ExpressionScan".to_string(), PhysicalPlan::CacheScan(_) => "CacheScan".to_string(), @@ -586,7 +568,6 @@ impl PhysicalPlan { pub fn children<'a>(&'a self) -> Box + 'a> { match self { PhysicalPlan::TableScan(_) - | PhysicalPlan::CteScan(_) | PhysicalPlan::ConstantTableScan(_) | PhysicalPlan::CacheScan(_) | PhysicalPlan::ExchangeSource(_) @@ -634,9 +615,6 @@ impl PhysicalPlan { } PhysicalPlan::MutationOrganize(plan) => Box::new(std::iter::once(plan.input.as_ref())), PhysicalPlan::AddStreamColumn(plan) => Box::new(std::iter::once(plan.input.as_ref())), - PhysicalPlan::MaterializedCte(plan) => Box::new( - std::iter::once(plan.right.as_ref()).chain(std::iter::once(plan.left.as_ref())), - ), PhysicalPlan::Udf(plan) => Box::new(std::iter::once(plan.input.as_ref())), PhysicalPlan::AsyncFunction(plan) => Box::new(std::iter::once(plan.input.as_ref())), PhysicalPlan::CopyIntoLocation(plan) => Box::new(std::iter::once(plan.input.as_ref())), @@ -680,7 +658,6 @@ impl PhysicalPlan { | PhysicalPlan::ExchangeSource(_) | PhysicalPlan::HashJoin(_) | PhysicalPlan::RangeJoin(_) - | PhysicalPlan::MaterializedCte(_) | PhysicalPlan::AggregateExpand(_) | PhysicalPlan::AggregateFinal(_) | PhysicalPlan::AggregatePartial(_) @@ -700,7 +677,6 @@ impl PhysicalPlan { | PhysicalPlan::ConstantTableScan(_) | PhysicalPlan::ExpressionScan(_) | PhysicalPlan::CacheScan(_) - | PhysicalPlan::CteScan(_) | PhysicalPlan::RecursiveCteScan(_) | PhysicalPlan::Recluster(_) | PhysicalPlan::Duplicate(_) @@ -860,9 +836,6 @@ impl PhysicalPlan { .iter() .map(|x| format!("{}({})", x.func_name, x.arg_exprs.join(", "))) .join(", "), - PhysicalPlan::CteScan(v) => { - format!("CTE index: {}, sub index: {}", v.cte_idx.0, v.cte_idx.1) - } PhysicalPlan::UnionAll(v) => v .left_outputs .iter() diff --git a/src/query/sql/src/executor/physical_plan_builder.rs b/src/query/sql/src/executor/physical_plan_builder.rs index e386d1d4b5d3..1b28a32d5f87 100644 --- a/src/query/sql/src/executor/physical_plan_builder.rs +++ b/src/query/sql/src/executor/physical_plan_builder.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::HashMap; -use std::collections::HashSet; use std::sync::Arc; use databend_common_catalog::plan::Partitions; @@ -30,8 +28,6 @@ use crate::optimizer::ColumnSet; use crate::optimizer::RelExpr; use crate::optimizer::SExpr; use crate::plans::RelOperator; -use crate::ColumnBinding; -use crate::IndexType; use crate::MetadataRef; pub struct PhysicalPlanBuilder { @@ -39,10 +35,6 @@ pub struct PhysicalPlanBuilder { pub(crate) ctx: Arc, pub(crate) func_ctx: FunctionContext, pub(crate) dry_run: bool, - // Record cte_idx and the cte's output columns - pub(crate) cte_output_columns: HashMap>, - // The used column offsets of each materialized cte. - pub(crate) cet_used_column_offsets: HashMap>, // DataMutation info, used to build MergeInto physical plan pub(crate) mutation_build_info: Option, } @@ -55,8 +47,6 @@ impl PhysicalPlanBuilder { ctx, func_ctx, dry_run, - cte_output_columns: Default::default(), - cet_used_column_offsets: Default::default(), mutation_build_info: None, } } @@ -115,10 +105,6 @@ impl PhysicalPlanBuilder { self.build_project_set(s_expr, project_set, required, stat_info) .await } - RelOperator::CteScan(cte_scan) => self.build_cte_scan(cte_scan, required).await, - RelOperator::MaterializedCte(cte) => { - self.build_materialized_cte(s_expr, cte, required).await - } RelOperator::ConstantTableScan(scan) => { self.build_constant_table_scan(scan, required).await } diff --git a/src/query/sql/src/executor/physical_plan_visitor.rs b/src/query/sql/src/executor/physical_plan_visitor.rs index dd86b4cda858..7f53a14b515f 100644 --- a/src/query/sql/src/executor/physical_plan_visitor.rs +++ b/src/query/sql/src/executor/physical_plan_visitor.rs @@ -40,7 +40,6 @@ use crate::executor::physical_plans::ConstantTableScan; use crate::executor::physical_plans::CopyIntoLocation; use crate::executor::physical_plans::CopyIntoTable; use crate::executor::physical_plans::CopyIntoTableSource; -use crate::executor::physical_plans::CteScan; use crate::executor::physical_plans::DistributedInsertSelect; use crate::executor::physical_plans::Duplicate; use crate::executor::physical_plans::EvalScalar; @@ -50,7 +49,6 @@ use crate::executor::physical_plans::ExchangeSource; use crate::executor::physical_plans::Filter; use crate::executor::physical_plans::HashJoin; use crate::executor::physical_plans::Limit; -use crate::executor::physical_plans::MaterializedCte; use crate::executor::physical_plans::Mutation; use crate::executor::physical_plans::MutationSource; use crate::executor::physical_plans::ProjectSet; @@ -72,7 +70,6 @@ pub trait PhysicalPlanReplacer { fn replace(&mut self, plan: &PhysicalPlan) -> Result { match plan { PhysicalPlan::TableScan(plan) => self.replace_table_scan(plan), - PhysicalPlan::CteScan(plan) => self.replace_cte_scan(plan), PhysicalPlan::RecursiveCteScan(plan) => self.replace_recursive_cte_scan(plan), PhysicalPlan::Filter(plan) => self.replace_filter(plan), PhysicalPlan::EvalScalar(plan) => self.replace_eval_scalar(plan), @@ -106,7 +103,6 @@ pub trait PhysicalPlanReplacer { PhysicalPlan::MutationManipulate(plan) => self.replace_mutation_manipulate(plan), PhysicalPlan::MutationOrganize(plan) => self.replace_mutation_organize(plan), PhysicalPlan::AddStreamColumn(plan) => self.replace_add_stream_column(plan), - PhysicalPlan::MaterializedCte(plan) => self.replace_materialized_cte(plan), PhysicalPlan::ConstantTableScan(plan) => self.replace_constant_table_scan(plan), PhysicalPlan::ExpressionScan(plan) => self.replace_expression_scan(plan), PhysicalPlan::CacheScan(plan) => self.replace_cache_scan(plan), @@ -133,10 +129,6 @@ pub trait PhysicalPlanReplacer { Ok(PhysicalPlan::TableScan(plan.clone())) } - fn replace_cte_scan(&mut self, plan: &CteScan) -> Result { - Ok(PhysicalPlan::CteScan(plan.clone())) - } - fn replace_recursive_cte_scan(&mut self, plan: &RecursiveCteScan) -> Result { Ok(PhysicalPlan::RecursiveCteScan(plan.clone())) } @@ -277,20 +269,6 @@ pub trait PhysicalPlanReplacer { })) } - fn replace_materialized_cte(&mut self, plan: &MaterializedCte) -> Result { - let right = self.replace(&plan.right)?; - let left = self.replace(&plan.left)?; - - Ok(PhysicalPlan::MaterializedCte(MaterializedCte { - plan_id: plan.plan_id, - left: Box::new(left), - right: Box::new(right), - cte_idx: plan.cte_idx, - cte_scan_offset: plan.cte_scan_offset.clone(), - materialized_output_columns: plan.materialized_output_columns.clone(), - })) - } - fn replace_range_join(&mut self, plan: &RangeJoin) -> Result { let left = self.replace(&plan.left)?; let right = self.replace(&plan.right)?; @@ -656,7 +634,6 @@ impl PhysicalPlan { match plan { PhysicalPlan::TableScan(_) | PhysicalPlan::ReplaceAsyncSourcer(_) - | PhysicalPlan::CteScan(_) | PhysicalPlan::RecursiveCteScan(_) | PhysicalPlan::ConstantTableScan(_) | PhysicalPlan::ExpressionScan(_) @@ -757,10 +734,6 @@ impl PhysicalPlan { PhysicalPlan::AddStreamColumn(plan) => { Self::traverse(&plan.input, pre_visit, visit, post_visit); } - PhysicalPlan::MaterializedCte(plan) => { - Self::traverse(&plan.left, pre_visit, visit, post_visit); - Self::traverse(&plan.right, pre_visit, visit, post_visit); - } PhysicalPlan::Udf(plan) => { Self::traverse(&plan.input, pre_visit, visit, post_visit); } diff --git a/src/query/sql/src/executor/physical_plans/mod.rs b/src/query/sql/src/executor/physical_plans/mod.rs index 957443396364..2bb277f10a73 100644 --- a/src/query/sql/src/executor/physical_plans/mod.rs +++ b/src/query/sql/src/executor/physical_plans/mod.rs @@ -25,7 +25,6 @@ mod physical_compact_source; mod physical_constant_table_scan; mod physical_copy_into_location; mod physical_copy_into_table; -mod physical_cte_scan; mod physical_distributed_insert_select; mod physical_eval_scalar; mod physical_exchange; @@ -36,7 +35,6 @@ mod physical_filter; mod physical_hash_join; mod physical_join; mod physical_limit; -mod physical_materialized_cte; mod physical_multi_table_insert; mod physical_mutation; mod physical_mutation_into_organize; @@ -74,7 +72,6 @@ pub use physical_compact_source::CompactSource; pub use physical_constant_table_scan::ConstantTableScan; pub use physical_copy_into_location::CopyIntoLocation; pub use physical_copy_into_table::*; -pub use physical_cte_scan::CteScan; pub use physical_distributed_insert_select::DistributedInsertSelect; pub use physical_eval_scalar::EvalScalar; pub use physical_exchange::Exchange; @@ -85,7 +82,6 @@ pub use physical_filter::Filter; pub use physical_hash_join::HashJoin; pub use physical_join::PhysicalJoinType; pub use physical_limit::Limit; -pub use physical_materialized_cte::MaterializedCte; pub use physical_multi_table_insert::*; pub use physical_mutation::*; pub use physical_mutation_into_organize::MutationOrganize; diff --git a/src/query/sql/src/executor/physical_plans/physical_cte_scan.rs b/src/query/sql/src/executor/physical_plans/physical_cte_scan.rs deleted file mode 100644 index ff2fb6823132..000000000000 --- a/src/query/sql/src/executor/physical_plans/physical_cte_scan.rs +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use databend_common_exception::Result; -use databend_common_expression::DataSchemaRef; -use databend_common_expression::DataSchemaRefExt; - -use crate::executor::explain::PlanStatsInfo; -use crate::executor::PhysicalPlan; -use crate::executor::PhysicalPlanBuilder; -use crate::ColumnSet; -use crate::IndexType; - -#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] -pub struct CteScan { - // A unique id of operator in a `PhysicalPlan` tree, only used for display. - pub plan_id: u32, - pub cte_idx: (IndexType, IndexType), - pub output_schema: DataSchemaRef, - pub offsets: Vec, - pub stat: PlanStatsInfo, -} - -impl CteScan { - pub fn output_schema(&self) -> Result { - Ok(self.output_schema.clone()) - } -} - -impl PhysicalPlanBuilder { - pub(crate) async fn build_cte_scan( - &mut self, - cte_scan: &crate::plans::CteScan, - required: ColumnSet, - ) -> Result { - // 1. Prune unused Columns. - let mut used_columns = cte_scan.used_columns()?; - used_columns = required.intersection(&used_columns).cloned().collect(); - let mut pruned_fields = vec![]; - let mut pruned_offsets = vec![]; - let mut pruned_materialized_indexes = vec![]; - let cte_output_columns = self.cte_output_columns.get(&cte_scan.cte_idx.0).unwrap(); - for (field, column_index) in cte_scan - .fields - .iter() - .zip(cte_scan.materialized_indexes.iter()) - { - if used_columns.contains(&field.name().parse()?) { - pruned_fields.push(field.clone()); - pruned_materialized_indexes.push(*column_index); - } - } - for column_index in pruned_materialized_indexes.iter() { - for (offset, col) in cte_output_columns.iter().enumerate() { - if col.index.eq(column_index) { - pruned_offsets.push(offset); - self.cet_used_column_offsets - .entry(cte_scan.cte_idx.0) - .and_modify(|column_offsets| { - column_offsets.insert(offset); - }); - break; - } - } - } - - let plan_stat = PlanStatsInfo { - estimated_rows: cte_scan.stat.cardinality, - }; - - // 2. Build physical plan. - Ok(PhysicalPlan::CteScan(CteScan { - plan_id: 0, - cte_idx: cte_scan.cte_idx, - output_schema: DataSchemaRefExt::create(pruned_fields), - offsets: pruned_offsets, - stat: plan_stat, - })) - } -} diff --git a/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs b/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs deleted file mode 100644 index 66c9e3d225ab..000000000000 --- a/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::HashSet; - -use databend_common_exception::Result; -use databend_common_expression::DataSchemaRef; -use databend_common_expression::DataSchemaRefExt; - -use crate::executor::PhysicalPlan; -use crate::executor::PhysicalPlanBuilder; -use crate::optimizer::SExpr; -use crate::ColumnBinding; -use crate::ColumnSet; -use crate::IndexType; - -#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] -pub struct MaterializedCte { - // A unique id of operator in a `PhysicalPlan` tree, only used for display. - pub plan_id: u32, - pub left: Box, - pub right: Box, - pub cte_idx: IndexType, - pub cte_scan_offset: Vec, - pub materialized_output_columns: Vec, -} - -impl MaterializedCte { - pub fn output_schema(&self) -> Result { - let fields = self.left.output_schema()?.fields().clone(); - Ok(DataSchemaRefExt::create(fields)) - } -} - -impl PhysicalPlanBuilder { - pub(crate) async fn build_materialized_cte( - &mut self, - s_expr: &SExpr, - cte: &crate::plans::MaterializedCte, - required: ColumnSet, - ) -> Result { - // 1. Prune unused Columns. - self.cte_output_columns - .insert(cte.cte_idx, cte.materialized_output_columns.clone()); - self.cet_used_column_offsets - .insert(cte.cte_idx, HashSet::new()); - let left = Box::new(self.build(s_expr.child(0)?, required).await?); - - let mut materialize_required = ColumnSet::new(); - let mut materialized_output_columns = vec![]; - let mut cte_scan_offset = Vec::with_capacity(cte.materialized_output_columns.len()); - let used_column_offset = self.cet_used_column_offsets.get(&cte.cte_idx).unwrap(); - for (offset, column) in cte.materialized_output_columns.iter().enumerate() { - if used_column_offset.contains(&offset) { - cte_scan_offset.push(materialized_output_columns.len()); - materialize_required.insert(column.index); - materialized_output_columns.push(column.clone()); - } else { - cte_scan_offset.push(0); - } - } - let right = Box::new(self.build(s_expr.child(1)?, materialize_required).await?); - - // 2. Build physical plan. - Ok(PhysicalPlan::MaterializedCte(MaterializedCte { - plan_id: 0, - left, - right, - cte_idx: cte.cte_idx, - cte_scan_offset, - materialized_output_columns, - })) - } -} diff --git a/src/query/sql/src/planner/binder/util.rs b/src/query/sql/src/planner/binder/util.rs index f082224649b4..e4acdd0ddccf 100644 --- a/src/query/sql/src/planner/binder/util.rs +++ b/src/query/sql/src/planner/binder/util.rs @@ -46,7 +46,7 @@ impl Binder { cte_types: &mut Vec, ) -> Result<()> { match expr.plan() { - RelOperator::Join(_) | RelOperator::UnionAll(_) | RelOperator::MaterializedCte(_) => { + RelOperator::Join(_) | RelOperator::UnionAll(_) => { self.count_r_cte_scan(expr.child(0)?, cte_scan_names, cte_types)?; self.count_r_cte_scan(expr.child(1)?, cte_scan_names, cte_types)?; } @@ -72,7 +72,6 @@ impl Binder { RelOperator::Exchange(_) | RelOperator::Scan(_) - | RelOperator::CteScan(_) | RelOperator::DummyTableScan(_) | RelOperator::ConstantTableScan(_) | RelOperator::ExpressionScan(_) diff --git a/src/query/sql/src/planner/optimizer/cost/cost_model.rs b/src/query/sql/src/planner/optimizer/cost/cost_model.rs index bdc566b3d85e..894f68a96fee 100644 --- a/src/query/sql/src/planner/optimizer/cost/cost_model.rs +++ b/src/query/sql/src/planner/optimizer/cost/cost_model.rs @@ -80,11 +80,10 @@ impl DefaultCostModel { match m_expr.plan.as_ref() { RelOperator::Scan(plan) => self.compute_cost_scan(memo, m_expr, plan), RelOperator::ConstantTableScan(plan) => self.compute_cost_constant_scan(plan), - RelOperator::DummyTableScan(_) | RelOperator::CteScan(_) => Ok(Cost(0.0)), + RelOperator::DummyTableScan(_) => Ok(Cost(0.0)), RelOperator::Join(plan) => self.compute_cost_join(memo, m_expr, plan), RelOperator::UnionAll(_) => self.compute_cost_union_all(memo, m_expr), RelOperator::Aggregate(_) => self.compute_aggregate(memo, m_expr), - RelOperator::MaterializedCte(_) => self.compute_materialized_cte(memo, m_expr), RelOperator::EvalScalar(_) | RelOperator::Filter(_) @@ -129,12 +128,6 @@ impl DefaultCostModel { Ok(Cost(cost)) } - fn compute_materialized_cte(&self, memo: &Memo, m_expr: &MExpr) -> Result { - let right_group = m_expr.child_group(memo, 1)?; - let cost = right_group.stat_info.cardinality * self.compute_per_row; - Ok(Cost(cost)) - } - /// Compute cost for the unary operators that perform simple computation(e.g. `Project`, `Filter`, `EvalScalar`). /// /// TODO(leiysky): Since we don't have alternation for `Aggregate` for now, we just diff --git a/src/query/sql/src/planner/optimizer/decorrelate/subquery_rewriter.rs b/src/query/sql/src/planner/optimizer/decorrelate/subquery_rewriter.rs index 5a2b34ceec35..dd945f90b290 100644 --- a/src/query/sql/src/planner/optimizer/decorrelate/subquery_rewriter.rs +++ b/src/query/sql/src/planner/optimizer/decorrelate/subquery_rewriter.rs @@ -174,13 +174,11 @@ impl SubqueryRewriter { Ok(SExpr::create_unary(Arc::new(sort.into()), Arc::new(input))) } - RelOperator::Join(_) | RelOperator::UnionAll(_) | RelOperator::MaterializedCte(_) => { - Ok(SExpr::create_binary( - Arc::new(s_expr.plan().clone()), - Arc::new(self.rewrite(s_expr.child(0)?)?), - Arc::new(self.rewrite(s_expr.child(1)?)?), - )) - } + RelOperator::Join(_) | RelOperator::UnionAll(_) => Ok(SExpr::create_binary( + Arc::new(s_expr.plan().clone()), + Arc::new(self.rewrite(s_expr.child(0)?)?), + Arc::new(self.rewrite(s_expr.child(1)?)?), + )), RelOperator::Limit(_) | RelOperator::Udf(_) | RelOperator::AsyncFunction(_) => { Ok(SExpr::create_unary( @@ -191,7 +189,6 @@ impl SubqueryRewriter { RelOperator::DummyTableScan(_) | RelOperator::Scan(_) - | RelOperator::CteScan(_) | RelOperator::ConstantTableScan(_) | RelOperator::ExpressionScan(_) | RelOperator::CacheScan(_) diff --git a/src/query/sql/src/planner/optimizer/dynamic_sample/dynamic_sample.rs b/src/query/sql/src/planner/optimizer/dynamic_sample/dynamic_sample.rs index a2363d69029b..a7b21ae4579f 100644 --- a/src/query/sql/src/planner/optimizer/dynamic_sample/dynamic_sample.rs +++ b/src/query/sql/src/planner/optimizer/dynamic_sample/dynamic_sample.rs @@ -86,7 +86,6 @@ pub async fn dynamic_sample( } RelOperator::Scan(_) | RelOperator::DummyTableScan(_) - | RelOperator::CteScan(_) | RelOperator::ConstantTableScan(_) | RelOperator::CacheScan(_) | RelOperator::ExpressionScan(_) @@ -135,14 +134,6 @@ pub async fn dynamic_sample( let project_set = ProjectSet::try_from(s_expr.plan().clone())?; project_set.derive_project_set_stats(&mut child_stat_info) } - RelOperator::MaterializedCte(_) => { - let right_stat_info = - dynamic_sample(ctx, metadata, s_expr.child(1)?, sample_executor).await?; - Ok(Arc::new(StatInfo { - cardinality: right_stat_info.cardinality, - statistics: right_stat_info.statistics.clone(), - })) - } RelOperator::EvalScalar(_) | RelOperator::Sort(_) diff --git a/src/query/sql/src/planner/optimizer/filter/pull_up_filter.rs b/src/query/sql/src/planner/optimizer/filter/pull_up_filter.rs index 058e863ee78d..1382f1521dea 100644 --- a/src/query/sql/src/planner/optimizer/filter/pull_up_filter.rs +++ b/src/query/sql/src/planner/optimizer/filter/pull_up_filter.rs @@ -72,7 +72,6 @@ impl PullUpFilterOptimizer { self.pull_up_join(s_expr, join) } RelOperator::EvalScalar(eval_scalar) => self.pull_up_eval_scalar(s_expr, eval_scalar), - RelOperator::MaterializedCte(_) => Ok(s_expr.clone()), _ => self.pull_up_others(s_expr), } } diff --git a/src/query/sql/src/planner/optimizer/format.rs b/src/query/sql/src/planner/optimizer/format.rs index b72f217ae4cb..b7aaa58ece77 100644 --- a/src/query/sql/src/planner/optimizer/format.rs +++ b/src/query/sql/src/planner/optimizer/format.rs @@ -65,8 +65,6 @@ pub fn display_rel_op(rel_op: &RelOperator) -> String { RelOperator::DummyTableScan(_) => "DummyTableScan".to_string(), RelOperator::ProjectSet(_) => "ProjectSet".to_string(), RelOperator::Window(_) => "WindowFunc".to_string(), - RelOperator::CteScan(_) => "CteScan".to_string(), - RelOperator::MaterializedCte(_) => "MaterializedCte".to_string(), RelOperator::ConstantTableScan(s) => s.name().to_string(), RelOperator::ExpressionScan(_) => "ExpressionScan".to_string(), RelOperator::CacheScan(_) => "CacheScan".to_string(), diff --git a/src/query/sql/src/planner/optimizer/hyper_dp/dphyp.rs b/src/query/sql/src/planner/optimizer/hyper_dp/dphyp.rs index bd968b999508..e741f6ac8195 100644 --- a/src/query/sql/src/planner/optimizer/hyper_dp/dphyp.rs +++ b/src/query/sql/src/planner/optimizer/hyper_dp/dphyp.rs @@ -292,9 +292,7 @@ impl DPhpy { | RelOperator::ConstantTableScan(_) | RelOperator::ExpressionScan(_) | RelOperator::CacheScan(_) - | RelOperator::CteScan(_) | RelOperator::AsyncFunction(_) - | RelOperator::MaterializedCte(_) | RelOperator::RecursiveCteScan(_) | RelOperator::Mutation(_) | RelOperator::MutationSource(_) diff --git a/src/query/sql/src/planner/optimizer/rule/rewrite/rule_semi_to_inner_join.rs b/src/query/sql/src/planner/optimizer/rule/rewrite/rule_semi_to_inner_join.rs index 29dfe4872f60..fab24e2e62d3 100644 --- a/src/query/sql/src/planner/optimizer/rule/rewrite/rule_semi_to_inner_join.rs +++ b/src/query/sql/src/planner/optimizer/rule/rewrite/rule_semi_to_inner_join.rs @@ -129,13 +129,11 @@ fn find_group_by_keys(child: &SExpr, group_by_keys: &mut HashSet) -> | RelOperator::UnionAll(_) | RelOperator::DummyTableScan(_) | RelOperator::ProjectSet(_) - | RelOperator::MaterializedCte(_) | RelOperator::ConstantTableScan(_) | RelOperator::ExpressionScan(_) | RelOperator::CacheScan(_) | RelOperator::Udf(_) | RelOperator::Scan(_) - | RelOperator::CteScan(_) | RelOperator::AsyncFunction(_) | RelOperator::Join(_) | RelOperator::RecursiveCteScan(_) diff --git a/src/query/sql/src/planner/optimizer/s_expr.rs b/src/query/sql/src/planner/optimizer/s_expr.rs index d56f921e5053..df4907d28beb 100644 --- a/src/query/sql/src/planner/optimizer/s_expr.rs +++ b/src/query/sql/src/planner/optimizer/s_expr.rs @@ -332,8 +332,6 @@ impl SExpr { | RelOperator::UnionAll(_) | RelOperator::Sort(_) | RelOperator::DummyTableScan(_) - | RelOperator::CteScan(_) - | RelOperator::MaterializedCte(_) | RelOperator::ConstantTableScan(_) | RelOperator::ExpressionScan(_) | RelOperator::CacheScan(_) @@ -432,8 +430,6 @@ fn find_subquery(rel_op: &RelOperator) -> bool { | RelOperator::UnionAll(_) | RelOperator::Sort(_) | RelOperator::DummyTableScan(_) - | RelOperator::CteScan(_) - | RelOperator::MaterializedCte(_) | RelOperator::ConstantTableScan(_) | RelOperator::ExpressionScan(_) | RelOperator::CacheScan(_) diff --git a/src/query/sql/src/planner/optimizer/statistics/collect_statistics.rs b/src/query/sql/src/planner/optimizer/statistics/collect_statistics.rs index f7c0d28e0d9c..f48b100e6ebe 100644 --- a/src/query/sql/src/planner/optimizer/statistics/collect_statistics.rs +++ b/src/query/sql/src/planner/optimizer/statistics/collect_statistics.rs @@ -22,9 +22,7 @@ use databend_common_expression::types::F64; use databend_common_expression::ColumnId; use databend_common_expression::Scalar; -use crate::optimizer::RelExpr; use crate::optimizer::SExpr; -use crate::optimizer::StatInfo; use crate::plans::ConstantExpr; use crate::plans::Filter; use crate::plans::FunctionCall; @@ -32,7 +30,6 @@ use crate::plans::RelOperator; use crate::plans::Statistics; use crate::BaseTableColumn; use crate::ColumnEntry; -use crate::IndexType; use crate::MetadataRef; use crate::ScalarExpr; @@ -40,7 +37,6 @@ use crate::ScalarExpr; pub struct CollectStatisticsOptimizer { table_ctx: Arc, metadata: MetadataRef, - cte_statistics: HashMap>, } impl CollectStatisticsOptimizer { @@ -48,7 +44,6 @@ impl CollectStatisticsOptimizer { CollectStatisticsOptimizer { table_ctx, metadata, - cte_statistics: HashMap::new(), } } @@ -145,25 +140,6 @@ impl CollectStatisticsOptimizer { } Ok(s_expr) } - RelOperator::MaterializedCte(materialized_cte) => { - // Collect the common table expression statistics first. - let right = Box::pin(self.collect(s_expr.child(1)?)).await?; - let cte_stat_info = RelExpr::with_s_expr(&right).derive_cardinality()?; - self.cte_statistics - .insert(materialized_cte.cte_idx, cte_stat_info); - let left = Box::pin(self.collect(s_expr.child(0)?)).await?; - Ok(s_expr.replace_children(vec![Arc::new(left), Arc::new(right)])) - } - RelOperator::CteScan(cte_scan) => { - let stat_info = self - .cte_statistics - .get(&cte_scan.cte_idx.0) - .unwrap() - .clone(); - let mut cte_scan = cte_scan.clone(); - cte_scan.stat = stat_info; - Ok(s_expr.replace_plan(Arc::new(RelOperator::CteScan(cte_scan)))) - } _ => { let mut children = Vec::with_capacity(s_expr.arity()); for child in s_expr.children() { diff --git a/src/query/sql/src/planner/plans/cte_scan.rs b/src/query/sql/src/planner/plans/cte_scan.rs deleted file mode 100644 index 5ca368cbb756..000000000000 --- a/src/query/sql/src/planner/plans/cte_scan.rs +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::hash::Hash; -use std::sync::Arc; - -use databend_common_catalog::table_context::TableContext; -use databend_common_exception::ErrorCode; -use databend_common_exception::Result; -use databend_common_expression::DataField; - -use crate::optimizer::ColumnSet; -use crate::optimizer::Distribution; -use crate::optimizer::PhysicalProperty; -use crate::optimizer::RelExpr; -use crate::optimizer::RelationalProperty; -use crate::optimizer::RequiredProperty; -use crate::optimizer::StatInfo; -use crate::plans::Operator; -use crate::plans::RelOp; -use crate::IndexType; - -#[derive(Clone, Debug)] -pub struct CteScan { - pub cte_idx: (usize, usize), - pub fields: Vec, - pub materialized_indexes: Vec, - pub offsets: Vec, - pub stat: Arc, -} - -impl CteScan { - pub fn used_columns(&self) -> Result { - let mut used_columns = ColumnSet::new(); - for field in self.fields.iter() { - used_columns.insert(field.name().parse()?); - } - Ok(used_columns) - } -} - -impl PartialEq for CteScan { - fn eq(&self, other: &Self) -> bool { - self.cte_idx == other.cte_idx - } -} - -impl Eq for CteScan {} - -impl Hash for CteScan { - fn hash(&self, state: &mut H) { - self.cte_idx.hash(state); - } -} - -impl Operator for CteScan { - fn rel_op(&self) -> RelOp { - RelOp::CteScan - } - - fn arity(&self) -> usize { - 0 - } - - fn derive_relational_prop(&self, _rel_expr: &RelExpr) -> Result> { - Ok(Arc::new(RelationalProperty { - output_columns: self.used_columns()?, - outer_columns: ColumnSet::new(), - used_columns: self.used_columns()?, - orderings: vec![], - partition_orderings: None, - })) - } - - fn derive_physical_prop(&self, _rel_expr: &RelExpr) -> Result { - Ok(PhysicalProperty { - distribution: Distribution::Serial, - }) - } - - fn derive_stats(&self, _rel_expr: &RelExpr) -> Result> { - Ok(Arc::new(StatInfo { - cardinality: self.stat.cardinality, - statistics: self.stat.statistics.clone(), - })) - } - - fn compute_required_prop_child( - &self, - _ctx: Arc, - _rel_expr: &RelExpr, - _child_index: usize, - _required: &RequiredProperty, - ) -> Result { - Err(ErrorCode::Internal( - "Cannot compute required property for CteScan".to_string(), - )) - } -} diff --git a/src/query/sql/src/planner/plans/materialized_cte.rs b/src/query/sql/src/planner/plans/materialized_cte.rs deleted file mode 100644 index 8a096b04fdaf..000000000000 --- a/src/query/sql/src/planner/plans/materialized_cte.rs +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::HashMap; -use std::sync::Arc; - -use databend_common_catalog::table_context::TableContext; -use databend_common_exception::Result; - -use crate::optimizer::Distribution; -use crate::optimizer::PhysicalProperty; -use crate::optimizer::RelExpr; -use crate::optimizer::RelationalProperty; -use crate::optimizer::RequiredProperty; -use crate::optimizer::StatInfo; -use crate::plans::Operator; -use crate::plans::RelOp; -use crate::ColumnBinding; -use crate::IndexType; - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct MaterializedCte { - pub(crate) cte_idx: IndexType, - pub(crate) materialized_output_columns: Vec, - pub(crate) materialized_indexes: HashMap, -} - -impl Operator for MaterializedCte { - fn rel_op(&self) -> RelOp { - RelOp::MaterializedCte - } - - fn arity(&self) -> usize { - 2 - } - - fn derive_relational_prop(&self, rel_expr: &RelExpr) -> Result> { - let left_prop = rel_expr.derive_relational_prop_child(0)?; - - let output_columns = left_prop.output_columns.clone(); - let outer_columns = left_prop.outer_columns.clone(); - let used_columns = left_prop.used_columns.clone(); - let orderings = left_prop.orderings.clone(); - - Ok(Arc::new(RelationalProperty { - output_columns, - outer_columns, - used_columns, - orderings, - partition_orderings: None, - })) - } - - fn derive_physical_prop(&self, _rel_expr: &RelExpr) -> Result { - Ok(PhysicalProperty { - distribution: Distribution::Serial, - }) - } - - fn derive_stats(&self, rel_expr: &RelExpr) -> Result> { - let right_stat_info = rel_expr.derive_cardinality_child(1)?; - Ok(Arc::new(StatInfo { - cardinality: right_stat_info.cardinality, - statistics: right_stat_info.statistics.clone(), - })) - } - - fn compute_required_prop_child( - &self, - _ctx: Arc, - _rel_expr: &RelExpr, - _child_index: usize, - _required: &RequiredProperty, - ) -> Result { - // Todo(xudong): consider cluster for materialized cte - Ok(RequiredProperty { - distribution: Distribution::Serial, - }) - } - - fn compute_required_prop_children( - &self, - _ctx: Arc, - _rel_expr: &RelExpr, - _required: &RequiredProperty, - ) -> Result>> { - Ok(vec![vec![ - RequiredProperty { - distribution: Distribution::Serial, - }, - RequiredProperty { - distribution: Distribution::Serial, - }, - ]]) - } -} - -impl std::hash::Hash for MaterializedCte { - fn hash(&self, state: &mut H) { - self.cte_idx.hash(state); - self.materialized_output_columns.hash(state); - } -} diff --git a/src/query/sql/src/planner/plans/mod.rs b/src/query/sql/src/planner/plans/mod.rs index 2acc83d081d7..e1876f090ba8 100644 --- a/src/query/sql/src/planner/plans/mod.rs +++ b/src/query/sql/src/planner/plans/mod.rs @@ -19,7 +19,6 @@ mod call; mod constant_table_scan; mod copy_into_location; mod copy_into_table; -mod cte_scan; mod data_mask; mod ddl; mod dummy_table_scan; @@ -32,7 +31,6 @@ mod insert_multi_table; mod join; mod kill; mod limit; -mod materialized_cte; mod mutation; mod mutation_source; mod operator; @@ -61,7 +59,6 @@ pub use call::CallPlan; pub use constant_table_scan::ConstantTableScan; pub use copy_into_location::*; pub use copy_into_table::*; -pub use cte_scan::CteScan; pub use data_mask::*; pub use ddl::*; pub use dummy_table_scan::DummyTableScan; @@ -74,7 +71,6 @@ pub use insert_multi_table::*; pub use join::*; pub use kill::KillPlan; pub use limit::*; -pub use materialized_cte::MaterializedCte; pub use mutation::MatchedEvaluator; pub use mutation::Mutation; pub use mutation::UnmatchedEvaluator; diff --git a/src/query/sql/src/planner/plans/operator.rs b/src/query/sql/src/planner/plans/operator.rs index d7f6b20c5be5..2836f89bfd9b 100644 --- a/src/query/sql/src/planner/plans/operator.rs +++ b/src/query/sql/src/planner/plans/operator.rs @@ -24,13 +24,11 @@ use crate::optimizer::RelExpr; use crate::optimizer::RelationalProperty; use crate::optimizer::RequiredProperty; use crate::optimizer::StatInfo; -use crate::plans::materialized_cte::MaterializedCte; use crate::plans::r_cte_scan::RecursiveCteScan; use crate::plans::Aggregate; use crate::plans::AsyncFunction; use crate::plans::CacheScan; use crate::plans::ConstantTableScan; -use crate::plans::CteScan; use crate::plans::DummyTableScan; use crate::plans::EvalScalar; use crate::plans::Exchange; @@ -98,7 +96,6 @@ pub trait Operator { #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub enum RelOp { Scan, - CteScan, Join, EvalScalar, Filter, @@ -110,7 +107,6 @@ pub enum RelOp { DummyTableScan, Window, ProjectSet, - MaterializedCte, ConstantTableScan, ExpressionScan, CacheScan, @@ -130,7 +126,6 @@ pub enum RelOp { #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub enum RelOperator { Scan(Scan), - CteScan(CteScan), Join(Join), EvalScalar(EvalScalar), Filter(Filter), @@ -142,7 +137,6 @@ pub enum RelOperator { DummyTableScan(DummyTableScan), Window(Window), ProjectSet(ProjectSet), - MaterializedCte(MaterializedCte), ConstantTableScan(ConstantTableScan), ExpressionScan(ExpressionScan), CacheScan(CacheScan), @@ -170,8 +164,6 @@ impl Operator for RelOperator { RelOperator::DummyTableScan(rel_op) => rel_op.rel_op(), RelOperator::ProjectSet(rel_op) => rel_op.rel_op(), RelOperator::Window(rel_op) => rel_op.rel_op(), - RelOperator::CteScan(rel_op) => rel_op.rel_op(), - RelOperator::MaterializedCte(rel_op) => rel_op.rel_op(), RelOperator::ConstantTableScan(rel_op) => rel_op.rel_op(), RelOperator::ExpressionScan(rel_op) => rel_op.rel_op(), RelOperator::CacheScan(rel_op) => rel_op.rel_op(), @@ -188,7 +180,6 @@ impl Operator for RelOperator { fn arity(&self) -> usize { match self { RelOperator::Scan(rel_op) => rel_op.arity(), - RelOperator::CteScan(rel_op) => rel_op.arity(), RelOperator::Join(rel_op) => rel_op.arity(), RelOperator::EvalScalar(rel_op) => rel_op.arity(), RelOperator::Filter(rel_op) => rel_op.arity(), @@ -200,7 +191,6 @@ impl Operator for RelOperator { RelOperator::DummyTableScan(rel_op) => rel_op.arity(), RelOperator::Window(rel_op) => rel_op.arity(), RelOperator::ProjectSet(rel_op) => rel_op.arity(), - RelOperator::MaterializedCte(rel_op) => rel_op.arity(), RelOperator::ConstantTableScan(rel_op) => rel_op.arity(), RelOperator::ExpressionScan(rel_op) => rel_op.arity(), RelOperator::CacheScan(rel_op) => rel_op.arity(), @@ -228,8 +218,6 @@ impl Operator for RelOperator { RelOperator::DummyTableScan(rel_op) => rel_op.derive_relational_prop(rel_expr), RelOperator::ProjectSet(rel_op) => rel_op.derive_relational_prop(rel_expr), RelOperator::Window(rel_op) => rel_op.derive_relational_prop(rel_expr), - RelOperator::CteScan(rel_op) => rel_op.derive_relational_prop(rel_expr), - RelOperator::MaterializedCte(rel_op) => rel_op.derive_relational_prop(rel_expr), RelOperator::ConstantTableScan(rel_op) => rel_op.derive_relational_prop(rel_expr), RelOperator::ExpressionScan(rel_op) => rel_op.derive_relational_prop(rel_expr), RelOperator::CacheScan(rel_op) => rel_op.derive_relational_prop(rel_expr), @@ -257,8 +245,6 @@ impl Operator for RelOperator { RelOperator::DummyTableScan(rel_op) => rel_op.derive_physical_prop(rel_expr), RelOperator::ProjectSet(rel_op) => rel_op.derive_physical_prop(rel_expr), RelOperator::Window(rel_op) => rel_op.derive_physical_prop(rel_expr), - RelOperator::CteScan(rel_op) => rel_op.derive_physical_prop(rel_expr), - RelOperator::MaterializedCte(rel_op) => rel_op.derive_physical_prop(rel_expr), RelOperator::ConstantTableScan(rel_op) => rel_op.derive_physical_prop(rel_expr), RelOperator::ExpressionScan(rel_op) => rel_op.derive_physical_prop(rel_expr), RelOperator::CacheScan(rel_op) => rel_op.derive_physical_prop(rel_expr), @@ -286,8 +272,6 @@ impl Operator for RelOperator { RelOperator::DummyTableScan(rel_op) => rel_op.derive_stats(rel_expr), RelOperator::ProjectSet(rel_op) => rel_op.derive_stats(rel_expr), RelOperator::Window(rel_op) => rel_op.derive_stats(rel_expr), - RelOperator::CteScan(rel_op) => rel_op.derive_stats(rel_expr), - RelOperator::MaterializedCte(rel_op) => rel_op.derive_stats(rel_expr), RelOperator::ConstantTableScan(rel_op) => rel_op.derive_stats(rel_expr), RelOperator::ExpressionScan(rel_op) => rel_op.derive_stats(rel_expr), RelOperator::CacheScan(rel_op) => rel_op.derive_stats(rel_expr), @@ -345,12 +329,6 @@ impl Operator for RelOperator { RelOperator::ProjectSet(rel_op) => { rel_op.compute_required_prop_child(ctx, rel_expr, child_index, required) } - RelOperator::CteScan(rel_op) => { - rel_op.compute_required_prop_child(ctx, rel_expr, child_index, required) - } - RelOperator::MaterializedCte(rel_op) => { - rel_op.compute_required_prop_child(ctx, rel_expr, child_index, required) - } RelOperator::ConstantTableScan(rel_op) => { rel_op.compute_required_prop_child(ctx, rel_expr, child_index, required) } @@ -394,9 +372,6 @@ impl Operator for RelOperator { RelOperator::Scan(rel_op) => { rel_op.compute_required_prop_children(ctx, rel_expr, required) } - RelOperator::CteScan(rel_op) => { - rel_op.compute_required_prop_children(ctx, rel_expr, required) - } RelOperator::Join(rel_op) => { rel_op.compute_required_prop_children(ctx, rel_expr, required) } @@ -430,9 +405,6 @@ impl Operator for RelOperator { RelOperator::ProjectSet(rel_op) => { rel_op.compute_required_prop_children(ctx, rel_expr, required) } - RelOperator::MaterializedCte(rel_op) => { - rel_op.compute_required_prop_children(ctx, rel_expr, required) - } RelOperator::ConstantTableScan(rel_op) => { rel_op.compute_required_prop_children(ctx, rel_expr, required) } @@ -488,48 +460,6 @@ impl TryFrom for Scan { } } -impl From for RelOperator { - fn from(value: CteScan) -> Self { - Self::CteScan(value) - } -} - -impl TryFrom for CteScan { - type Error = ErrorCode; - - fn try_from(value: RelOperator) -> Result { - if let RelOperator::CteScan(value) = value { - Ok(value) - } else { - Err(ErrorCode::Internal(format!( - "Cannot downcast {:?} to CteScan", - value.rel_op() - ))) - } - } -} - -impl From for RelOperator { - fn from(value: MaterializedCte) -> Self { - Self::MaterializedCte(value) - } -} - -impl TryFrom for MaterializedCte { - type Error = ErrorCode; - - fn try_from(value: RelOperator) -> Result { - if let RelOperator::MaterializedCte(value) = value { - Ok(value) - } else { - Err(ErrorCode::Internal(format!( - "Cannot downcast {:?} to MaterializedCte", - value.rel_op() - ))) - } - } -} - impl From for RelOperator { fn from(v: Join) -> Self { Self::Join(v) diff --git a/tests/sqllogictests/suites/mode/standalone/explain/push_down_filter/push_down_filter_join/push_down_filter_self_join.test b/tests/sqllogictests/suites/mode/standalone/explain/push_down_filter/push_down_filter_join/push_down_filter_self_join.test index ff198237bf8a..e422891145d6 100644 --- a/tests/sqllogictests/suites/mode/standalone/explain/push_down_filter/push_down_filter_join/push_down_filter_self_join.test +++ b/tests/sqllogictests/suites/mode/standalone/explain/push_down_filter/push_down_filter_join/push_down_filter_self_join.test @@ -11,41 +11,28 @@ C as (select * from B as b1 left outer join B as b2 on b1.a = b2.a where b1.b < D as (select * from C) select * from D; ---- -MaterializedCTE -├── output columns: [a (#4), b (#5), b (#7), a (#6)] -├── UnionAll -│ ├── output columns: [t1.a (#0), t1.b (#1)] -│ ├── estimated rows: 20.00 -│ ├── TableScan -│ │ ├── table: default.default.t1 -│ │ ├── output columns: [a (#0), b (#1)] -│ │ ├── read rows: 10 -│ │ ├── read size: < 1 KiB -│ │ ├── partitions total: 1 -│ │ ├── partitions scanned: 1 -│ │ ├── pruning stats: [segments: , blocks: ] -│ │ ├── push downs: [filters: [], limit: NONE] -│ │ └── estimated rows: 10.00 -│ └── TableScan -│ ├── table: default.default.t2 -│ ├── output columns: [a (#2), b (#3)] -│ ├── read rows: 10 -│ ├── read size: < 1 KiB -│ ├── partitions total: 1 -│ ├── partitions scanned: 1 -│ ├── pruning stats: [segments: , blocks: ] -│ ├── push downs: [filters: [], limit: NONE] -│ └── estimated rows: 10.00 -└── HashJoin - ├── output columns: [a (#4), b (#5), b (#7), a (#6)] - ├── join type: INNER - ├── build keys: [b2.a (#6)] - ├── probe keys: [b1.a (#4)] - ├── filters: [b1.b (#5) < b2.b (#7)] - ├── estimated rows: 400.00 - ├── CTEScan(Build) - │ ├── CTE index: 0, sub index: 2 - │ └── estimated rows: 20.00 - └── CTEScan(Probe) - ├── CTE index: 0, sub index: 1 - └── estimated rows: 20.00 +HashJoin +├── output columns: [a.a (#0), a.b (#1), a.b (#3), a.a (#2)] +├── join type: INNER +├── build keys: [b2.a (#2)] +├── probe keys: [b1.a (#0)] +├── filters: [d.b (#1) < d.b (#3)] +├── estimated rows: 0.00 +├── TableScan(Build) +│ ├── table: default.default.a +│ ├── output columns: [a (#2), b (#3)] +│ ├── read rows: 20 +│ ├── read size: < 1 KiB +│ ├── partitions total: 0 +│ ├── partitions scanned: 0 +│ ├── push downs: [filters: [], limit: NONE] +│ └── estimated rows: 0.00 +└── TableScan(Probe) + ├── table: default.default.a + ├── output columns: [a (#0), b (#1)] + ├── read rows: 20 + ├── read size: < 1 KiB + ├── partitions total: 0 + ├── partitions scanned: 0 + ├── push downs: [filters: [], limit: NONE] + └── estimated rows: 0.00 diff --git a/tests/sqllogictests/suites/tpch/join_order.test b/tests/sqllogictests/suites/tpch/join_order.test index 260dc8daf55b..493f0d3188a8 100644 --- a/tests/sqllogictests/suites/tpch/join_order.test +++ b/tests/sqllogictests/suites/tpch/join_order.test @@ -721,23 +721,15 @@ where order by s_suppkey; ---- -MaterializedCte: 0 -├── Right -│ └── Scan: default.tpch_test.lineitem (#1) (read rows: 600572) -└── Left - └── HashJoin: INNER - ├── Build - │ └── CTEScan - │ ├── CTE index: 0, sub index: 2 - │ └── estimated rows: 85.00 - └── Probe - └── HashJoin: INNER - ├── Build - │ └── CTEScan - │ ├── CTE index: 0, sub index: 1 - │ └── estimated rows: 85.00 - └── Probe - └── Scan: default.tpch_test.supplier (#0) (read rows: 1000) +HashJoin: INNER +├── Build +│ └── HashJoin: INNER +│ ├── Build +│ │ └── Scan: default.tpch_test.revenue (#1) (read rows: 1000) +│ └── Probe +│ └── Scan: default.tpch_test.supplier (#0) (read rows: 1000) +└── Probe + └── Scan: default.tpch_test.revenue (#2) (read rows: 1000) # Q16 query I From 76bc3e00f490a43f1e433a1ea26e48ddd4153b73 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Mon, 25 Nov 2024 08:41:44 +0800 Subject: [PATCH 03/18] specify database name --- src/query/sql/src/planner/binder/bind_query/bind.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/query/sql/src/planner/binder/bind_query/bind.rs b/src/query/sql/src/planner/binder/bind_query/bind.rs index a903a9d8c810..5b8cf9f82580 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind.rs @@ -18,11 +18,13 @@ use databend_common_ast::ast::CreateOption; use databend_common_ast::ast::CreateTableStmt; use databend_common_ast::ast::Engine; use databend_common_ast::ast::Expr; +use databend_common_ast::ast::Identifier; use databend_common_ast::ast::Query; use databend_common_ast::ast::SetExpr; use databend_common_ast::ast::TableType; use databend_common_ast::ast::With; use databend_common_ast::ast::CTE; +use databend_common_ast::Span; use databend_common_exception::ErrorCode; use databend_common_exception::Result; @@ -181,11 +183,11 @@ impl Binder { } else { Engine::Memory }; - + let database = self.ctx.get_current_database(); let create_table_stmt = CreateTableStmt { create_option: CreateOption::CreateOrReplace, catalog: None, - database: None, + database: Some(Identifier::from_name(Span::None, database)), table: cte.alias.name.clone(), source: None, engine: Some(engine), From 1c916843f9e4a02c2be5ff4a0e062cccb6b2c322 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Mon, 25 Nov 2024 09:29:24 +0800 Subject: [PATCH 04/18] find reason and add todo --- src/query/sql/src/planner/binder/bind_query/bind.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/query/sql/src/planner/binder/bind_query/bind.rs b/src/query/sql/src/planner/binder/bind_query/bind.rs index 5b8cf9f82580..62ac440ffb9a 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind.rs @@ -208,6 +208,9 @@ impl Binder { } else { return Err(ErrorCode::Internal("Binder's Subquery executor is not set")); }; + + // Todo: clear the table with the same name in table cache + Ok(()) } } From 82c89c440476385439a0509c59520242b533ae48 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Mon, 25 Nov 2024 19:26:32 +0800 Subject: [PATCH 05/18] clear cache --- src/query/catalog/src/table_context.rs | 2 ++ src/query/service/src/sessions/query_ctx.rs | 5 +++++ src/query/service/src/sessions/query_ctx_shared.rs | 10 ++++++++++ .../service/tests/it/sql/exec/get_table_bind_test.rs | 4 ++++ .../tests/it/storages/fuse/operations/commit.rs | 4 ++++ src/query/sql/src/planner/binder/bind_query/bind.rs | 8 +++++--- 6 files changed, 30 insertions(+), 3 deletions(-) diff --git a/src/query/catalog/src/table_context.rs b/src/query/catalog/src/table_context.rs index 0b6ed28fbc70..5b728ce1d999 100644 --- a/src/query/catalog/src/table_context.rs +++ b/src/query/catalog/src/table_context.rs @@ -261,6 +261,8 @@ pub trait TableContext: Send + Sync { async fn get_table(&self, catalog: &str, database: &str, table: &str) -> Result>; + fn remove_table_from_cache(&self, catalog: &str, database: &str, table: &str); + async fn get_table_with_batch( &self, catalog: &str, diff --git a/src/query/service/src/sessions/query_ctx.rs b/src/query/service/src/sessions/query_ctx.rs index 3ce989de87bd..e745c82357b5 100644 --- a/src/query/service/src/sessions/query_ctx.rs +++ b/src/query/service/src/sessions/query_ctx.rs @@ -965,6 +965,11 @@ impl TableContext for QueryContext { .await } + fn remove_table_from_cache(&self, catalog: &str, database: &str, table: &str) { + self.shared + .remove_table_from_cache(catalog, database, table) + } + #[async_backtrace::framed] async fn get_table_with_batch( &self, diff --git a/src/query/service/src/sessions/query_ctx_shared.rs b/src/query/service/src/sessions/query_ctx_shared.rs index 7864f495d39f..a5ab6c0207e1 100644 --- a/src/query/service/src/sessions/query_ctx_shared.rs +++ b/src/query/service/src/sessions/query_ctx_shared.rs @@ -357,6 +357,16 @@ impl QueryContextShared { Ok(res) } + pub fn remove_table_from_cache(&self, catalog_name: &str, database: &str, table: &str) { + let table_meta_key = ( + catalog_name.to_string(), + database.to_string(), + table.to_string(), + ); + let mut tables_refs = self.tables_refs.lock(); + tables_refs.remove(&table_meta_key); + } + #[async_backtrace::framed] async fn get_table_to_cache( &self, diff --git a/src/query/service/tests/it/sql/exec/get_table_bind_test.rs b/src/query/service/tests/it/sql/exec/get_table_bind_test.rs index 89ff1fc3aa23..d07e1e81b0d9 100644 --- a/src/query/service/tests/it/sql/exec/get_table_bind_test.rs +++ b/src/query/service/tests/it/sql/exec/get_table_bind_test.rs @@ -818,6 +818,10 @@ impl TableContext for CtxDelegation { } } + fn remove_table_from_cache(&self, _catalog: &str, _database: &str, _table: &str) { + todo!() + } + async fn get_table_with_batch( &self, catalog: &str, diff --git a/src/query/service/tests/it/storages/fuse/operations/commit.rs b/src/query/service/tests/it/storages/fuse/operations/commit.rs index 00d6ddf6bd8f..0840ac188ea7 100644 --- a/src/query/service/tests/it/storages/fuse/operations/commit.rs +++ b/src/query/service/tests/it/storages/fuse/operations/commit.rs @@ -697,6 +697,10 @@ impl TableContext for CtxDelegation { todo!() } + fn remove_table_from_cache(&self, _catalog: &str, _database: &str, _table: &str) { + todo!() + } + async fn get_table_with_batch( &self, _catalog: &str, diff --git a/src/query/sql/src/planner/binder/bind_query/bind.rs b/src/query/sql/src/planner/binder/bind_query/bind.rs index 62ac440ffb9a..b4f7c45ef4a0 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind.rs @@ -184,10 +184,11 @@ impl Binder { Engine::Memory }; let database = self.ctx.get_current_database(); + let catalog = self.ctx.get_current_catalog(); let create_table_stmt = CreateTableStmt { create_option: CreateOption::CreateOrReplace, - catalog: None, - database: Some(Identifier::from_name(Span::None, database)), + catalog: Some(Identifier::from_name(Span::None, catalog.clone())), + database: Some(Identifier::from_name(Span::None, database.clone())), table: cte.alias.name.clone(), source: None, engine: Some(engine), @@ -210,7 +211,8 @@ impl Binder { }; // Todo: clear the table with the same name in table cache - + self.ctx + .remove_table_from_cache(&catalog, &database, &cte.alias.name.name); Ok(()) } } From 95ff69d2eb6e21556f35c5d85eb671794c9d9c65 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Tue, 26 Nov 2024 12:51:53 +0800 Subject: [PATCH 06/18] fix stream --- .../storages/fuse/src/operations/changes.rs | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/query/storages/fuse/src/operations/changes.rs b/src/query/storages/fuse/src/operations/changes.rs index ed414cdb7f61..e98c07b94d2d 100644 --- a/src/query/storages/fuse/src/operations/changes.rs +++ b/src/query/storages/fuse/src/operations/changes.rs @@ -164,44 +164,44 @@ impl FuseTable { .join(", "); format!( - "with _change({a_cols}, change$action, change$row_id, \ - {d_cols}, d_change$action, d_change$row_id) as materialized \ + "with _change as materialized \ ( \ select * \ from ( \ select *, \ _row_version, \ - 'INSERT' as change$action, \ + 'INSERT' as a_change$action, \ if(is_not_null(_origin_block_id), \ concat(to_uuid(_origin_block_id), lpad(hex(_origin_block_row_num), 6, '0')), \ {a_table_alias}._base_row_id \ - ) as change$row_id \ + ) as a_change$row_id \ from {table_desc} as {a_table_alias} \ ) as A \ FULL OUTER JOIN ( \ - select *, \ + select a as d_a, \ + b as d_b, \ _row_version, \ - 'DELETE' as change$action, \ + 'DELETE' as d_change$action, \ if(is_not_null(_origin_block_id), \ concat(to_uuid(_origin_block_id), lpad(hex(_origin_block_row_num), 6, '0')), \ {d_table_alias}._base_row_id \ - ) as change$row_id \ + ) as d_change$row_id \ from {table_desc} as {d_table_alias} \ ) as D \ - on A.change$row_id = D.change$row_id \ - where A.change$row_id is null or D.change$row_id is null or A._row_version > D._row_version \ + on A.a_change$row_id = D.d_change$row_id \ + where A.a_change$row_id is null or D.d_change$row_id is null or A._row_version > D._row_version \ ) \ select {a_cols}, \ - change$action, \ - change$row_id, \ + a_change$action as change$action, \ + a_change$row_id as change$row_id, \ d_change$action is not null as change$is_update \ from _change \ - where change$action is not null \ + where a_change$action is not null \ union all \ select {d_cols}, \ d_change$action, \ d_change$row_id, \ - change$action is not null as change$is_update \ + a_change$action is not null as change$is_update \ from _change \ where d_change$action is not null", ) From e4518d9156768c294205fa51b70276b6bdd22cc5 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Wed, 27 Nov 2024 11:16:34 +0800 Subject: [PATCH 07/18] fix with consume --- src/query/catalog/src/query_kind.rs | 1 + src/query/sql/src/planner/plans/plan.rs | 1 + 2 files changed, 2 insertions(+) diff --git a/src/query/catalog/src/query_kind.rs b/src/query/catalog/src/query_kind.rs index 9e8744a29850..d9efe35306ac 100644 --- a/src/query/catalog/src/query_kind.rs +++ b/src/query/catalog/src/query_kind.rs @@ -25,6 +25,7 @@ pub enum QueryKind { Explain, CopyIntoTable, CopyIntoLocation, + CreateTable, Update, Insert, Other, diff --git a/src/query/sql/src/planner/plans/plan.rs b/src/query/sql/src/planner/plans/plan.rs index e886fefba18f..d7f768eec72d 100644 --- a/src/query/sql/src/planner/plans/plan.rs +++ b/src/query/sql/src/planner/plans/plan.rs @@ -434,6 +434,7 @@ impl Plan { | Plan::OptimizePurge(_) | Plan::OptimizeCompactSegment(_) | Plan::OptimizeCompactBlock { .. } => QueryKind::Update, + Plan::CreateTable(_) => QueryKind::CreateTable, _ => QueryKind::Other, } } From 8b813acdfb57081c4323cc101699eacc97b23e56 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Wed, 27 Nov 2024 14:54:32 +0800 Subject: [PATCH 08/18] temp table uses fuse engine if cluster mode --- src/query/sql/src/planner/binder/bind_query/bind.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/query/sql/src/planner/binder/bind_query/bind.rs b/src/query/sql/src/planner/binder/bind_query/bind.rs index b4f7c45ef4a0..a010dd097ccd 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind.rs @@ -178,7 +178,9 @@ impl Binder { } fn m_cte_to_temp_table(&self, cte: &CTE) -> Result<()> { - let engine = if self.ctx.get_settings().get_persist_materialized_cte()? { + let engine = if self.ctx.get_settings().get_persist_materialized_cte()? + || !self.ctx.get_cluster().is_empty() + { Engine::Fuse } else { Engine::Memory From 80e3d4ae149478a912c1301f052885a6d1c9de1a Mon Sep 17 00:00:00 2001 From: xudong963 Date: Wed, 27 Nov 2024 22:43:48 +0800 Subject: [PATCH 09/18] fix --- .../binder/bind_table_reference/bind_table.rs | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs index 3d5ea59fbd0b..93bb3bed0fc0 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs @@ -86,11 +86,7 @@ impl Binder { )) .set_span(*span)); } - return if cte_info.materialized - && self.ctx.get_settings().get_enable_materialized_cte()? - { - self.bind_m_cte(bind_context, cte_info, &table_name, alias, span) - } else if cte_info.recursive { + return if cte_info.recursive { if self.bind_recursive_cte { self.bind_r_cte_scan(bind_context, cte_info, &table_name, alias) } else { @@ -123,13 +119,13 @@ impl Binder { let bind_context = parent.unwrap().as_mut(); let cte_map = bind_context.cte_context.cte_map.clone(); if let Some(cte_info) = cte_map.get(&table_name) { - return if cte_info.materialized - && self.ctx.get_settings().get_enable_materialized_cte()? - { - self.bind_m_cte(bind_context, cte_info, &table_name, alias, span) - } else { - self.bind_cte(*span, bind_context, &table_name, alias, cte_info) - }; + return self.bind_cte( + *span, + bind_context, + &table_name, + alias, + cte_info, + ); } parent = bind_context.parent.as_mut(); } From 2d175529e642b5fb569318ea97dabafb4cd244c4 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Thu, 28 Nov 2024 15:41:44 +0800 Subject: [PATCH 10/18] update test --- .../mode/standalone/explain/explain_cte.test | 36 ------------------- .../sqllogictests/suites/tpch/join_order.test | 2 +- 2 files changed, 1 insertion(+), 37 deletions(-) delete mode 100644 tests/sqllogictests/suites/mode/standalone/explain/explain_cte.test diff --git a/tests/sqllogictests/suites/mode/standalone/explain/explain_cte.test b/tests/sqllogictests/suites/mode/standalone/explain/explain_cte.test deleted file mode 100644 index 6a85eb0b5331..000000000000 --- a/tests/sqllogictests/suites/mode/standalone/explain/explain_cte.test +++ /dev/null @@ -1,36 +0,0 @@ -query T -EXPLAIN WITH t AS MATERIALIZED (SELECT * FROM numbers(5)) SELECT * FROM t; ----- -MaterializedCTE -├── output columns: [number (#1)] -├── TableScan -│ ├── table: default.system.numbers -│ ├── output columns: [number (#0)] -│ ├── read rows: 5 -│ ├── read size: < 1 KiB -│ ├── partitions total: 1 -│ ├── partitions scanned: 1 -│ ├── push downs: [filters: [], limit: NONE] -│ └── estimated rows: 5.00 -└── CTEScan - ├── CTE index: 0, sub index: 1 - └── estimated rows: 5.00 - -statement ok -set enable_materialized_cte = 0; - -query T -EXPLAIN WITH t AS MATERIALIZED (SELECT * FROM numbers(5)) SELECT * FROM t; ----- -TableScan -├── table: default.system.numbers -├── output columns: [number (#0)] -├── read rows: 5 -├── read size: < 1 KiB -├── partitions total: 1 -├── partitions scanned: 1 -├── push downs: [filters: [], limit: NONE] -└── estimated rows: 5.00 - -statement ok -set enable_materialized_cte = 1; diff --git a/tests/sqllogictests/suites/tpch/join_order.test b/tests/sqllogictests/suites/tpch/join_order.test index 493f0d3188a8..084b9a096660 100644 --- a/tests/sqllogictests/suites/tpch/join_order.test +++ b/tests/sqllogictests/suites/tpch/join_order.test @@ -689,7 +689,7 @@ HashJoin: INNER └── Scan: default.tpch_test.supplier (#0) (read rows: 1000) # Q15 -query T +statement ok explain join with revenue as materialized ( select l_suppkey as supplier_no, From 6aed7c87f43fdd671322fceba8563e3ead53b4b8 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Thu, 28 Nov 2024 19:02:00 +0800 Subject: [PATCH 11/18] remove materialized keyword for stream --- src/query/storages/fuse/src/operations/changes.rs | 2 +- tests/sqllogictests/suites/tpch/join_order.test | 12 ++---------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/src/query/storages/fuse/src/operations/changes.rs b/src/query/storages/fuse/src/operations/changes.rs index e98c07b94d2d..f2279881602f 100644 --- a/src/query/storages/fuse/src/operations/changes.rs +++ b/src/query/storages/fuse/src/operations/changes.rs @@ -164,7 +164,7 @@ impl FuseTable { .join(", "); format!( - "with _change as materialized \ + "with _change as \ ( \ select * \ from ( \ diff --git a/tests/sqllogictests/suites/tpch/join_order.test b/tests/sqllogictests/suites/tpch/join_order.test index 084b9a096660..c4cfe86159f3 100644 --- a/tests/sqllogictests/suites/tpch/join_order.test +++ b/tests/sqllogictests/suites/tpch/join_order.test @@ -689,6 +689,7 @@ HashJoin: INNER └── Scan: default.tpch_test.supplier (#0) (read rows: 1000) # Q15 +onlyif todo statement ok explain join with revenue as materialized ( select @@ -720,16 +721,7 @@ where ) order by s_suppkey; ----- -HashJoin: INNER -├── Build -│ └── HashJoin: INNER -│ ├── Build -│ │ └── Scan: default.tpch_test.revenue (#1) (read rows: 1000) -│ └── Probe -│ └── Scan: default.tpch_test.supplier (#0) (read rows: 1000) -└── Probe - └── Scan: default.tpch_test.revenue (#2) (read rows: 1000) + # Q16 query I From c5184f7621dd96e272bfa177f716f67ddc6bab81 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Thu, 28 Nov 2024 23:59:57 +0800 Subject: [PATCH 12/18] make temp table name unique --- .../sql/src/planner/binder/bind_context.rs | 1 + .../sql/src/planner/binder/bind_query/bind.rs | 35 ++++++++++++------- .../binder/bind_table_reference/bind_table.rs | 22 +++++++++++- .../bind_table_function.rs | 2 ++ src/query/sql/src/planner/binder/table.rs | 1 + src/query/sql/src/planner/dataframe.rs | 1 + .../sql/src/planner/expression_parser.rs | 1 + src/query/sql/src/planner/metadata.rs | 7 +++- 8 files changed, 56 insertions(+), 14 deletions(-) diff --git a/src/query/sql/src/planner/binder/bind_context.rs b/src/query/sql/src/planner/binder/bind_context.rs index b52c3a733aa8..93dd83d68d06 100644 --- a/src/query/sql/src/planner/binder/bind_context.rs +++ b/src/query/sql/src/planner/binder/bind_context.rs @@ -212,6 +212,7 @@ pub struct CteInfo { pub cte_idx: IndexType, // If cte is materialized, save its columns pub columns: Vec, + pub m_cte_name_to_temp_table: HashMap, } impl BindContext { diff --git a/src/query/sql/src/planner/binder/bind_query/bind.rs b/src/query/sql/src/planner/binder/bind_query/bind.rs index a010dd097ccd..a13726f90fa2 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::HashMap; use std::sync::Arc; use databend_common_ast::ast::CreateOption; @@ -88,22 +89,26 @@ impl Binder { .iter() .map(|ident| self.normalize_identifier(ident).name) .collect(); - let cte_info = CteInfo { + let mut cte_info = CteInfo { columns_alias: column_name, query: *cte.query.clone(), recursive: with.recursive, cte_idx: idx, columns: vec![], materialized: cte.materialized, + m_cte_name_to_temp_table: HashMap::new(), }; + // If the CTE is materialized, we'll construct a temp table for it. + if cte.materialized { + let temp_table_name = self.m_cte_to_temp_table(cte)?; + cte_info + .m_cte_name_to_temp_table + .insert(cte.alias.name.name.clone(), temp_table_name); + } bind_context .cte_context .cte_map .insert(table_name, cte_info); - // If the CTE is materialized, we'll construct a temp table for it. - if cte.materialized { - self.m_cte_to_temp_table(cte)?; - } } Ok(()) @@ -177,21 +182,28 @@ impl Binder { )) } - fn m_cte_to_temp_table(&self, cte: &CTE) -> Result<()> { - let engine = if self.ctx.get_settings().get_persist_materialized_cte()? - || !self.ctx.get_cluster().is_empty() - { + // The return value is temp_table name` + fn m_cte_to_temp_table(&self, cte: &CTE) -> Result { + let engine = if self.ctx.get_settings().get_persist_materialized_cte()? { Engine::Fuse } else { Engine::Memory }; let database = self.ctx.get_current_database(); let catalog = self.ctx.get_current_catalog(); + let query_id = self.ctx.get_id(); + // Navigate the temp table for cte to `cte_name + query_id` + // Avoid the conflict of the temp table name with the same name of user's temp table. + let table_name = format!( + "{}_{}", + cte.alias.name.name, + query_id.split('-').next().unwrap_or(&query_id) + ); let create_table_stmt = CreateTableStmt { create_option: CreateOption::CreateOrReplace, catalog: Some(Identifier::from_name(Span::None, catalog.clone())), database: Some(Identifier::from_name(Span::None, database.clone())), - table: cte.alias.name.clone(), + table: Identifier::from_name(cte.alias.name.span, table_name.clone()), source: None, engine: Some(engine), uri_location: None, @@ -212,9 +224,8 @@ impl Binder { return Err(ErrorCode::Internal("Binder's Subquery executor is not set")); }; - // Todo: clear the table with the same name in table cache self.ctx .remove_table_from_cache(&catalog, &database, &cte.alias.name.name); - Ok(()) + Ok(table_name) } } diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs index 93bb3bed0fc0..574a4d081b47 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs @@ -97,6 +97,19 @@ impl Binder { }; } + let mut temp_table_name = None; + if let Some(cte_info) = cte_map.get(&table_name) + && cte_info.materialized + { + temp_table_name = Some( + cte_info + .m_cte_name_to_temp_table + .get(&table_name) + .unwrap() + .to_string(), + ); + } + let navigation = self.resolve_temporal_clause(bind_context, temporal)?; // Resolve table with catalog @@ -104,7 +117,11 @@ impl Binder { match self.resolve_data_source( catalog.as_str(), database.as_str(), - table_name.as_str(), + if let Some(temp_table_name) = temp_table_name.as_ref() { + temp_table_name.as_str() + } else { + table_name.as_str() + }, navigation.as_ref(), max_batch_size, self.ctx.clone().get_abort_checker(), @@ -150,6 +167,7 @@ impl Binder { database.clone(), table_meta, table_name_alias, + None, bind_context.view_info.is_some(), bind_context.planning_agg_index, false, @@ -228,6 +246,7 @@ impl Binder { database.clone(), table_meta, table_name_alias, + None, false, false, false, @@ -259,6 +278,7 @@ impl Binder { catalog, database.clone(), table_meta, + Some(table_name.clone()), table_name_alias, bind_context.view_info.is_some(), bind_context.planning_agg_index, diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_table_function.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_table_function.rs index 30eb9e5e889a..719673be4741 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_table_function.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_table_function.rs @@ -143,6 +143,7 @@ impl Binder { "system".to_string(), table.clone(), table_alias_name, + None, false, false, false, @@ -205,6 +206,7 @@ impl Binder { "system".to_string(), table.clone(), table_alias_name, + None, false, false, false, diff --git a/src/query/sql/src/planner/binder/table.rs b/src/query/sql/src/planner/binder/table.rs index fb2443ee870e..25dd3ba8c67b 100644 --- a/src/query/sql/src/planner/binder/table.rs +++ b/src/query/sql/src/planner/binder/table.rs @@ -137,6 +137,7 @@ impl Binder { "system".to_string(), table.clone(), table_alias_name, + None, false, false, true, diff --git a/src/query/sql/src/planner/dataframe.rs b/src/query/sql/src/planner/dataframe.rs index a590b9bbad8d..49403dabc84d 100644 --- a/src/query/sql/src/planner/dataframe.rs +++ b/src/query/sql/src/planner/dataframe.rs @@ -98,6 +98,7 @@ impl Dataframe { database.to_string(), table_meta, None, + None, false, false, false, diff --git a/src/query/sql/src/planner/expression_parser.rs b/src/query/sql/src/planner/expression_parser.rs index 9e16429a6d00..7c43798b514a 100644 --- a/src/query/sql/src/planner/expression_parser.rs +++ b/src/query/sql/src/planner/expression_parser.rs @@ -68,6 +68,7 @@ pub fn bind_table(table_meta: Arc) -> Result<(BindContext, MetadataRe "default".to_string(), table_meta, None, + None, false, false, false, diff --git a/src/query/sql/src/planner/metadata.rs b/src/query/sql/src/planner/metadata.rs index 042766c848d5..e3f942e88c29 100644 --- a/src/query/sql/src/planner/metadata.rs +++ b/src/query/sql/src/planner/metadata.rs @@ -349,13 +349,18 @@ impl Metadata { catalog: String, database: String, table_meta: Arc, + table_name: Option, table_alias_name: Option, source_of_view: bool, source_of_index: bool, source_of_stage: bool, consume: bool, ) -> IndexType { - let table_name = table_meta.name().to_string(); + let table_name = if let Some(table_name) = table_name { + table_name + } else { + table_meta.name().to_string() + }; let table_index = self.tables.len(); // If exists table alias name, use it instead of origin name From 6fba8aa42a20cb584417a49a3c7740cdd733b184 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Tue, 3 Dec 2024 12:05:11 +0800 Subject: [PATCH 13/18] clear temp table --- src/query/catalog/src/table_context.rs | 4 ++ .../service/src/interpreters/interpreter.rs | 9 +++- src/query/service/src/sessions/query_ctx.rs | 44 +++++++++++++++++++ .../tests/it/sql/exec/get_table_bind_test.rs | 8 ++++ .../it/storages/fuse/operations/commit.rs | 8 ++++ .../sql/src/planner/binder/bind_context.rs | 1 - .../sql/src/planner/binder/bind_query/bind.rs | 41 +++++++++-------- .../binder/bind_table_reference/bind_table.rs | 22 +--------- .../bind_table_function.rs | 2 - src/query/sql/src/planner/binder/table.rs | 1 - src/query/sql/src/planner/dataframe.rs | 1 - .../sql/src/planner/expression_parser.rs | 1 - src/query/sql/src/planner/metadata.rs | 7 +-- 13 files changed, 93 insertions(+), 56 deletions(-) diff --git a/src/query/catalog/src/table_context.rs b/src/query/catalog/src/table_context.rs index 5b728ce1d999..3deaa3918d2f 100644 --- a/src/query/catalog/src/table_context.rs +++ b/src/query/catalog/src/table_context.rs @@ -375,6 +375,10 @@ pub trait TableContext: Send + Sync { fn get_shared_settings(&self) -> Arc; fn get_runtime(&self) -> Result>; + + fn add_m_cte_temp_table(&self, database_name: &str, table_name: &str); + + async fn drop_m_cte_temp_table(&self) -> Result<()>; } pub type AbortChecker = Arc; diff --git a/src/query/service/src/interpreters/interpreter.rs b/src/query/service/src/interpreters/interpreter.rs index 861d9d754f3d..30a80033f976 100644 --- a/src/query/service/src/interpreters/interpreter.rs +++ b/src/query/service/src/interpreters/interpreter.rs @@ -85,7 +85,10 @@ pub trait Interpreter: Sync + Send { #[fastrace::trace] async fn execute(&self, ctx: Arc) -> Result { log_query_start(&ctx); - match self.execute_inner(ctx.clone()).await { + let res = self.execute_inner(ctx.clone()).await; + // Drop the temp tables that are generated by materialized cte. + ctx.drop_m_cte_temp_table().await?; + match res { Ok(stream) => Ok(stream), Err(err) => { log_query_finished(&ctx, Some(err.clone()), false); @@ -230,7 +233,9 @@ async fn plan_sql( ) -> Result<(Plan, PlanExtras, AcquireQueueGuard)> { let mut planner = Planner::new_with_query_executor( ctx.clone(), - Arc::new(ServiceQueryExecutor::new(ctx.clone())), + Arc::new(ServiceQueryExecutor::new(QueryContext::create_from( + ctx.clone(), + ))), ); // Parse the SQL query, get extract additional information. diff --git a/src/query/service/src/sessions/query_ctx.rs b/src/query/service/src/sessions/query_ctx.rs index e745c82357b5..751975750431 100644 --- a/src/query/service/src/sessions/query_ctx.rs +++ b/src/query/service/src/sessions/query_ctx.rs @@ -79,6 +79,7 @@ use databend_common_meta_app::principal::UserPrivilegeType; use databend_common_meta_app::principal::COPY_MAX_FILES_COMMIT_MSG; use databend_common_meta_app::principal::COPY_MAX_FILES_PER_COMMIT; use databend_common_meta_app::schema::CatalogType; +use databend_common_meta_app::schema::DropTableByIdReq; use databend_common_meta_app::schema::GetTableCopiedFileReq; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::storage::StorageParams; @@ -107,9 +108,11 @@ use databend_common_storages_stage::StageTable; use databend_common_storages_stream::stream_table::StreamTable; use databend_common_users::GrantObjectVisibilityChecker; use databend_common_users::UserApiProvider; +use databend_storages_common_session::drop_table_by_id; use databend_storages_common_session::SessionState; use databend_storages_common_session::TxnManagerRef; use databend_storages_common_table_meta::meta::Location; +use databend_storages_common_table_meta::table::OPT_KEY_TEMP_PREFIX; use jiff::tz::TimeZone; use jiff::Zoned; use log::debug; @@ -149,6 +152,9 @@ pub struct QueryContext { fragment_id: Arc, // Used by synchronized generate aggregating indexes when new data written. inserted_segment_locs: Arc>>, + // Temp table for materialized CTE, first string is the database_name, second string is the table_name + // All temp tables' catalog is `CATALOG_DEFAULT`, so we don't need to store it. + m_cte_temp_table: Arc>>, } impl QueryContext { @@ -174,6 +180,7 @@ impl QueryContext { fragment_id: Arc::new(AtomicUsize::new(0)), inserted_segment_locs: Arc::new(RwLock::new(HashSet::new())), block_threshold: Arc::new(RwLock::new(BlockThresholds::default())), + m_cte_temp_table: Arc::new(Default::default()), }) } @@ -1477,6 +1484,43 @@ impl TableContext for QueryContext { fn get_runtime(&self) -> Result> { self.shared.try_get_runtime() } + + fn add_m_cte_temp_table(&self, database_name: &str, table_name: &str) { + self.m_cte_temp_table + .write() + .push((database_name.to_string(), table_name.to_string())); + } + + async fn drop_m_cte_temp_table(&self) -> Result<()> { + let temp_tbl_mgr = self.shared.session.session_ctx.temp_tbl_mgr(); + let m_cte_temp_table = self.m_cte_temp_table.read().clone(); + let tenant = self.get_tenant(); + for (db_name, table_name) in m_cte_temp_table.iter() { + let table = self.get_table(CATALOG_DEFAULT, db_name, table_name).await?; + let db = self + .get_catalog(CATALOG_DEFAULT) + .await? + .get_database(&tenant, db_name) + .await?; + let drop_table_req = DropTableByIdReq { + if_exists: true, + tenant: tenant.clone(), + tb_id: table.get_table_info().ident.table_id, + table_name: table_name.to_string(), + db_id: db.get_db_info().database_id.db_id, + engine: table.engine().to_string(), + session_id: table + .options() + .get(OPT_KEY_TEMP_PREFIX) + .cloned() + .unwrap_or_default(), + }; + drop_table_by_id(temp_tbl_mgr.clone(), drop_table_req).await?; + } + let mut m_cte_temp_table = self.m_cte_temp_table.write(); + m_cte_temp_table.clear(); + Ok(()) + } } impl TrySpawn for QueryContext { diff --git a/src/query/service/tests/it/sql/exec/get_table_bind_test.rs b/src/query/service/tests/it/sql/exec/get_table_bind_test.rs index d07e1e81b0d9..3d3d2fc7e23f 100644 --- a/src/query/service/tests/it/sql/exec/get_table_bind_test.rs +++ b/src/query/service/tests/it/sql/exec/get_table_bind_test.rs @@ -1000,6 +1000,14 @@ impl TableContext for CtxDelegation { fn get_runtime(&self) -> Result> { todo!() } + + fn add_m_cte_temp_table(&self, _database_name: &str, _table_name: &str) { + todo!() + } + + async fn drop_m_cte_temp_table(&self) -> Result<()> { + todo!() + } } #[tokio::test(flavor = "multi_thread")] diff --git a/src/query/service/tests/it/storages/fuse/operations/commit.rs b/src/query/service/tests/it/storages/fuse/operations/commit.rs index 0840ac188ea7..8d494ca256f8 100644 --- a/src/query/service/tests/it/storages/fuse/operations/commit.rs +++ b/src/query/service/tests/it/storages/fuse/operations/commit.rs @@ -880,6 +880,14 @@ impl TableContext for CtxDelegation { fn get_runtime(&self) -> Result> { todo!() } + + fn add_m_cte_temp_table(&self, _database_name: &str, _table_name: &str) { + todo!() + } + + async fn drop_m_cte_temp_table(&self) -> Result<()> { + todo!() + } } #[derive(Clone, Debug)] diff --git a/src/query/sql/src/planner/binder/bind_context.rs b/src/query/sql/src/planner/binder/bind_context.rs index 93dd83d68d06..b52c3a733aa8 100644 --- a/src/query/sql/src/planner/binder/bind_context.rs +++ b/src/query/sql/src/planner/binder/bind_context.rs @@ -212,7 +212,6 @@ pub struct CteInfo { pub cte_idx: IndexType, // If cte is materialized, save its columns pub columns: Vec, - pub m_cte_name_to_temp_table: HashMap, } impl BindContext { diff --git a/src/query/sql/src/planner/binder/bind_query/bind.rs b/src/query/sql/src/planner/binder/bind_query/bind.rs index a13726f90fa2..d74b7e0aa0b1 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::HashMap; use std::sync::Arc; use databend_common_ast::ast::CreateOption; @@ -26,6 +25,7 @@ use databend_common_ast::ast::TableType; use databend_common_ast::ast::With; use databend_common_ast::ast::CTE; use databend_common_ast::Span; +use databend_common_catalog::catalog::CATALOG_DEFAULT; use databend_common_exception::ErrorCode; use databend_common_exception::Result; @@ -89,21 +89,17 @@ impl Binder { .iter() .map(|ident| self.normalize_identifier(ident).name) .collect(); - let mut cte_info = CteInfo { + let cte_info = CteInfo { columns_alias: column_name, query: *cte.query.clone(), recursive: with.recursive, cte_idx: idx, columns: vec![], materialized: cte.materialized, - m_cte_name_to_temp_table: HashMap::new(), }; // If the CTE is materialized, we'll construct a temp table for it. if cte.materialized { - let temp_table_name = self.m_cte_to_temp_table(cte)?; - cte_info - .m_cte_name_to_temp_table - .insert(cte.alias.name.name.clone(), temp_table_name); + self.m_cte_to_temp_table(cte)?; } bind_context .cte_context @@ -183,27 +179,27 @@ impl Binder { } // The return value is temp_table name` - fn m_cte_to_temp_table(&self, cte: &CTE) -> Result { + fn m_cte_to_temp_table(&self, cte: &CTE) -> Result<()> { let engine = if self.ctx.get_settings().get_persist_materialized_cte()? { Engine::Fuse } else { Engine::Memory }; let database = self.ctx.get_current_database(); - let catalog = self.ctx.get_current_catalog(); - let query_id = self.ctx.get_id(); - // Navigate the temp table for cte to `cte_name + query_id` - // Avoid the conflict of the temp table name with the same name of user's temp table. - let table_name = format!( - "{}_{}", - cte.alias.name.name, - query_id.split('-').next().unwrap_or(&query_id) - ); + if self + .ctx + .is_temp_table(CATALOG_DEFAULT, &database, &cte.alias.name.name) + { + return Err(ErrorCode::Internal(format!( + "Temporary table {:?} already exists in current session, please change the materialized CTE name", + cte.alias.name.name + ))); + } let create_table_stmt = CreateTableStmt { create_option: CreateOption::CreateOrReplace, - catalog: Some(Identifier::from_name(Span::None, catalog.clone())), + catalog: Some(Identifier::from_name(Span::None, CATALOG_DEFAULT)), database: Some(Identifier::from_name(Span::None, database.clone())), - table: Identifier::from_name(cte.alias.name.span, table_name.clone()), + table: cte.alias.name.clone(), source: None, engine: Some(engine), uri_location: None, @@ -225,7 +221,10 @@ impl Binder { }; self.ctx - .remove_table_from_cache(&catalog, &database, &cte.alias.name.name); - Ok(table_name) + .add_m_cte_temp_table(&database, &cte.alias.name.name); + + self.ctx + .remove_table_from_cache(CATALOG_DEFAULT, &database, &cte.alias.name.name); + Ok(()) } } diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs index 574a4d081b47..93bb3bed0fc0 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs @@ -97,19 +97,6 @@ impl Binder { }; } - let mut temp_table_name = None; - if let Some(cte_info) = cte_map.get(&table_name) - && cte_info.materialized - { - temp_table_name = Some( - cte_info - .m_cte_name_to_temp_table - .get(&table_name) - .unwrap() - .to_string(), - ); - } - let navigation = self.resolve_temporal_clause(bind_context, temporal)?; // Resolve table with catalog @@ -117,11 +104,7 @@ impl Binder { match self.resolve_data_source( catalog.as_str(), database.as_str(), - if let Some(temp_table_name) = temp_table_name.as_ref() { - temp_table_name.as_str() - } else { - table_name.as_str() - }, + table_name.as_str(), navigation.as_ref(), max_batch_size, self.ctx.clone().get_abort_checker(), @@ -167,7 +150,6 @@ impl Binder { database.clone(), table_meta, table_name_alias, - None, bind_context.view_info.is_some(), bind_context.planning_agg_index, false, @@ -246,7 +228,6 @@ impl Binder { database.clone(), table_meta, table_name_alias, - None, false, false, false, @@ -278,7 +259,6 @@ impl Binder { catalog, database.clone(), table_meta, - Some(table_name.clone()), table_name_alias, bind_context.view_info.is_some(), bind_context.planning_agg_index, diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_table_function.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_table_function.rs index 719673be4741..30eb9e5e889a 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_table_function.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_table_function.rs @@ -143,7 +143,6 @@ impl Binder { "system".to_string(), table.clone(), table_alias_name, - None, false, false, false, @@ -206,7 +205,6 @@ impl Binder { "system".to_string(), table.clone(), table_alias_name, - None, false, false, false, diff --git a/src/query/sql/src/planner/binder/table.rs b/src/query/sql/src/planner/binder/table.rs index 25dd3ba8c67b..fb2443ee870e 100644 --- a/src/query/sql/src/planner/binder/table.rs +++ b/src/query/sql/src/planner/binder/table.rs @@ -137,7 +137,6 @@ impl Binder { "system".to_string(), table.clone(), table_alias_name, - None, false, false, true, diff --git a/src/query/sql/src/planner/dataframe.rs b/src/query/sql/src/planner/dataframe.rs index 49403dabc84d..a590b9bbad8d 100644 --- a/src/query/sql/src/planner/dataframe.rs +++ b/src/query/sql/src/planner/dataframe.rs @@ -98,7 +98,6 @@ impl Dataframe { database.to_string(), table_meta, None, - None, false, false, false, diff --git a/src/query/sql/src/planner/expression_parser.rs b/src/query/sql/src/planner/expression_parser.rs index 7c43798b514a..9e16429a6d00 100644 --- a/src/query/sql/src/planner/expression_parser.rs +++ b/src/query/sql/src/planner/expression_parser.rs @@ -68,7 +68,6 @@ pub fn bind_table(table_meta: Arc) -> Result<(BindContext, MetadataRe "default".to_string(), table_meta, None, - None, false, false, false, diff --git a/src/query/sql/src/planner/metadata.rs b/src/query/sql/src/planner/metadata.rs index e3f942e88c29..042766c848d5 100644 --- a/src/query/sql/src/planner/metadata.rs +++ b/src/query/sql/src/planner/metadata.rs @@ -349,18 +349,13 @@ impl Metadata { catalog: String, database: String, table_meta: Arc, - table_name: Option, table_alias_name: Option, source_of_view: bool, source_of_index: bool, source_of_stage: bool, consume: bool, ) -> IndexType { - let table_name = if let Some(table_name) = table_name { - table_name - } else { - table_meta.name().to_string() - }; + let table_name = table_meta.name().to_string(); let table_index = self.tables.len(); // If exists table alias name, use it instead of origin name From cc74a7484c46a5a281be8d552093f2ba11bdeac2 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Tue, 3 Dec 2024 15:16:45 +0800 Subject: [PATCH 14/18] fix all tests --- .../sql/src/planner/binder/bind_query/bind.rs | 11 ++++---- .../storages/fuse/src/operations/changes.rs | 26 +++++++++---------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/src/query/sql/src/planner/binder/bind_query/bind.rs b/src/query/sql/src/planner/binder/bind_query/bind.rs index d74b7e0aa0b1..1f853b3957d4 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind.rs @@ -30,6 +30,7 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use crate::binder::CteInfo; +use crate::normalize_identifier; use crate::optimizer::SExpr; use crate::planner::binder::scalar::ScalarBinder; use crate::planner::binder::BindContext; @@ -186,13 +187,14 @@ impl Binder { Engine::Memory }; let database = self.ctx.get_current_database(); + let table_name = normalize_identifier(&cte.alias.name, &self.name_resolution_ctx).name; if self .ctx - .is_temp_table(CATALOG_DEFAULT, &database, &cte.alias.name.name) + .is_temp_table(CATALOG_DEFAULT, &database, &table_name) { return Err(ErrorCode::Internal(format!( "Temporary table {:?} already exists in current session, please change the materialized CTE name", - cte.alias.name.name + table_name ))); } let create_table_stmt = CreateTableStmt { @@ -220,11 +222,10 @@ impl Binder { return Err(ErrorCode::Internal("Binder's Subquery executor is not set")); }; - self.ctx - .add_m_cte_temp_table(&database, &cte.alias.name.name); + self.ctx.add_m_cte_temp_table(&database, &table_name); self.ctx - .remove_table_from_cache(CATALOG_DEFAULT, &database, &cte.alias.name.name); + .remove_table_from_cache(CATALOG_DEFAULT, &database, &table_name); Ok(()) } } diff --git a/src/query/storages/fuse/src/operations/changes.rs b/src/query/storages/fuse/src/operations/changes.rs index f2279881602f..e4c10299fc1f 100644 --- a/src/query/storages/fuse/src/operations/changes.rs +++ b/src/query/storages/fuse/src/operations/changes.rs @@ -164,44 +164,44 @@ impl FuseTable { .join(", "); format!( - "with _change as \ + "with _change({a_cols}, change$action, change$row_id, \ + {d_cols}, d_change$action, d_change$row_id) as \ ( \ select * \ from ( \ select *, \ _row_version, \ - 'INSERT' as a_change$action, \ + 'INSERT' as change$action, \ if(is_not_null(_origin_block_id), \ concat(to_uuid(_origin_block_id), lpad(hex(_origin_block_row_num), 6, '0')), \ {a_table_alias}._base_row_id \ - ) as a_change$row_id \ + ) as change$row_id \ from {table_desc} as {a_table_alias} \ ) as A \ FULL OUTER JOIN ( \ - select a as d_a, \ - b as d_b, \ + select *, \ _row_version, \ - 'DELETE' as d_change$action, \ + 'DELETE' as change$action, \ if(is_not_null(_origin_block_id), \ concat(to_uuid(_origin_block_id), lpad(hex(_origin_block_row_num), 6, '0')), \ {d_table_alias}._base_row_id \ - ) as d_change$row_id \ + ) as change$row_id \ from {table_desc} as {d_table_alias} \ ) as D \ - on A.a_change$row_id = D.d_change$row_id \ - where A.a_change$row_id is null or D.d_change$row_id is null or A._row_version > D._row_version \ + on A.change$row_id = D.change$row_id \ + where A.change$row_id is null or D.change$row_id is null or A._row_version > D._row_version \ ) \ select {a_cols}, \ - a_change$action as change$action, \ - a_change$row_id as change$row_id, \ + change$action, \ + change$row_id, \ d_change$action is not null as change$is_update \ from _change \ - where a_change$action is not null \ + where change$action is not null \ union all \ select {d_cols}, \ d_change$action, \ d_change$row_id, \ - a_change$action is not null as change$is_update \ + change$action is not null as change$is_update \ from _change \ where d_change$action is not null", ) From f42c56950571c76b594779de455b18e887410770 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Tue, 3 Dec 2024 16:41:40 +0800 Subject: [PATCH 15/18] remove useless code --- src/query/catalog/src/query_kind.rs | 1 - src/query/sql/src/planner/plans/plan.rs | 1 - tests/sqllogictests/suites/tpch/join_order.test | 13 +++++++++++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/query/catalog/src/query_kind.rs b/src/query/catalog/src/query_kind.rs index d9efe35306ac..9e8744a29850 100644 --- a/src/query/catalog/src/query_kind.rs +++ b/src/query/catalog/src/query_kind.rs @@ -25,7 +25,6 @@ pub enum QueryKind { Explain, CopyIntoTable, CopyIntoLocation, - CreateTable, Update, Insert, Other, diff --git a/src/query/sql/src/planner/plans/plan.rs b/src/query/sql/src/planner/plans/plan.rs index d7f768eec72d..e886fefba18f 100644 --- a/src/query/sql/src/planner/plans/plan.rs +++ b/src/query/sql/src/planner/plans/plan.rs @@ -434,7 +434,6 @@ impl Plan { | Plan::OptimizePurge(_) | Plan::OptimizeCompactSegment(_) | Plan::OptimizeCompactBlock { .. } => QueryKind::Update, - Plan::CreateTable(_) => QueryKind::CreateTable, _ => QueryKind::Other, } } diff --git a/tests/sqllogictests/suites/tpch/join_order.test b/tests/sqllogictests/suites/tpch/join_order.test index c4cfe86159f3..fb91b7297f1c 100644 --- a/tests/sqllogictests/suites/tpch/join_order.test +++ b/tests/sqllogictests/suites/tpch/join_order.test @@ -689,8 +689,7 @@ HashJoin: INNER └── Scan: default.tpch_test.supplier (#0) (read rows: 1000) # Q15 -onlyif todo -statement ok +query T explain join with revenue as materialized ( select l_suppkey as supplier_no, @@ -721,6 +720,16 @@ where ) order by s_suppkey; +---- +HashJoin: INNER +├── Build +│ └── HashJoin: INNER +│ ├── Build +│ │ └── Scan: default.tpch_test.revenue (#1) (read rows: 1000) +│ └── Probe +│ └── Scan: default.tpch_test.supplier (#0) (read rows: 1000) +└── Probe + └── Scan: default.tpch_test.revenue (#2) (read rows: 1000) # Q16 From 99567d72a05c3fc83756460466f72caa15ecf24a Mon Sep 17 00:00:00 2001 From: xudong963 Date: Wed, 4 Dec 2024 11:04:59 +0800 Subject: [PATCH 16/18] address some comments --- src/query/catalog/src/table_context.rs | 2 +- .../service/src/interpreters/interpreter.rs | 16 ++++++++++++---- src/query/service/src/sessions/query_ctx.rs | 9 ++------- .../service/src/sessions/query_ctx_shared.rs | 10 ---------- .../tests/it/sql/exec/get_table_bind_test.rs | 2 +- .../tests/it/storages/fuse/operations/commit.rs | 2 +- .../sql/src/planner/binder/bind_query/bind.rs | 5 ++--- 7 files changed, 19 insertions(+), 27 deletions(-) diff --git a/src/query/catalog/src/table_context.rs b/src/query/catalog/src/table_context.rs index 3deaa3918d2f..8bd8ae92411c 100644 --- a/src/query/catalog/src/table_context.rs +++ b/src/query/catalog/src/table_context.rs @@ -261,7 +261,7 @@ pub trait TableContext: Send + Sync { async fn get_table(&self, catalog: &str, database: &str, table: &str) -> Result>; - fn remove_table_from_cache(&self, catalog: &str, database: &str, table: &str); + fn evict_table_from_cache(&self, catalog: &str, database: &str, table: &str) -> Result<()>; async fn get_table_with_batch( &self, diff --git a/src/query/service/src/interpreters/interpreter.rs b/src/query/service/src/interpreters/interpreter.rs index 30a80033f976..1c79b737c5b5 100644 --- a/src/query/service/src/interpreters/interpreter.rs +++ b/src/query/service/src/interpreters/interpreter.rs @@ -27,6 +27,7 @@ use databend_common_base::base::short_sql; use databend_common_base::runtime::profile::get_statistics_desc; use databend_common_base::runtime::profile::ProfileDesc; use databend_common_base::runtime::profile::ProfileStatisticsName; +use databend_common_base::runtime::GlobalIORuntime; use databend_common_catalog::query_kind::QueryKind; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; @@ -85,10 +86,7 @@ pub trait Interpreter: Sync + Send { #[fastrace::trace] async fn execute(&self, ctx: Arc) -> Result { log_query_start(&ctx); - let res = self.execute_inner(ctx.clone()).await; - // Drop the temp tables that are generated by materialized cte. - ctx.drop_m_cte_temp_table().await?; - match res { + match self.execute_inner(ctx.clone()).await { Ok(stream) => Ok(stream), Err(err) => { log_query_finished(&ctx, Some(err.clone()), false); @@ -321,6 +319,8 @@ pub fn on_execution_finished(info: &ExecutionInfo, query_ctx: Arc) })?; } + hook_clear_m_cte_temp_table(&query_ctx)?; + hook_vacuum_temp_files(&query_ctx)?; hook_disk_temp_dir(&query_ctx)?; @@ -363,3 +363,11 @@ fn need_acquire_lock(ctx: Arc, stmt: &Statement) -> bool { _ => false, } } + +fn hook_clear_m_cte_temp_table(query_ctx: &Arc) -> Result<()> { + let _ = GlobalIORuntime::instance().block_on(async move { + query_ctx.drop_m_cte_temp_table().await?; + Ok(()) + }); + Ok(()) +} diff --git a/src/query/service/src/sessions/query_ctx.rs b/src/query/service/src/sessions/query_ctx.rs index 751975750431..2a284a2d22ff 100644 --- a/src/query/service/src/sessions/query_ctx.rs +++ b/src/query/service/src/sessions/query_ctx.rs @@ -353,10 +353,6 @@ impl QueryContext { *self.shared.finish_time.write() = Some(time) } - pub fn evict_table_from_cache(&self, catalog: &str, database: &str, table: &str) -> Result<()> { - self.shared.evict_table_from_cache(catalog, database, table) - } - pub fn clear_tables_cache(&self) { self.shared.clear_tables_cache() } @@ -972,9 +968,8 @@ impl TableContext for QueryContext { .await } - fn remove_table_from_cache(&self, catalog: &str, database: &str, table: &str) { - self.shared - .remove_table_from_cache(catalog, database, table) + fn evict_table_from_cache(&self, catalog: &str, database: &str, table: &str) -> Result<()> { + self.shared.evict_table_from_cache(catalog, database, table) } #[async_backtrace::framed] diff --git a/src/query/service/src/sessions/query_ctx_shared.rs b/src/query/service/src/sessions/query_ctx_shared.rs index a5ab6c0207e1..7864f495d39f 100644 --- a/src/query/service/src/sessions/query_ctx_shared.rs +++ b/src/query/service/src/sessions/query_ctx_shared.rs @@ -357,16 +357,6 @@ impl QueryContextShared { Ok(res) } - pub fn remove_table_from_cache(&self, catalog_name: &str, database: &str, table: &str) { - let table_meta_key = ( - catalog_name.to_string(), - database.to_string(), - table.to_string(), - ); - let mut tables_refs = self.tables_refs.lock(); - tables_refs.remove(&table_meta_key); - } - #[async_backtrace::framed] async fn get_table_to_cache( &self, diff --git a/src/query/service/tests/it/sql/exec/get_table_bind_test.rs b/src/query/service/tests/it/sql/exec/get_table_bind_test.rs index 3d3d2fc7e23f..cbce3003936b 100644 --- a/src/query/service/tests/it/sql/exec/get_table_bind_test.rs +++ b/src/query/service/tests/it/sql/exec/get_table_bind_test.rs @@ -818,7 +818,7 @@ impl TableContext for CtxDelegation { } } - fn remove_table_from_cache(&self, _catalog: &str, _database: &str, _table: &str) { + fn evict_table_from_cache(&self, _catalog: &str, _database: &str, _table: &str) -> Result<()> { todo!() } diff --git a/src/query/service/tests/it/storages/fuse/operations/commit.rs b/src/query/service/tests/it/storages/fuse/operations/commit.rs index 8d494ca256f8..df0129b412be 100644 --- a/src/query/service/tests/it/storages/fuse/operations/commit.rs +++ b/src/query/service/tests/it/storages/fuse/operations/commit.rs @@ -697,7 +697,7 @@ impl TableContext for CtxDelegation { todo!() } - fn remove_table_from_cache(&self, _catalog: &str, _database: &str, _table: &str) { + fn evict_table_from_cache(&self, _catalog: &str, _database: &str, _table: &str) -> Result<()> { todo!() } diff --git a/src/query/sql/src/planner/binder/bind_query/bind.rs b/src/query/sql/src/planner/binder/bind_query/bind.rs index 1f853b3957d4..94386ce8b817 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind.rs @@ -198,7 +198,7 @@ impl Binder { ))); } let create_table_stmt = CreateTableStmt { - create_option: CreateOption::CreateOrReplace, + create_option: CreateOption::Create, catalog: Some(Identifier::from_name(Span::None, CATALOG_DEFAULT)), database: Some(Identifier::from_name(Span::None, database.clone())), table: cte.alias.name.clone(), @@ -225,7 +225,6 @@ impl Binder { self.ctx.add_m_cte_temp_table(&database, &table_name); self.ctx - .remove_table_from_cache(CATALOG_DEFAULT, &database, &table_name); - Ok(()) + .evict_table_from_cache(CATALOG_DEFAULT, &database, &table_name) } } From 5127147a834195038c3a2e240185dcf38eb2de33 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Wed, 4 Dec 2024 11:51:23 +0800 Subject: [PATCH 17/18] add basic table stats for memery table --- src/query/storages/memory/src/memory_table.rs | 21 +++++++++++++++++++ .../standalone/explain/materialized_cte.test | 12 +++++------ .../push_down_filter_self_join.test | 10 ++++----- 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/src/query/storages/memory/src/memory_table.rs b/src/query/storages/memory/src/memory_table.rs index b1830ac3c4f2..2ec25250b6c7 100644 --- a/src/query/storages/memory/src/memory_table.rs +++ b/src/query/storages/memory/src/memory_table.rs @@ -27,6 +27,7 @@ use databend_common_catalog::plan::PartitionsShuffleKind; use databend_common_catalog::plan::Projection; use databend_common_catalog::plan::PushDownInfo; use databend_common_catalog::table::Table; +use databend_common_catalog::table::TableStatistics; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; @@ -50,6 +51,7 @@ use databend_common_storage::StorageMetrics; use databend_storages_common_blocks::memory::InMemoryDataKey; use databend_storages_common_blocks::memory::IN_MEMORY_DATA; use databend_storages_common_table_meta::meta::SnapshotId; +use databend_storages_common_table_meta::table::ChangeType; use databend_storages_common_table_meta::table::OPT_KEY_TEMP_PREFIX; use parking_lot::Mutex; use parking_lot::RwLock; @@ -264,6 +266,25 @@ impl Table for MemoryTable { self.truncate(); Ok(()) } + + async fn table_statistics( + &self, + _ctx: Arc, + _require_fresh: bool, + _change_type: Option, + ) -> Result> { + let blocks = self.blocks.read(); + let num_rows = blocks.iter().map(|b| b.num_rows() as u64).sum(); + let data_bytes = blocks.iter().map(|b| b.memory_size() as u64).sum(); + Ok(Some(TableStatistics { + num_rows: Some(num_rows), + data_size: Some(data_bytes), + data_size_compressed: Some(data_bytes), + index_size: None, + number_of_blocks: None, + number_of_segments: None, + })) + } } struct MemoryTableSource { diff --git a/tests/sqllogictests/suites/mode/standalone/explain/materialized_cte.test b/tests/sqllogictests/suites/mode/standalone/explain/materialized_cte.test index 6638183e384b..534c788c31ac 100644 --- a/tests/sqllogictests/suites/mode/standalone/explain/materialized_cte.test +++ b/tests/sqllogictests/suites/mode/standalone/explain/materialized_cte.test @@ -7,7 +7,7 @@ HashJoin ├── build keys: [t2.b (#1)] ├── probe keys: [t1.a (#0)] ├── filters: [] -├── estimated rows: 0.00 +├── estimated rows: 100.00 ├── TableScan(Build) │ ├── table: default.default.t1 │ ├── output columns: [a (#1)] @@ -16,7 +16,7 @@ HashJoin │ ├── partitions total: 0 │ ├── partitions scanned: 0 │ ├── push downs: [filters: [], limit: NONE] -│ └── estimated rows: 0.00 +│ └── estimated rows: 10.00 └── TableScan(Probe) ├── table: default.default.t1 ├── output columns: [a (#0)] @@ -25,7 +25,7 @@ HashJoin ├── partitions total: 0 ├── partitions scanned: 0 ├── push downs: [filters: [], limit: NONE] - └── estimated rows: 0.00 + └── estimated rows: 10.00 query T explain with t1 as materialized (select number as a from numbers(10)), t2 as materialized (select a as b from t1) select t1.a from t1 join t2 on t1.a = t2.b; @@ -36,7 +36,7 @@ HashJoin ├── build keys: [t2.b (#1)] ├── probe keys: [t1.a (#0)] ├── filters: [] -├── estimated rows: 0.00 +├── estimated rows: 100.00 ├── TableScan(Build) │ ├── table: default.default.t2 │ ├── output columns: [b (#1)] @@ -45,7 +45,7 @@ HashJoin │ ├── partitions total: 0 │ ├── partitions scanned: 0 │ ├── push downs: [filters: [], limit: NONE] -│ └── estimated rows: 0.00 +│ └── estimated rows: 10.00 └── TableScan(Probe) ├── table: default.default.t1 ├── output columns: [a (#0)] @@ -54,4 +54,4 @@ HashJoin ├── partitions total: 0 ├── partitions scanned: 0 ├── push downs: [filters: [], limit: NONE] - └── estimated rows: 0.00 + └── estimated rows: 10.00 diff --git a/tests/sqllogictests/suites/mode/standalone/explain/push_down_filter/push_down_filter_join/push_down_filter_self_join.test b/tests/sqllogictests/suites/mode/standalone/explain/push_down_filter/push_down_filter_join/push_down_filter_self_join.test index e422891145d6..61a01456e561 100644 --- a/tests/sqllogictests/suites/mode/standalone/explain/push_down_filter/push_down_filter_join/push_down_filter_self_join.test +++ b/tests/sqllogictests/suites/mode/standalone/explain/push_down_filter/push_down_filter_join/push_down_filter_self_join.test @@ -1,8 +1,8 @@ statement ok -create table t1 as select number as a, number as b from numbers(10); +create or replace table t1 as select number as a, number as b from numbers(10); statement ok -create table t2 as select number as a, number as b from numbers(10); +create or replace table t2 as select number as a, number as b from numbers(10); query T explain with A as materialized (select * from t1 union all select * from t2), @@ -17,7 +17,7 @@ HashJoin ├── build keys: [b2.a (#2)] ├── probe keys: [b1.a (#0)] ├── filters: [d.b (#1) < d.b (#3)] -├── estimated rows: 0.00 +├── estimated rows: 400.00 ├── TableScan(Build) │ ├── table: default.default.a │ ├── output columns: [a (#2), b (#3)] @@ -26,7 +26,7 @@ HashJoin │ ├── partitions total: 0 │ ├── partitions scanned: 0 │ ├── push downs: [filters: [], limit: NONE] -│ └── estimated rows: 0.00 +│ └── estimated rows: 20.00 └── TableScan(Probe) ├── table: default.default.a ├── output columns: [a (#0), b (#1)] @@ -35,4 +35,4 @@ HashJoin ├── partitions total: 0 ├── partitions scanned: 0 ├── push downs: [filters: [], limit: NONE] - └── estimated rows: 0.00 + └── estimated rows: 20.00 From 81ddf034dbded07eacc6344755375484ddd1fc03 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Wed, 4 Dec 2024 15:06:29 +0800 Subject: [PATCH 18/18] update test --- tests/sqllogictests/suites/tpch/join_order.test | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/sqllogictests/suites/tpch/join_order.test b/tests/sqllogictests/suites/tpch/join_order.test index fb91b7297f1c..41303a945e54 100644 --- a/tests/sqllogictests/suites/tpch/join_order.test +++ b/tests/sqllogictests/suites/tpch/join_order.test @@ -723,13 +723,13 @@ order by ---- HashJoin: INNER ├── Build -│ └── HashJoin: INNER -│ ├── Build -│ │ └── Scan: default.tpch_test.revenue (#1) (read rows: 1000) -│ └── Probe -│ └── Scan: default.tpch_test.supplier (#0) (read rows: 1000) +│ └── Scan: default.tpch_test.revenue (#2) (read rows: 1000) └── Probe - └── Scan: default.tpch_test.revenue (#2) (read rows: 1000) + └── HashJoin: INNER + ├── Build + │ └── Scan: default.tpch_test.revenue (#1) (read rows: 1000) + └── Probe + └── Scan: default.tpch_test.supplier (#0) (read rows: 1000) # Q16