From 4ba84d5cccfa8b57a8815a1ffcde3714689ce01a Mon Sep 17 00:00:00 2001 From: "shuming.li" Date: Tue, 10 Dec 2024 15:36:38 +0800 Subject: [PATCH] Add more constraints for partition_retention_condition Signed-off-by: shuming.li --- .../common/util/PropertyAnalyzer.java | 2 +- .../scalar/OperatorFunctionChecker.java | 65 ++++++++ .../rewrite/ScalarOperatorEvaluator.java | 15 ++ .../partition/PartitionSelector.java | 30 ++++ .../analysis/CreateMaterializedViewTest.java | 143 ++++++++++++++++++ .../CreateTableWithPartitionTest.java | 109 +++++++++++++ 6 files changed, 363 insertions(+), 1 deletion(-) create mode 100644 fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/scalar/OperatorFunctionChecker.java diff --git a/fe/fe-core/src/main/java/com/starrocks/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/common/util/PropertyAnalyzer.java index 61c6be1389949..35c76afb75d0c 100644 --- a/fe/fe-core/src/main/java/com/starrocks/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/common/util/PropertyAnalyzer.java @@ -1497,7 +1497,7 @@ public static void analyzeMVProperties(Database db, materializedView.getTableProperty().getProperties() .put(PropertyAnalyzer.PROPERTIES_AUTO_REFRESH_PARTITIONS_LIMIT, String.valueOf(limit)); materializedView.getTableProperty().setAutoRefreshPartitionsLimit(limit); - if (!materializedView.getPartitionInfo().isRangePartition()) { + if (isNonPartitioned) { throw new AnalysisException(PropertyAnalyzer.PROPERTIES_AUTO_REFRESH_PARTITIONS_LIMIT + " does not support non-range-partitioned materialized view."); } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/scalar/OperatorFunctionChecker.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/scalar/OperatorFunctionChecker.java new file mode 100644 index 0000000000000..284e0ad2481a0 --- /dev/null +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/scalar/OperatorFunctionChecker.java @@ -0,0 +1,65 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License + +package com.starrocks.sql.optimizer.operator.scalar; + +import com.starrocks.common.Pair; +import com.starrocks.sql.optimizer.rewrite.ScalarOperatorEvaluator; + +import java.util.function.Predicate; + +/** + * FunctionChecker is used to check whether a ScalarOperator only contains a specific type of functions. + */ +public class OperatorFunctionChecker { + static class FunctionCheckerVisitor extends ScalarOperatorVisitor, Void> { + private final Predicate predicate; + + public FunctionCheckerVisitor(Predicate predicate) { + this.predicate = predicate; + } + + @Override + public Pair visit(ScalarOperator scalarOperator, Void context) { + for (ScalarOperator child : scalarOperator.getChildren()) { + Pair result = child.accept(this, null); + if (!result.first) { + return result; + } + } + return Pair.create(true, ""); + } + + public Pair visitCall(CallOperator call, Void context) { + if (predicate.test(call)) { + return Pair.create(true, ""); + } else { + return Pair.create(false, call.getFnName()); + } + } + } + + public static Pair onlyContainPredicates(ScalarOperator scalarOperator, + Predicate predicate) { + return scalarOperator.accept(new FunctionCheckerVisitor(predicate), null); + } + + public static Pair onlyContainMonotonicFunctions(ScalarOperator scalarOperator) { + return onlyContainPredicates(scalarOperator, call -> ScalarOperatorEvaluator.INSTANCE.isMonotonicFunction(call)); + } + + public static Pair onlyContainFEConstantFunctions(ScalarOperator scalarOperator) { + return onlyContainPredicates(scalarOperator, call -> ScalarOperatorEvaluator.INSTANCE.isFEConstantFunction(call)); + } +} diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/ScalarOperatorEvaluator.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/ScalarOperatorEvaluator.java index a422a974c28a0..8963e2bdec479 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/ScalarOperatorEvaluator.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/ScalarOperatorEvaluator.java @@ -225,6 +225,21 @@ public boolean isMonotonicFunction(CallOperator call) { return invoker != null && isMonotonicFunc(invoker, call); } + public boolean isFEConstantFunction(CallOperator call) { + FunctionSignature signature; + if (call.getFunction() != null) { + Function fn = call.getFunction(); + List argTypes = Arrays.asList(fn.getArgs()); + signature = new FunctionSignature(fn.functionName().toUpperCase(), argTypes, fn.getReturnType()); + } else { + List argTypes = call.getArguments().stream().map(ScalarOperator::getType).collect(Collectors.toList()); + signature = new FunctionSignature(call.getFnName().toUpperCase(), argTypes, call.getType()); + } + + FunctionInvoker invoker = functions.get(signature); + return invoker != null; + } + private boolean isMonotonicFunc(FunctionInvoker invoker, CallOperator operator) { if (!invoker.isMonotonic) { return false; diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/partition/PartitionSelector.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/partition/PartitionSelector.java index 32e20ce650c96..bbfc5b0202812 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/partition/PartitionSelector.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/partition/PartitionSelector.java @@ -27,6 +27,7 @@ import com.starrocks.catalog.Column; import com.starrocks.catalog.Database; import com.starrocks.catalog.ListPartitionInfo; +import com.starrocks.catalog.MaterializedView; import com.starrocks.catalog.OlapTable; import com.starrocks.catalog.Partition; import com.starrocks.catalog.PartitionInfo; @@ -35,6 +36,7 @@ import com.starrocks.catalog.Table; import com.starrocks.catalog.system.information.InfoSchemaDb; import com.starrocks.catalog.system.information.PartitionsMetaSystemTable; +import com.starrocks.common.Pair; import com.starrocks.load.pipe.filelist.RepoExecutor; import com.starrocks.planner.PartitionColumnFilter; import com.starrocks.planner.PartitionPruner; @@ -59,6 +61,7 @@ import com.starrocks.sql.optimizer.operator.scalar.ColumnRefOperator; import com.starrocks.sql.optimizer.operator.scalar.CompoundPredicateOperator; import com.starrocks.sql.optimizer.operator.scalar.ConstantOperator; +import com.starrocks.sql.optimizer.operator.scalar.OperatorFunctionChecker; import com.starrocks.sql.optimizer.operator.scalar.ScalarOperator; import com.starrocks.sql.optimizer.rewrite.ReplaceColumnRefRewriter; import com.starrocks.sql.optimizer.rewrite.ScalarOperatorRewriter; @@ -188,6 +191,8 @@ public static List getPartitionIdsByExpr(ConnectContext context, if (scalarOperator == null) { throw new SemanticException("Failed to translate where expression to scalar operator:" + whereExpr.toSql()); } + // validate scalar operator + validateRetentionConditionPredicate(olapTable, scalarOperator); List usedPartitionColumnRefs = Lists.newArrayList(); scalarOperator.getColumnRefs(usedPartitionColumnRefs); @@ -598,4 +603,29 @@ private static Expr buildJsonQuery(List partitionCols, Expr expr = SqlParser.parseSqlToExpr(jsonQuery, SqlModeHelper.MODE_DEFAULT); return expr; } + + public static void validateRetentionConditionPredicate(OlapTable olapTable, + ScalarOperator predicate) { + PartitionInfo partitionInfo = olapTable.getPartitionInfo(); + if (partitionInfo.isListPartition()) { + // support common partition expressions for list partition tables + } else if (partitionInfo.isRangePartition()) { + Pair result = OperatorFunctionChecker.onlyContainMonotonicFunctions(predicate); + if (!result.first) { + throw new SemanticException("Retention condition must only contain monotonic functions for range partition " + + "tables but contains: " + result.second); + } + } else { + throw new SemanticException("Unsupported partition type: " + partitionInfo.getType() + " for retention condition"); + } + + // extra check for materialized view + if (olapTable instanceof MaterializedView) { + Pair result = OperatorFunctionChecker.onlyContainFEConstantFunctions(predicate); + if (!result.first) { + throw new SemanticException("Retention condition must only contain FE constant functions for materialized" + + " view but contains: " + result.second); + } + } + } } \ No newline at end of file diff --git a/fe/fe-core/src/test/java/com/starrocks/analysis/CreateMaterializedViewTest.java b/fe/fe-core/src/test/java/com/starrocks/analysis/CreateMaterializedViewTest.java index 104a84c188fa1..97bb08486045b 100644 --- a/fe/fe-core/src/test/java/com/starrocks/analysis/CreateMaterializedViewTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/analysis/CreateMaterializedViewTest.java @@ -5053,6 +5053,42 @@ public void testCreateMVWithMultiPartitionColumns3() throws Exception { starRocksAssert.dropTable("t4"); } + @Test + public void testCreateMVWithAutoRefreshPartitionsLimit() throws Exception { + starRocksAssert.withTable("CREATE TABLE t3 (\n" + + " id BIGINT,\n" + + " age SMALLINT,\n" + + " dt VARCHAR(10) not null,\n" + + " province VARCHAR(64) not null\n" + + ")\n" + + "DUPLICATE KEY(id)\n" + + "PARTITION BY LIST (province, dt, age) (\n" + + " PARTITION p1 VALUES IN ((\"beijing\", \"2024-01-01\", \"10\")),\n" + + " PARTITION p2 VALUES IN ((\"guangdong\", \"2024-01-01\", \"20\")), \n" + + " PARTITION p3 VALUES IN ((\"beijing\", \"2024-01-02\", \"30\")),\n" + + " PARTITION p4 VALUES IN ((\"guangdong\", \"2024-01-02\", \"40\")) \n" + + ")\n" + + "DISTRIBUTED BY RANDOM\n"); + starRocksAssert.withMaterializedView("create materialized view mv1\n" + + "partition by (province, dt, age) \n" + + "REFRESH ASYNC\n" + + "properties (\n" + + "'replication_num' = '1',\n" + + // check auto_refresh_partitions_limit parameter + "'auto_refresh_partitions_limit' = '1'," + + "'partition_retention_condition' = 'dt > current_date() - interval 1 month'\n" + + ") \n" + + "as select dt, province, age, sum(id) from t3 group by dt, province, age;"); + MaterializedView mv = starRocksAssert.getMv("test", "mv1"); + List mvPartitionCols = mv.getPartitionColumns(); + Assert.assertEquals(3, mvPartitionCols.size()); + Assert.assertEquals("province", mvPartitionCols.get(0).getName()); + Assert.assertEquals("dt", mvPartitionCols.get(1).getName()); + Assert.assertEquals("age", mvPartitionCols.get(2).getName()); + starRocksAssert.dropMaterializedView("mv1"); + starRocksAssert.dropTable("t3"); + } + @Test public void testCreateMVWithRetentionCondition1() throws Exception { starRocksAssert.withTable("CREATE TABLE t3 (\n" + @@ -5144,4 +5180,111 @@ public void testCreateMVWithRetentionCondition3() throws Exception { } starRocksAssert.dropTable("t3"); } + + @Test + public void testCreateMVWithRetentionCondition4() throws Exception { + starRocksAssert.withTable("CREATE TABLE t3 (\n" + + " id BIGINT,\n" + + " age SMALLINT,\n" + + " dt VARCHAR(10) not null,\n" + + " province VARCHAR(64) not null\n" + + ")\n" + + "DUPLICATE KEY(id)\n" + + "PARTITION BY province, dt, age\n" + + "DISTRIBUTED BY RANDOM\n"); + starRocksAssert.withMaterializedView("create materialized view mv1\n" + + "partition by (province, dt, age) \n" + + "REFRESH DEFERRED MANUAL \n" + + "properties (\n" + + "'replication_num' = '1',\n" + + "'partition_refresh_number' = '-1'," + + "'partition_retention_condition' = 'dt > current_date() - interval 1 month'\n" + + ") \n" + + "as select dt, province, age, sum(id) from t3 group by dt, province, age;"); + MaterializedView mv = starRocksAssert.getMv("test", "mv1"); + List mvPartitionCols = mv.getPartitionColumns(); + Assert.assertEquals(3, mvPartitionCols.size()); + Assert.assertEquals("province", mvPartitionCols.get(0).getName()); + Assert.assertEquals("dt", mvPartitionCols.get(1).getName()); + Assert.assertEquals("age", mvPartitionCols.get(2).getName()); + + String retentionCondition = mv.getTableProperty().getPartitionRetentionCondition(); + Assert.assertEquals("dt > current_date() - interval 1 month", retentionCondition); + + try { + String alterTableSql = "ALTER MATERIALIZED VIEW mv1 SET ('partition_retention_condition' = " + + "'last_day(dt) > current_date() - interval 2 month')"; + starRocksAssert.alterMvProperties(alterTableSql); + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains("Retention condition must only contain FE constant functions " + + "for materialized view but contains: last_day")); + } + + String alterTableSql = "ALTER MATERIALIZED VIEW mv1 SET ('partition_retention_condition' = " + + "'date_format(dt, \\'%m月%Y年\\') > current_date() - interval 2 month')"; + starRocksAssert.alterMvProperties(alterTableSql); + + retentionCondition = mv.getTableProperty().getPartitionRetentionCondition(); + Assert.assertEquals("date_format(dt, '%m月%Y年') > current_date() - interval 2 month", retentionCondition); + starRocksAssert.dropMaterializedView("mv1"); + starRocksAssert.dropTable("t3"); + } + + @Test + public void testCreateMVWithRetentionCondition5() throws Exception { + starRocksAssert.withTable("CREATE TABLE r1 \n" + + "(\n" + + " dt date,\n" + + " k2 int,\n" + + " v1 int \n" + + ")\n" + + "PARTITION BY RANGE(dt)\n" + + "(\n" + + " PARTITION p0 values [('2024-01-29'),('2024-01-30')),\n" + + " PARTITION p1 values [('2024-01-30'),('2024-01-31')),\n" + + " PARTITION p2 values [('2024-01-31'),('2024-02-01')),\n" + + " PARTITION p3 values [('2024-02-01'),('2024-02-02')) \n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + + "PROPERTIES (\n" + + "'replication_num' = '1',\n" + + "'partition_retention_condition' = 'dt > current_date() - interval 1 month'\n" + + ")"); + starRocksAssert.withMaterializedView("create materialized view mv1\n" + + "partition by (dt) \n" + + "REFRESH DEFERRED MANUAL \n" + + "properties (\n" + + "'replication_num' = '1',\n" + + "'partition_refresh_number' = '-1'," + + "'partition_retention_condition' = 'dt > current_date() - interval 1 month'\n" + + ") \n" + + "as select * from r1;"); + MaterializedView mv = starRocksAssert.getMv("test", "mv1"); + String retentionCondition = mv.getTableProperty().getPartitionRetentionCondition(); + Assert.assertEquals("dt > current_date() - interval 1 month", retentionCondition); + + try { + String alterTableSql = "ALTER MATERIALIZED VIEW mv1 SET ('partition_retention_condition' = " + + "'last_day(dt) > current_date() - interval 2 month')"; + starRocksAssert.alterMvProperties(alterTableSql); + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains("Retention condition must only contain monotonic functions for " + + "range partition tables but contains: last_day")); + } + + try { + + String alterTableSql = "ALTER MATERIALIZED VIEW mv1 SET ('partition_retention_condition' = " + + "'date_format(dt, \\'%m月%Y年\\') > current_date() - interval 2 month')"; + starRocksAssert.alterMvProperties(alterTableSql); + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains("Retention condition must only contain monotonic functions for " + + "range partition tables but contains: date_format")); + } + + retentionCondition = mv.getTableProperty().getPartitionRetentionCondition(); + Assert.assertEquals("dt > current_date() - interval 1 month", retentionCondition); + starRocksAssert.dropMaterializedView("mv1"); + starRocksAssert.dropTable("r1"); + } } \ No newline at end of file diff --git a/fe/fe-core/src/test/java/com/starrocks/analysis/CreateTableWithPartitionTest.java b/fe/fe-core/src/test/java/com/starrocks/analysis/CreateTableWithPartitionTest.java index 7576d5f7884c9..496ce2db067c6 100644 --- a/fe/fe-core/src/test/java/com/starrocks/analysis/CreateTableWithPartitionTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/analysis/CreateTableWithPartitionTest.java @@ -15,6 +15,7 @@ package com.starrocks.analysis; +import com.starrocks.catalog.OlapTable; import com.starrocks.common.AnalysisException; import com.starrocks.common.Config; import com.starrocks.qe.ConnectContext; @@ -987,5 +988,113 @@ public void testAnalyzeRetentionConditionWithRangePartition1() { Assert.fail(e.getMessage()); } } + + @Test + public void testListTableWithRetentionCondition() { + try { + starRocksAssert.withTable("CREATE TABLE t1 (\n" + + " id BIGINT,\n" + + " age SMALLINT,\n" + + " dt datetime not null,\n" + + " province VARCHAR(64) not null\n" + + ")\n" + + "PARTITION BY (province, dt) \n" + + "DISTRIBUTED BY RANDOM\n" + + "PROPERTIES (\n" + + "'replication_num' = '1',\n" + + "'partition_retention_condition' = 'dt > current_date() - interval 1 month'\n" + + ")"); + OlapTable t1 = (OlapTable) starRocksAssert.getTable("db1", "t1"); + String retentionCondition = t1.getTableProperty().getPartitionRetentionCondition(); + Assert.assertEquals("dt > current_date() - interval 1 month", retentionCondition); + + { + String alterTableSql = "ALTER TABLE t1 SET ('partition_retention_condition' = " + + "'last_day(dt) > current_date() - interval 2 month')"; + starRocksAssert.alterTableProperties(alterTableSql); + } + { + String alterTableSql = "ALTER TABLE t1 SET ('partition_retention_condition' = " + + "'dt > current_date() - interval 1 month or last_day(dt) > current_date() - interval 2 month')"; + starRocksAssert.alterTableProperties(alterTableSql); + } + + { + String alterTableSql = "ALTER TABLE t1 SET ('partition_retention_condition' = " + + "'date_format(dt, \\'%m月%Y年\\') > current_date() - interval 2 month')"; + starRocksAssert.alterTableProperties(alterTableSql); + } + + retentionCondition = t1.getTableProperty().getPartitionRetentionCondition(); + Assert.assertEquals("date_format(dt, '%m月%Y年') > current_date() - interval 2 month", retentionCondition); + starRocksAssert.dropTable("t1"); + } catch (Exception e) { + Assert.fail(e.getMessage()); + } + } + + @Test + public void testRangeTableWithRetentionCondition() throws Exception { + starRocksAssert.withTable("CREATE TABLE r1 \n" + + "(\n" + + " dt date,\n" + + " k2 int,\n" + + " v1 int \n" + + ")\n" + + "PARTITION BY RANGE(dt)\n" + + "(\n" + + " PARTITION p0 values [('2024-01-29'),('2024-01-30')),\n" + + " PARTITION p1 values [('2024-01-30'),('2024-01-31')),\n" + + " PARTITION p2 values [('2024-01-31'),('2024-02-01')),\n" + + " PARTITION p3 values [('2024-02-01'),('2024-02-02')) \n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + + "PROPERTIES (\n" + + "'replication_num' = '1',\n" + + "'partition_retention_condition' = 'dt > current_date() - interval 1 month'\n" + + ")"); + OlapTable r1 = (OlapTable) starRocksAssert.getTable("db1", "r1"); + String retentionCondition = r1.getTableProperty().getPartitionRetentionCondition(); + Assert.assertEquals("dt > current_date() - interval 1 month", retentionCondition); + + try { + String alterTableSql = "ALTER TABLE r1 SET ('partition_retention_condition' = " + + "'last_day(dt) > current_date() - interval 2 month')"; + starRocksAssert.alterTableProperties(alterTableSql); + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains("Retention condition must only contain monotonic functions " + + "for range partition tables but contains: last_day")); + } + + try { + String alterTableSql = "ALTER TABLE r1 SET ('partition_retention_condition' = " + + "'dt > current_date() - interval 1 month or last_day(dt) > current_date() - interval 2 month')"; + starRocksAssert.alterTableProperties(alterTableSql); + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains("Retention condition must only contain monotonic functions " + + "for range partition tables but contains: last_day")); + } + + try { + String alterTableSql = "ALTER TABLE r1 SET ('partition_retention_condition' = " + + "'date_format(dt, \\'%m月%Y年\\') > current_date() - interval 2 month')"; + starRocksAssert.alterTableProperties(alterTableSql); + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains("Retention condition must only contain monotonic functions " + + "for range partition tables but contains: date_format")); + } + + try { + String alterTableSql = "ALTER TABLE r1 SET ('partition_retention_condition' = " + + "'date_format(dt, \\'%a-%Y\\') > current_date() - interval 2 month')"; + starRocksAssert.alterTableProperties(alterTableSql); + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains("Retention condition must only contain monotonic functions " + + "for range partition tables but contains: date_format")); + } + retentionCondition = r1.getTableProperty().getPartitionRetentionCondition(); + Assert.assertEquals("dt > current_date() - interval 1 month", retentionCondition); + starRocksAssert.dropTable("r1"); + } }