feat(shannon): parallel query processing ii

part ii of parallel query processing.
Shannon-Data · Jun 6, 2024 · a2fccc5 · a2fccc5
1 parent d5bb69e
commit a2fccc5
Show file tree

Hide file tree

Showing 23 changed files with 731 additions and 104 deletions.
diff --git a/mysql-test/r/mysqld--help-notwin.result b/mysql-test/r/mysqld--help-notwin.result
@@ -400,6 +400,8 @@ The following options may be given as the first argument:
  --flush             Flush MyISAM tables to disk between SQL commands
  --flush-time=#      A dedicated thread is created to flush all tables at the
  given interval
+ --force-parallel-execute 
+ force parallel execute in session
  --ft-boolean-syntax=name 
  List of operators for MATCH ... AGAINST ( ... IN BOOLEAN
  MODE)
@@ -842,6 +844,17 @@ The following options may be given as the first argument:
  Maximum allowed cumulated size of stored optimizer traces
  --optimizer-trace-offset=# 
  Offset of first optimizer trace to show; see manual
+ --parallel-cost-threshold=# 
+ Cost threshold for parallel query.
+ --parallel-default-dop=# 
+ default degree of parallel query.
+ --parallel-max-threads=# 
+ max running threads of parallel query.
+ --parallel-memory-limit=# 
+ upper limit memory size that parallel query can use
+ --parallel-queue-timeout=# 
+ queue timeout for parallel query when resource is not
+ enough .the unit is microseconds
  --parser-max-mem-size=# 
  Maximum amount of memory available to the parser
  --partial-revokes   Access of database objects can be restricted, even if
@@ -1754,6 +1767,7 @@ explicit-defaults-for-timestamp TRUE
 external-locking FALSE
 flush FALSE
 flush-time 0
+force-parallel-execute FALSE
 ft-boolean-syntax + -><()~*:""&|
 ft-max-word-len 84
 ft-min-word-len 4
@@ -1881,6 +1895,11 @@ optimizer-trace-features greedy_search=on,range_optimizer=on,dynamic_range=on,re
 optimizer-trace-limit 1
 optimizer-trace-max-mem-size 1048576
 optimizer-trace-offset -1
+parallel-cost-threshold 1000
+parallel-default-dop 4
+parallel-max-threads 64
+parallel-memory-limit 104857600
+parallel-queue-timeout 0
 parser-max-mem-size 18446744073709551615
 partial-revokes ####
 password-history 0

diff --git a/sql/field.cc b/sql/field.cc
@@ -2757,9 +2757,11 @@ Field_new_decimal::Field_new_decimal(uint32 len_arg, bool is_nullable_arg,
   bin_size = my_decimal_get_binary_size(precision, dec);
 }
 
-Field *Field_new_decimal::create_from_item(const Item *item) {
+Field *Field_new_decimal::create_from_item(const Item *item, MEM_ROOT *root) {
+  MEM_ROOT *pq_check_root = root ? root : *THR_MALLOC;
+
   uint8 dec = item->decimals;
-  const uint8 intg = item->decimal_precision() - dec;
+  uint8 intg = item->decimal_precision() - dec;
   uint32 len = item->max_char_length();
 
   assert(item->result_type() == DECIMAL_RESULT);
@@ -2793,7 +2795,7 @@ Field *Field_new_decimal::create_from_item(const Item *item) {
       /* Corrected value fits. */
       len = required_length;
   }
-  return new (*THR_MALLOC)
+  return new (pq_check_root)
       Field_new_decimal(len, item->is_nullable(), item->item_name.ptr(), dec,
                         item->unsigned_flag);
 }

diff --git a/sql/field.h b/sql/field.h
@@ -2167,7 +2167,7 @@ class Field_new_decimal : public Field_num {
     return new (mem_root) Field_new_decimal(*this);
   }
   const uchar *unpack(uchar *to, const uchar *from, uint param_data) final;
-  static Field *create_from_item(const Item *item);
+  static Field *create_from_item(const Item *item, MEM_ROOT *root = nullptr);
   bool send_to_protocol(Protocol *protocol) const final;
   void set_keep_precision(bool arg) { m_keep_precision = arg; }
 };

diff --git a/sql/item.cc b/sql/item.cc
@@ -6467,19 +6467,21 @@ bool Item::eq_by_collation(Item *item, bool binary_cmp,
   @param table		Table for which the field is created
 */
 
-Field *Item::make_string_field(TABLE *table) const {
+Field *Item::make_string_field(TABLE *table, MEM_ROOT *root) const {
   Field *field;
+  MEM_ROOT *pq_check_root = root ? root : *THR_MALLOC;
+
   assert(collation.collation);
   if (data_type() == MYSQL_TYPE_JSON)
     field =
-        new (*THR_MALLOC) Field_json(max_length, m_nullable, item_name.ptr());
+        new (pq_check_root) Field_json(max_length, m_nullable, item_name.ptr());
   else if (data_type() == MYSQL_TYPE_GEOMETRY) {
-    field = new (*THR_MALLOC)
+    field = new (pq_check_root)
         Field_geom(max_length, m_nullable, item_name.ptr(),
                    Field::GEOM_GEOMETRY, std::optional<gis::srid_t>());
   } else if (max_length / collation.collation->mbmaxlen >
              CONVERT_IF_BIGGER_TO_BLOB)
-    field = new (*THR_MALLOC) Field_blob(
+    field = new (pq_check_root) Field_blob(
         max_length, m_nullable, item_name.ptr(), collation.collation, true);
   /* Item_type_holder holds the exact type, do not change it */
   else if (max_length > 0 &&
@@ -6502,70 +6504,67 @@ Field *Item::make_string_field(TABLE *table) const {
   @return Created field
   @retval NULL  error
 */
-
-Field *Item::tmp_table_field_from_field_type(TABLE *table,
-                                             bool fixed_length) const {
-  /*
-    The field functions defines a field to be not null if null_ptr is not 0
-  */
+Field *Item::tmp_table_field_from_field_type(TABLE *table, bool fixed_length,
+                                             MEM_ROOT *root) const {
+  /*The field functions defines a field to be not null if null_ptr is not 0*/
   Field *field;
+  MEM_ROOT *pq_check_root = root ? root : *THR_MALLOC;
 
   switch (data_type()) {
     case MYSQL_TYPE_DECIMAL:
     case MYSQL_TYPE_NEWDECIMAL:
-      field = Field_new_decimal::create_from_item(this);
+      field = Field_new_decimal::create_from_item(this, root);
       break;
     case MYSQL_TYPE_TINY:
-      field = new (*THR_MALLOC)
+      field = new (pq_check_root)
           Field_tiny(max_length, m_nullable, item_name.ptr(), unsigned_flag);
       break;
     case MYSQL_TYPE_SHORT:
-      field = new (*THR_MALLOC)
+      field = new (pq_check_root)
           Field_short(max_length, m_nullable, item_name.ptr(), unsigned_flag);
       break;
     case MYSQL_TYPE_LONG:
-      field = new (*THR_MALLOC)
+      field = new (pq_check_root)
           Field_long(max_length, m_nullable, item_name.ptr(), unsigned_flag);
       break;
     case MYSQL_TYPE_LONGLONG:
-      field = new (*THR_MALLOC) Field_longlong(max_length, m_nullable,
-                                               item_name.ptr(), unsigned_flag);
+      field = new (pq_check_root) Field_longlong(
+          max_length, m_nullable, item_name.ptr(), unsigned_flag);
       break;
     case MYSQL_TYPE_FLOAT:
-      field = new (*THR_MALLOC) Field_float(
+      field = new (pq_check_root) Field_float(
           max_length, m_nullable, item_name.ptr(), decimals, unsigned_flag);
       break;
     case MYSQL_TYPE_DOUBLE:
-      field = new (*THR_MALLOC) Field_double(
+      field = new (pq_check_root) Field_double(
           max_length, m_nullable, item_name.ptr(), decimals, unsigned_flag);
       break;
     case MYSQL_TYPE_INT24:
-      field = new (*THR_MALLOC)
+      field = new (pq_check_root)
           Field_medium(max_length, m_nullable, item_name.ptr(), unsigned_flag);
       break;
     case MYSQL_TYPE_DATE:
     case MYSQL_TYPE_NEWDATE:
-      field = new (*THR_MALLOC) Field_newdate(m_nullable, item_name.ptr());
+      field = new (pq_check_root) Field_newdate(m_nullable, item_name.ptr());
       break;
     case MYSQL_TYPE_TIME:
-      field =
-          new (*THR_MALLOC) Field_timef(m_nullable, item_name.ptr(), decimals);
+      field = new (pq_check_root)
+          Field_timef(m_nullable, item_name.ptr(), decimals);
       break;
     case MYSQL_TYPE_TIMESTAMP:
-      field = new (*THR_MALLOC)
+      field = new (pq_check_root)
           Field_timestampf(m_nullable, item_name.ptr(), decimals);
       break;
     case MYSQL_TYPE_DATETIME:
-      field = new (*THR_MALLOC)
+      field = new (pq_check_root)
           Field_datetimef(m_nullable, item_name.ptr(), decimals);
       break;
     case MYSQL_TYPE_YEAR:
       assert(max_length == 4);  // Field_year is only for length 4.
-      assert(decimal_precision() == 4);
-      field = new (*THR_MALLOC) Field_year(m_nullable, item_name.ptr());
+      field = new (pq_check_root) Field_year(m_nullable, item_name.ptr());
       break;
     case MYSQL_TYPE_BIT:
-      field = new (*THR_MALLOC)
+      field = new (pq_check_root)
           Field_bit_as_char(max_length, m_nullable, item_name.ptr());
       break;
     case MYSQL_TYPE_INVALID:
@@ -6578,7 +6577,7 @@ Field *Item::tmp_table_field_from_field_type(TABLE *table,
     case MYSQL_TYPE_STRING:
     case MYSQL_TYPE_NULL:
       if (fixed_length && max_length <= CONVERT_IF_BIGGER_TO_BLOB) {
-        field = new (*THR_MALLOC) Field_string(
+        field = new (pq_check_root) Field_string(
             max_length, m_nullable, item_name.ptr(), collation.collation);
         break;
       }
@@ -6588,26 +6587,28 @@ Field *Item::tmp_table_field_from_field_type(TABLE *table,
     case MYSQL_TYPE_SET:
     case MYSQL_TYPE_VAR_STRING:
     case MYSQL_TYPE_VARCHAR:
-      return make_string_field(table);
+      field = make_string_field(table, root);
+      // if (field) field->set_pseudo(is_pseudo);
+      return field;
     case MYSQL_TYPE_TINY_BLOB:
     case MYSQL_TYPE_MEDIUM_BLOB:
     case MYSQL_TYPE_LONG_BLOB:
     case MYSQL_TYPE_BLOB:
       if (this->type() == Item::TYPE_HOLDER)
-        field = new (*THR_MALLOC) Field_blob(
+        field = new (pq_check_root) Field_blob(
             max_length, m_nullable, item_name.ptr(), collation.collation, true);
       else
-        field = new (*THR_MALLOC)
+        field = new (pq_check_root)
             Field_blob(max_length, m_nullable, item_name.ptr(),
                        collation.collation, false);
       break;  // Blob handled outside of case
     case MYSQL_TYPE_GEOMETRY:
-      field = new (*THR_MALLOC) Field_geom(
+      field = new (pq_check_root) Field_geom(
           max_length, m_nullable, item_name.ptr(), get_geometry_type(), {});
       break;
     case MYSQL_TYPE_JSON:
-      field =
-          new (*THR_MALLOC) Field_json(max_length, m_nullable, item_name.ptr());
+      field = new (pq_check_root)
+          Field_json(max_length, m_nullable, item_name.ptr());
   }
   if (field) field->init(table);
   return field;
@@ -11026,4 +11027,8 @@ bool AllItemsAreEqual(const Item *const *a, const Item *const *b, int num_items,
   return true;
 }
 
+void set_has_notsupported_func_true(void) {
+  current_thd->lex->has_notsupported_func = true;
+}
+
 #include "sql/sql_pq_clone_item.inc"
diff --git a/sql/item.h b/sql/item.h
@@ -1239,7 +1239,8 @@ class Item : public Parse_tree_node {
   */
   virtual void notify_removal() {}
   virtual void make_field(Send_field *field);
-  virtual Field *make_string_field(TABLE *table) const;
+  virtual Field *make_string_field(TABLE *table,
+                                   MEM_ROOT *root = nullptr) const;
   virtual bool fix_fields(THD *, Item **);
   /**
     Fix after tables have been moved from one query_block level to the parent
@@ -3092,7 +3093,9 @@ class Item : public Parse_tree_node {
   // used in row subselects to get value of elements
   virtual void bring_value() {}
 
-  Field *tmp_table_field_from_field_type(TABLE *table, bool fixed_length) const;
+  Field *tmp_table_field_from_field_type(TABLE *table, bool fixed_length,
+                                         MEM_ROOT *root = nullptr) const;  
+
   virtual Item_field *field_for_view_update() { return nullptr; }
   /**
     Informs an item that it is wrapped in a truth test, in case it wants to
@@ -5870,16 +5873,15 @@ class Item_ref : public Item_ident {
   };
 
   PQ_copy_type copy_type;
+
+  /// Indirect pointer to the referenced item.
+  Item **m_ref_item{nullptr};
  private:
   /// True if referenced item has been unlinked, used during item tree removal
   bool m_unlinked{false};
 
   Field *result_field{nullptr}; /* Save result here */
 
- protected:
-  /// Indirect pointer to the referenced item.
-  Item **m_ref_item{nullptr};
-
  public:
   Item_ref(Name_resolution_context *context_arg, const char *db_name_arg,
            const char *table_name_arg, const char *field_name_arg)
@@ -5911,8 +5913,8 @@ class Item_ref : public Item_ident {
   /* Constructor need to process subselect with temporary tables (see Item) */
   Item_ref(THD *thd, Item_ref *item)
       : Item_ident(thd, item),
-        result_field(item->result_field),
-        m_ref_item(item->m_ref_item) {}
+      m_ref_item(item->m_ref_item),
+      result_field(item->result_field) {}
 
   /// @returns the item referenced by this object
   Item *ref_item() const { return *m_ref_item; }
@@ -6097,7 +6099,8 @@ class Item_ref : public Item_ident {
     return ref_item()->check_column_in_group_by(arg);
   }
   bool collect_item_field_or_ref_processor(uchar *arg) override;
-  Item *pq_clone(THD *thd, Query_block *select) override;  
+
+  Item *pq_clone(THD *thd, Query_block *select) override;
 };
 
 /**

diff --git a/sql/item_func.cc b/sql/item_func.cc
@@ -8036,6 +8036,7 @@ bool Item_func_sp::do_itemize(Parse_context *pc, Item **res) {
 
   context = lex->current_context();
   lex->safe_to_cache_query = false;
+  lex->has_sp = true;
 
   if (m_name->m_db.str == nullptr) {
     if (thd->lex->copy_db_to(&m_name->m_db.str, &m_name->m_db.length)) {

diff --git a/sql/item_strfunc.h b/sql/item_strfunc.h
@@ -786,6 +786,7 @@ class Item_func_make_set final : public Item_str_func {
   void print(const THD *thd, String *str,
              enum_query_type query_type) const override;
   Item *pq_clone(THD *thd, Query_block *select) override; 
+  Item* get_item() { return item; } 
 };
 
 class Item_func_format final : public Item_str_ascii_func {

diff --git a/sql/item_sum.cc b/sql/item_sum.cc
@@ -1,4 +1,5 @@
 /* Copyright (c) 2000, 2023, Oracle and/or its affiliates.
+   Copyright (c) 2021, Huawei Technologies Co., Ltd.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License, version 2.0,
@@ -18,7 +19,9 @@
 
    You should have received a copy of the GNU General Public License
    along with this program; if not, write to the Free Software
-   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA */
+   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA
+
+   Copyright (c) 2023, Shannon Data AI and/or its affiliates. */
 
 /**
   @file
@@ -4314,8 +4317,10 @@ void Item_func_group_concat::cleanup() {
   row_count = 0;
 }
 
-Field *Item_func_group_concat::make_string_field(TABLE *table_arg) const {
+Field *Item_func_group_concat::make_string_field(TABLE *table_arg,
+                                                 MEM_ROOT *root) const {
   Field *field;
+  MEM_ROOT *pq_check_root = root ? root : *THR_MALLOC;
   assert(collation.collation);
   /*
     Use mbminlen to determine maximum number of characters.
@@ -4335,11 +4340,11 @@ Field *Item_func_group_concat::make_string_field(TABLE *table_arg) const {
       UINT_MAX32);
 
   if (max_characters > CONVERT_IF_BIGGER_TO_BLOB)
-    field = new (*THR_MALLOC)
+    field = new (pq_check_root)
         Field_blob(field_length, is_nullable(), item_name.ptr(),
                    collation.collation, true);
   else
-    field = new (*THR_MALLOC)
+    field = new (pq_check_root)
         Field_varstring(field_length, is_nullable(), item_name.ptr(),
                         table_arg->s, collation.collation);
 

diff --git a/sql/item_sum.h b/sql/item_sum.h
@@ -2207,7 +2207,8 @@ class Item_func_group_concat final : public Item_sum {
   enum Sumfunctype sum_func() const override { return GROUP_CONCAT_FUNC; }
   const char *func_name() const override { return "group_concat"; }
   Item_result result_type() const override { return STRING_RESULT; }
-  Field *make_string_field(TABLE *table_arg) const override;
+  Field *make_string_field(TABLE *table_arg,
+                           MEM_ROOT *root = nullptr) const override;
   void clear() override;
   bool add() override;
   void reset_field() override { assert(0); }   // not used

diff --git a/sql/item_timefunc.h b/sql/item_timefunc.h
@@ -2,6 +2,7 @@
 #define ITEM_TIMEFUNC_INCLUDED
 
 /* Copyright (c) 2000, 2023, Oracle and/or its affiliates.
+   Copyright (c) 2021, Huawei Technologies Co., Ltd.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License, version 2.0,
@@ -21,7 +22,9 @@
 
    You should have received a copy of the GNU General Public License
    along with this program; if not, write to the Free Software
-   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA */
+   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA
+
+   Copyright (c) 2023, Shannon Data AI and/or its affiliates. */
 
 /* Function items used by mysql */