feat(shannon): Native Vector Embeddings Support

client/mysql.cc

@@ -1145,6 +1145,11 @@ static COMMANDS commands[] = {
     {"TO_DAYS", 0, nullptr, false, ""},
     {"TOUCHES", 0, nullptr, false, ""},
     {"TRIM", 0, nullptr, false, ""},
+    {"TO_VECTOR", 0, nullptr, false, ""},
+    {"STRING_TO_VECTOR", 0, nullptr, false, ""},
+    {"FROM_VECTOR", 0, nullptr, false, ""},
+    {"VECTOR_TO_STRING", 0, nullptr, false, ""},
+    {"VECTOR_DIM", 0, nullptr, false, ""},    
     {"UCASE", 0, nullptr, false, ""},
     {"UNCOMPRESS", 0, nullptr, false, ""},
     {"UNCOMPRESSED_LENGTH", 0, nullptr, false, ""},
@@ -3736,7 +3741,7 @@ static bool is_binary_field(MYSQL_FIELD *field) {
        field->type == MYSQL_TYPE_TINY_BLOB ||
        field->type == MYSQL_TYPE_VAR_STRING ||
        field->type == MYSQL_TYPE_STRING || field->type == MYSQL_TYPE_VARCHAR ||
-       field->type == MYSQL_TYPE_GEOMETRY))
+       field->type == MYSQL_TYPE_VECTOR || field->type == MYSQL_TYPE_GEOMETRY))
     return true;
   return false;
 }

client/mysqldump.cc

@@ -3938,6 +3938,7 @@ static void dump_table(char *table, char *db) {
                                         field->type == MYSQL_TYPE_VAR_STRING ||
                                         field->type == MYSQL_TYPE_VARCHAR ||
                                         field->type == MYSQL_TYPE_BLOB ||
+                                        field->type == MYSQL_TYPE_VECTOR ||
                                         field->type == MYSQL_TYPE_LONG_BLOB ||
                                         field->type == MYSQL_TYPE_MEDIUM_BLOB ||
                                         field->type == MYSQL_TYPE_TINY_BLOB ||

client/mysqltest.cc

@@ -8244,6 +8244,30 @@ static void append_field(DYNAMIC_STRING *ds, uint col_idx, MYSQL_FIELD *field,
   }
 #endif
 
+  const size_t temp_val_max_width = (1 << 14);
+  char temp_val[temp_val_max_width];
+  DYNAMIC_STRING ds_temp = {.str = nullptr, .length = 0, .max_length = 0};
+  if (field->type == MYSQL_TYPE_VECTOR && !is_null) {
+    /* Do a binary to hex conversion for vector type */
+    size_t orig_len = len;
+    len = 2 + orig_len * 2;
+    char *destination = temp_val;
+    if (len > temp_val_max_width) {
+      init_dynamic_string(&ds_temp, "", len + 1);
+      destination = ds_temp.str;
+    }
+    const char *ptr = val;
+    const char *end = ptr + orig_len;
+    val = destination;
+    int written = sprintf(destination, "0x");
+    destination += written;
+    for (; ptr < end; ptr++, destination += written) {
+      written = sprintf(
+          destination, "%02X",
+          *(static_cast<const uchar *>(static_cast<const void *>(ptr))));
+    }
+  }
+
   if (!display_result_vertically) {
     if (col_idx) dynstr_append_mem(ds, "\t", 1);
     replace_dynstr_append_mem(ds, val, len);
@@ -8253,6 +8277,10 @@ static void append_field(DYNAMIC_STRING *ds, uint col_idx, MYSQL_FIELD *field,
     replace_dynstr_append_mem(ds, val, len);
     dynstr_append_mem(ds, "\n", 1);
   }
+
+  if (ds_temp.str != nullptr) {
+    dynstr_free(&ds_temp);
+  }  
 }
 
 /*

include/field_types.h

@@ -76,8 +76,9 @@ enum enum_field_types {
   MYSQL_TYPE_DATETIME2,   /**< Internal to MySQL. Not used in protocol */
   MYSQL_TYPE_TIME2,       /**< Internal to MySQL. Not used in protocol */
   MYSQL_TYPE_TYPED_ARRAY, /**< Used for replication only */
-  MYSQL_TYPE_DB_TRX_ID = 242, /**Internal to MySQL. Not used in protocol */
-  MYSQL_TYPE_INVALID = 243,
+  MYSQL_TYPE_DB_TRX_ID = 241, /**Internal to MySQL. Not used in protocol */
+  MYSQL_TYPE_INVALID = 242,
+  MYSQL_TYPE_VECTOR = 243,
   MYSQL_TYPE_BOOL = 244, /**< Currently just a placeholder */
   MYSQL_TYPE_JSON = 245,
   MYSQL_TYPE_NEWDECIMAL = 246,

include/mysql.h.pp

@@ -23,8 +23,9 @@
   MYSQL_TYPE_DATETIME2,
   MYSQL_TYPE_TIME2,
   MYSQL_TYPE_TYPED_ARRAY,
-  MYSQL_TYPE_DB_TRX_ID =242,
-  MYSQL_TYPE_INVALID = 243,
+  MYSQL_TYPE_DB_TRX_ID =241,
+  MYSQL_TYPE_INVALID = 242,
+  MYSQL_TYPE_VECTOR = 243,
   MYSQL_TYPE_BOOL = 244,
   MYSQL_TYPE_JSON = 245,
   MYSQL_TYPE_NEWDECIMAL = 246,

include/mysql/components/services/bits/stored_program_bits.h

@@ -58,5 +58,6 @@
 #define MYSQL_SP_ARG_TYPE_VAR_STRING (1ULL << 31)
 #define MYSQL_SP_ARG_TYPE_STRING (1ULL << 32)
 #define MYSQL_SP_ARG_TYPE_GEOMETRY (1ULL << 33)
+#define MYSQL_SP_ARG_TYPE_VECTOR (1ULL << 34)
 
 #endif

libbinlogevents/include/rows_event.h

@@ -566,8 +566,9 @@ class Table_map_event : public Binary_log_event {
                                      columns, optimized to minimize
                                      space when many columns have the
                                      same charset. */
-    COLUMN_VISIBILITY             /* Flag to indicate column visibility
+    COLUMN_VISIBILITY,             /* Flag to indicate column visibility
                                      attribute. */
+    VECTOR_DIMENSIONALITY /* Vector column dimensionality */                                     
   };
 
   /**
@@ -604,6 +605,7 @@ class Table_map_event : public Binary_log_event {
     std::vector<str_vector> m_enum_str_value;
     std::vector<str_vector> m_set_str_value;
     std::vector<unsigned int> m_geometry_type;
+    std::vector<unsigned int> m_vector_dimensionality;
     /*
       The uint_pair means <column index, prefix length>.  Prefix length is 0 if
       whole column value is used.

libbinlogevents/src/binary_log_funcs.cpp

@@ -188,6 +188,7 @@ unsigned int max_display_length_for_field(enum_field_types sql_type,
        */
       return uint_max(metadata * 8);
 
+    case MYSQL_TYPE_VECTOR:
     case MYSQL_TYPE_LONG_BLOB:
     case MYSQL_TYPE_GEOMETRY:
     case MYSQL_TYPE_JSON:
@@ -346,6 +347,7 @@ uint32_t calc_field_size(unsigned char col, const unsigned char *master_data,
     case MYSQL_TYPE_MEDIUM_BLOB:
     case MYSQL_TYPE_LONG_BLOB:
     case MYSQL_TYPE_BLOB:
+    case MYSQL_TYPE_VECTOR:
     case MYSQL_TYPE_GEOMETRY:
     case MYSQL_TYPE_JSON: {
       /*

libbinlogevents/src/rows_event.cpp

@@ -252,6 +252,24 @@ static void parse_geometry_type(std::vector<unsigned int> &vec,
   }
 }
 
+/**
+   Parses VECTOR_DIMENSIONALITY field.
+   @param[out] vec        stores vector columns's dimensionality extracted from
+   field.
+   @param[in]  reader_obj the Event_reader object containing the serialized
+                          field.
+   @param[in]  length     length of the field
+ */
+static void parse_vector_dimensionality(std::vector<unsigned int> &vec,
+                                        Event_reader &reader_obj,
+                                        unsigned int length) {
+  const char *field = reader_obj.ptr();
+  while (reader_obj.ptr() < field + length) {
+    vec.push_back(reader_obj.net_field_length_ll());
+    if (reader_obj.has_error()) return;
+  }
+}
+
 /**
    Parses SIMPLE_PRIMARY_KEY field.
 
@@ -372,6 +390,9 @@ Table_map_event::Optional_metadata_fields::Optional_metadata_fields(
       case COLUMN_VISIBILITY:
         parse_column_visibility(&m_column_visibility, reader_obj, len);
         break;
+      case VECTOR_DIMENSIONALITY:
+        parse_vector_dimensionality(m_vector_dimensionality, reader_obj, len);
+        break;        
       default:
         BAPI_ASSERT(0);
     }

libmysql/libmysql.cc

@@ -2795,6 +2795,7 @@ static void fetch_string_with_conversion(MYSQL_BIND *param, char *value,
     case MYSQL_TYPE_MEDIUM_BLOB:
     case MYSQL_TYPE_LONG_BLOB:
     case MYSQL_TYPE_BLOB:
+    case MYSQL_TYPE_VECTOR:
     case MYSQL_TYPE_DECIMAL:
     case MYSQL_TYPE_NEWDECIMAL:
     default: {
@@ -3404,10 +3405,10 @@ static bool is_binary_compatible(enum enum_field_types type1,
       range2[] = {MYSQL_TYPE_INT24, MYSQL_TYPE_LONG, MYSQL_TYPE_NULL},
       range3[] = {MYSQL_TYPE_DATETIME, MYSQL_TYPE_TIMESTAMP, MYSQL_TYPE_NULL},
       range4[] = {
-          MYSQL_TYPE_ENUM,        MYSQL_TYPE_SET,       MYSQL_TYPE_TINY_BLOB,
-          MYSQL_TYPE_MEDIUM_BLOB, MYSQL_TYPE_LONG_BLOB, MYSQL_TYPE_BLOB,
-          MYSQL_TYPE_VAR_STRING,  MYSQL_TYPE_STRING,    MYSQL_TYPE_GEOMETRY,
-          MYSQL_TYPE_DECIMAL,     MYSQL_TYPE_NULL};
+          MYSQL_TYPE_ENUM,        MYSQL_TYPE_SET,        MYSQL_TYPE_TINY_BLOB,
+          MYSQL_TYPE_MEDIUM_BLOB, MYSQL_TYPE_LONG_BLOB,  MYSQL_TYPE_BLOB,
+          MYSQL_TYPE_VECTOR,      MYSQL_TYPE_VAR_STRING, MYSQL_TYPE_STRING,
+          MYSQL_TYPE_GEOMETRY,    MYSQL_TYPE_DECIMAL,    MYSQL_TYPE_NULL};
   static const enum enum_field_types *range_list[] = {range1, range2, range3,
                                                       range4},
                                      **range_list_end =
@@ -3508,6 +3509,7 @@ static bool setup_one_fetch_function(MYSQL_BIND *param, MYSQL_FIELD *field) {
     case MYSQL_TYPE_MEDIUM_BLOB:
     case MYSQL_TYPE_LONG_BLOB:
     case MYSQL_TYPE_BLOB:
+    case MYSQL_TYPE_VECTOR:
     case MYSQL_TYPE_BIT:
       assert(param->buffer_length != 0);
       param->fetch_result = fetch_result_bin;

mysql-test/r/archive.result

@@ -12975,5 +12975,14 @@ id	x
 5	6
 DROP VIEW v;
 DROP TABLE archive_table, innodb_table;
+CREATE TABLE ta (pk INT, embedding VECTOR(4)) ENGINE=ARCHIVE;
+INSERT INTO ta VALUES
+(0, TO_VECTOR("[1,2,3,4]")),
+(1, TO_VECTOR("[4,5,6,7]"));
+SELECT FROM_VECTOR(embedding) FROM ta;
+FROM_VECTOR(embedding)
+[1.00000e+00,2.00000e+00,3.00000e+00,4.00000e+00]
+[4.00000e+00,5.00000e+00,6.00000e+00,7.00000e+00]
+DROP TABLE ta;
 Warnings:
 Warning	1287	'@@binlog_format' is deprecated and will be removed in a future release.

mysql-test/r/blackhole.result

@@ -28,3 +28,10 @@ score
 SELECT 1 FROM t WHERE MATCH (a) AGAINST ('abc');
 1
 DROP TABLE t;
+CREATE TABLE ta (pk INT, embedding VECTOR(4), PRIMARY KEY (pk)) ENGINE=BLACKHOLE;
+INSERT INTO ta VALUES
+(0, TO_VECTOR("[1,2,3,4]")),
+(1, TO_VECTOR("[4,5,6,7]"));
+SELECT FROM_VECTOR(embedding) FROM ta;
+FROM_VECTOR(embedding)
+DROP TABLE ta;

mysql-test/r/csv.result

@@ -5496,3 +5496,12 @@ SELECT * FROM t;
 j
 {"a": 1, "b": 2}
 DROP TABLE t;
+CREATE TABLE ta (pk INT NOT NULL, embedding VECTOR(4) NOT NULL) ENGINE=csv;
+INSERT INTO ta VALUES
+(0, TO_VECTOR("[1,2,3,4]")),
+(1, TO_VECTOR("[4,5,6,7]"));
+SELECT FROM_VECTOR(embedding) FROM ta;
+FROM_VECTOR(embedding)
+[1.00000e+00,2.00000e+00,3.00000e+00,4.00000e+00]
+[4.00000e+00,5.00000e+00,6.00000e+00,7.00000e+00]
+DROP TABLE ta;

mysql-test/r/information_schema_parameters.result

@@ -2,7 +2,7 @@
 USE INFORMATION_SCHEMA;
 SHOW CREATE TABLE INFORMATION_SCHEMA.PARAMETERS;
 View	Create View	character_set_client	collation_connection
-PARAMETERS	CREATE ALGORITHM=UNDEFINED DEFINER=`mysql.infoschema`@`localhost` SQL SECURITY DEFINER VIEW `PARAMETERS` AS select `cat`.`name` AS `SPECIFIC_CATALOG`,`sch`.`name` AS `SPECIFIC_SCHEMA`,`rtn`.`name` AS `SPECIFIC_NAME`,if((`rtn`.`type` = 'FUNCTION'),(`prm`.`ordinal_position` - 1),`prm`.`ordinal_position`) AS `ORDINAL_POSITION`,if(((`rtn`.`type` = 'FUNCTION') and (`prm`.`ordinal_position` = 1)),NULL,`prm`.`mode`) AS `PARAMETER_MODE`,if(((`rtn`.`type` = 'FUNCTION') and (`prm`.`ordinal_position` = 1)),NULL,`prm`.`name`) AS `PARAMETER_NAME`,substring_index(substring_index(`prm`.`data_type_utf8`,'(',1),' ',1) AS `DATA_TYPE`,internal_dd_char_length(`prm`.`data_type`,`prm`.`char_length`,`col`.`name`,0) AS `CHARACTER_MAXIMUM_LENGTH`,internal_dd_char_length(`prm`.`data_type`,`prm`.`char_length`,`col`.`name`,1) AS `CHARACTER_OCTET_LENGTH`,`prm`.`numeric_precision` AS `NUMERIC_PRECISION`,if((`prm`.`numeric_precision` is null),NULL,ifnull(`prm`.`numeric_scale`,0)) AS `NUMERIC_SCALE`,`prm`.`datetime_precision` AS `DATETIME_PRECISION`,(case `prm`.`data_type` when 'MYSQL_TYPE_STRING' then if((`cs`.`name` = 'binary'),NULL,`cs`.`name`) when 'MYSQL_TYPE_VAR_STRING' then if((`cs`.`name` = 'binary'),NULL,`cs`.`name`) when 'MYSQL_TYPE_VARCHAR' then if((`cs`.`name` = 'binary'),NULL,`cs`.`name`) when 'MYSQL_TYPE_TINY_BLOB' then if((`cs`.`name` = 'binary'),NULL,`cs`.`name`) when 'MYSQL_TYPE_MEDIUM_BLOB' then if((`cs`.`name` = 'binary'),NULL,`cs`.`name`) when 'MYSQL_TYPE_BLOB' then if((`cs`.`name` = 'binary'),NULL,`cs`.`name`) when 'MYSQL_TYPE_LONG_BLOB' then if((`cs`.`name` = 'binary'),NULL,`cs`.`name`) when 'MYSQL_TYPE_ENUM' then if((`cs`.`name` = 'binary'),NULL,`cs`.`name`) when 'MYSQL_TYPE_SET' then if((`cs`.`name` = 'binary'),NULL,`cs`.`name`) else NULL end) AS `CHARACTER_SET_NAME`,(case `prm`.`data_type` when 'MYSQL_TYPE_STRING' then if((`cs`.`name` = 'binary'),NULL,`col`.`name`) when 'MYSQL_TYPE_VAR_STRING' then if((`cs`.`name` = 'binary'),NULL,`col`.`name`) when 'MYSQL_TYPE_VARCHAR' then if((`cs`.`name` = 'binary'),NULL,`col`.`name`) when 'MYSQL_TYPE_TINY_BLOB' then if((`cs`.`name` = 'binary'),NULL,`col`.`name`) when 'MYSQL_TYPE_MEDIUM_BLOB' then if((`cs`.`name` = 'binary'),NULL,`col`.`name`) when 'MYSQL_TYPE_BLOB' then if((`cs`.`name` = 'binary'),NULL,`col`.`name`) when 'MYSQL_TYPE_LONG_BLOB' then if((`cs`.`name` = 'binary'),NULL,`col`.`name`) when 'MYSQL_TYPE_ENUM' then if((`cs`.`name` = 'binary'),NULL,`col`.`name`) when 'MYSQL_TYPE_SET' then if((`cs`.`name` = 'binary'),NULL,`col`.`name`) else NULL end) AS `COLLATION_NAME`,`prm`.`data_type_utf8` AS `DTD_IDENTIFIER`,`rtn`.`type` AS `ROUTINE_TYPE` from (((((`mysql`.`parameters` `prm` join `mysql`.`routines` `rtn` on((`prm`.`routine_id` = `rtn`.`id`))) join `mysql`.`schemata` `sch` on((`rtn`.`schema_id` = `sch`.`id`))) join `mysql`.`catalogs` `cat` on((`cat`.`id` = `sch`.`catalog_id`))) join `mysql`.`collations` `col` on((`prm`.`collation_id` = `col`.`id`))) join `mysql`.`character_sets` `cs` on((`col`.`character_set_id` = `cs`.`id`))) where (0 <> can_access_routine(`sch`.`name`,`rtn`.`name`,`rtn`.`type`,`rtn`.`definer`,false))	utf8mb3	utf8mb3_general_ci
+PARAMETERS	CREATE ALGORITHM=UNDEFINED DEFINER=`mysql.infoschema`@`localhost` SQL SECURITY DEFINER VIEW `PARAMETERS` AS select `cat`.`name` AS `SPECIFIC_CATALOG`,`sch`.`name` AS `SPECIFIC_SCHEMA`,`rtn`.`name` AS `SPECIFIC_NAME`,if((`rtn`.`type` = 'FUNCTION'),(`prm`.`ordinal_position` - 1),`prm`.`ordinal_position`) AS `ORDINAL_POSITION`,if(((`rtn`.`type` = 'FUNCTION') and (`prm`.`ordinal_position` = 1)),NULL,`prm`.`mode`) AS `PARAMETER_MODE`,if(((`rtn`.`type` = 'FUNCTION') and (`prm`.`ordinal_position` = 1)),NULL,`prm`.`name`) AS `PARAMETER_NAME`,substring_index(substring_index(`prm`.`data_type_utf8`,'(',1),' ',1) AS `DATA_TYPE`,internal_dd_char_length(`prm`.`data_type`,`prm`.`char_length`,`col`.`name`,0) AS `CHARACTER_MAXIMUM_LENGTH`,internal_dd_char_length(`prm`.`data_type`,`prm`.`char_length`,`col`.`name`,1) AS `CHARACTER_OCTET_LENGTH`,`prm`.`numeric_precision` AS `NUMERIC_PRECISION`,if((`prm`.`numeric_precision` is null),NULL,ifnull(`prm`.`numeric_scale`,0)) AS `NUMERIC_SCALE`,`prm`.`datetime_precision` AS `DATETIME_PRECISION`,(case `prm`.`data_type` when 'MYSQL_TYPE_STRING' then if((`cs`.`name` = 'binary'),NULL,`cs`.`name`) when 'MYSQL_TYPE_VAR_STRING' then if((`cs`.`name` = 'binary'),NULL,`cs`.`name`) when 'MYSQL_TYPE_VARCHAR' then if((`cs`.`name` = 'binary'),NULL,`cs`.`name`) when 'MYSQL_TYPE_TINY_BLOB' then if((`cs`.`name` = 'binary'),NULL,`cs`.`name`) when 'MYSQL_TYPE_MEDIUM_BLOB' then if((`cs`.`name` = 'binary'),NULL,`cs`.`name`) when 'MYSQL_TYPE_BLOB' then if((`cs`.`name` = 'binary'),NULL,`cs`.`name`) when 'MYSQL_TYPE_VECTOR' then if((`cs`.`name` = 'binary'),NULL,`cs`.`name`) when 'MYSQL_TYPE_LONG_BLOB' then if((`cs`.`name` = 'binary'),NULL,`cs`.`name`) when 'MYSQL_TYPE_ENUM' then if((`cs`.`name` = 'binary'),NULL,`cs`.`name`) when 'MYSQL_TYPE_SET' then if((`cs`.`name` = 'binary'),NULL,`cs`.`name`) else NULL end) AS `CHARACTER_SET_NAME`,(case `prm`.`data_type` when 'MYSQL_TYPE_STRING' then if((`cs`.`name` = 'binary'),NULL,`col`.`name`) when 'MYSQL_TYPE_VAR_STRING' then if((`cs`.`name` = 'binary'),NULL,`col`.`name`) when 'MYSQL_TYPE_VARCHAR' then if((`cs`.`name` = 'binary'),NULL,`col`.`name`) when 'MYSQL_TYPE_TINY_BLOB' then if((`cs`.`name` = 'binary'),NULL,`col`.`name`) when 'MYSQL_TYPE_VECTOR' then if((`cs`.`name` = 'binary'),NULL,`col`.`name`) when 'MYSQL_TYPE_MEDIUM_BLOB' then if((`cs`.`name` = 'binary'),NULL,`col`.`name`) when 'MYSQL_TYPE_BLOB' then if((`cs`.`name` = 'binary'),NULL,`col`.`name`) when 'MYSQL_TYPE_LONG_BLOB' then if((`cs`.`name` = 'binary'),NULL,`col`.`name`) when 'MYSQL_TYPE_ENUM' then if((`cs`.`name` = 'binary'),NULL,`col`.`name`) when 'MYSQL_TYPE_SET' then if((`cs`.`name` = 'binary'),NULL,`col`.`name`) else NULL end) AS `COLLATION_NAME`,`prm`.`data_type_utf8` AS `DTD_IDENTIFIER`,`rtn`.`type` AS `ROUTINE_TYPE` from (((((`mysql`.`parameters` `prm` join `mysql`.`routines` `rtn` on((`prm`.`routine_id` = `rtn`.`id`))) join `mysql`.`schemata` `sch` on((`rtn`.`schema_id` = `sch`.`id`))) join `mysql`.`catalogs` `cat` on((`cat`.`id` = `sch`.`catalog_id`))) join `mysql`.`collations` `col` on((`prm`.`collation_id` = `col`.`id`))) join `mysql`.`character_sets` `cs` on((`col`.`character_set_id` = `cs`.`id`))) where (0 <> can_access_routine(`sch`.`name`,`rtn`.`name`,`rtn`.`type`,`rtn`.`definer`,false))	utf8mb3	utf8mb3_general_ci
 SELECT * FROM information_schema.columns
 WHERE table_schema = 'information_schema'
   AND table_name   = 'PARAMETERS'