From 6579df9c1ba4718125022450930e11369f57199a Mon Sep 17 00:00:00 2001 From: "vitess-bot[bot]" <108069721+vitess-bot[bot]@users.noreply.github.com> Date: Thu, 26 Dec 2024 11:54:08 +0100 Subject: [PATCH] Cherry-pick 524dec934f4423f728ab53df0e3698e9320bf3c6 with conflicts --- .../vreplication/lookup_vindex_helper_test.go | 113 ++++++++++ .../vreplication/lookup_vindex_test.go | 204 ++++++++++++++++++ 2 files changed, 317 insertions(+) create mode 100644 go/test/endtoend/vreplication/lookup_vindex_helper_test.go create mode 100644 go/test/endtoend/vreplication/lookup_vindex_test.go diff --git a/go/test/endtoend/vreplication/lookup_vindex_helper_test.go b/go/test/endtoend/vreplication/lookup_vindex_helper_test.go new file mode 100644 index 00000000000..1c74dadc642 --- /dev/null +++ b/go/test/endtoend/vreplication/lookup_vindex_helper_test.go @@ -0,0 +1,113 @@ +/* +Copyright 2024 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vreplication + +import ( + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" + + "vitess.io/vitess/go/vt/sqlparser" + + binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" +) + +type lookupVindex struct { + typ string + name string + tableKeyspace string + table string + columns []string + ownerTable string + ownerTableKeyspace string + ignoreNulls bool + + t *testing.T +} + +func (lv *lookupVindex) String() string { + return lv.typ + " " + lv.name + " on " + lv.tableKeyspace + "." + lv.table + " (" + lv.columns[0] + ")" +} + +func (lv *lookupVindex) create() { + cols := strings.Join(lv.columns, ",") + args := []string{ + "LookupVindex", + "--name", lv.name, + "--table-keyspace=" + lv.ownerTableKeyspace, + "create", + "--keyspace=" + lv.tableKeyspace, + "--type=" + lv.typ, + "--table-owner=" + lv.ownerTable, + "--table-owner-columns=" + cols, + "--tablet-types=PRIMARY", + } + if lv.ignoreNulls { + args = append(args, "--ignore-nulls") + } + + err := vc.VtctldClient.ExecuteCommand(args...) + require.NoError(lv.t, err, "error executing LookupVindex create: %v", err) + waitForWorkflowState(lv.t, vc, fmt.Sprintf("%s.%s", lv.ownerTableKeyspace, lv.name), binlogdatapb.VReplicationWorkflowState_Running.String()) + lv.expectWriteOnly(true) +} + +func (lv *lookupVindex) cancel() { + panic("not implemented") +} + +func (lv *lookupVindex) externalize() { + args := []string{ + "LookupVindex", + "--name", lv.name, + "--table-keyspace=" + lv.ownerTableKeyspace, + "externalize", + "--keyspace=" + lv.tableKeyspace, + } + err := vc.VtctldClient.ExecuteCommand(args...) + require.NoError(lv.t, err, "error executing LookupVindex externalize: %v", err) + lv.expectWriteOnly(false) +} + +func (lv *lookupVindex) show() error { + return nil +} + +func (lv *lookupVindex) expectWriteOnly(expected bool) { + vschema, err := vc.VtctldClient.ExecuteCommandWithOutput("GetVSchema", lv.ownerTableKeyspace) + require.NoError(lv.t, err, "error executing GetVSchema: %v", err) + vdx := gjson.Get(vschema, fmt.Sprintf("vindexes.%s", lv.name)) + require.NotNil(lv.t, vdx, "lookup vindex %s not found", lv.name) + want := "" + if expected { + want = "true" + } + require.Equal(lv.t, want, vdx.Get("params.write_only").String(), "expected write_only parameter to be %s", want) +} + +func getNumRowsInQuery(t *testing.T, query string) int { + stmt, err := sqlparser.NewTestParser().Parse(query) + require.NoError(t, err) + insertStmt, ok := stmt.(*sqlparser.Insert) + require.True(t, ok) + rows, ok := insertStmt.Rows.(sqlparser.Values) + require.True(t, ok) + return len(rows) +} diff --git a/go/test/endtoend/vreplication/lookup_vindex_test.go b/go/test/endtoend/vreplication/lookup_vindex_test.go new file mode 100644 index 00000000000..c0864b26cca --- /dev/null +++ b/go/test/endtoend/vreplication/lookup_vindex_test.go @@ -0,0 +1,204 @@ +/* +Copyright 2024 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vreplication + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/test/endtoend/cluster" + vttablet "vitess.io/vitess/go/vt/vttablet/common" +) + +type TestClusterSpec struct { + keyspaceName string + vschema string + schema string +} + +var lookupClusterSpec = TestClusterSpec{ + keyspaceName: "lookup", + vschema: ` +{ + "sharded": true, + "vindexes": { + "reverse_bits": { + "type": "reverse_bits" + } + }, + "tables": { + "t1": { + "column_vindexes": [ + { + "column": "c1", + "name": "reverse_bits" + } + ] + } + } +} +`, + schema: ` +create table t1( + c1 int, + c2 int, + val varchar(128), + primary key(c1) +); +`, +} + +func setupLookupVindexKeyspace(t *testing.T) map[string]*cluster.VttabletProcess { + tablets := make(map[string]*cluster.VttabletProcess) + if _, err := vc.AddKeyspace(t, []*Cell{vc.Cells["zone1"]}, lookupClusterSpec.keyspaceName, "-80,80-", + lookupClusterSpec.vschema, lookupClusterSpec.schema, defaultReplicas, defaultRdonly, 200, nil); err != nil { + require.NoError(t, err) + } + defaultCell := vc.Cells[vc.CellNames[0]] + ks := vc.Cells[defaultCell.Name].Keyspaces[lookupClusterSpec.keyspaceName] + targetTab1 = ks.Shards["-80"].Tablets["zone1-200"].Vttablet + targetTab2 = ks.Shards["80-"].Tablets["zone1-300"].Vttablet + tablets["-80"] = targetTab1 + tablets["80-"] = targetTab2 + return tablets +} + +type lookupTestCase struct { + name string + lv *lookupVindex + initQuery string + runningQuery string + postExternalizeQuery string + cleanupQuery string +} + +func TestLookupVindex(t *testing.T) { + setSidecarDBName("_vt") + origDefaultReplicas := defaultReplicas + origDefaultRdonly := defaultRdonly + defer func() { + defaultReplicas = origDefaultReplicas + defaultRdonly = origDefaultRdonly + }() + defaultReplicas = 1 + defaultRdonly = 0 + vc = setupMinimalCluster(t) + defer vc.TearDown() + vttablet.InitVReplicationConfigDefaults() + + _ = setupLookupVindexKeyspace(t) + + initQuery := "insert into t1 (c1, c2, val) values (1, 1, 'val1'), (2, 2, 'val2'), (3, 3, 'val3')" + runningQuery := "insert into t1 (c1, c2, val) values (4, 4, 'val4'), (5, 5, 'val5'), (6, 6, 'val6')" + postExternalizeQuery := "insert into t1 (c1, c2, val) values (7, 7, 'val7'), (8, 8, 'val8'), (9, 9, 'val9')" + cleanupQuery := "delete from t1" + + testCases := []lookupTestCase{ + { + name: "non-unique lookup index, one column", + lv: &lookupVindex{ + typ: "consistent_lookup", + name: "t1_c2_lookup", + tableKeyspace: lookupClusterSpec.keyspaceName, + table: "t1", + columns: []string{"c2"}, + ownerTable: "t1", + ownerTableKeyspace: lookupClusterSpec.keyspaceName, + ignoreNulls: true, + t: t, + }, + }, + { + name: "lookup index, two columns", + lv: &lookupVindex{ + typ: "lookup", + name: "t1_c2_val_lookup", + tableKeyspace: lookupClusterSpec.keyspaceName, + table: "t1", + columns: []string{"c2", "val"}, + ownerTable: "t1", + ownerTableKeyspace: lookupClusterSpec.keyspaceName, + ignoreNulls: true, + t: t, + }, + }, + { + name: "unique lookup index, one column", + lv: &lookupVindex{ + typ: "lookup_unique", + name: "t1_c2_unique_lookup", + tableKeyspace: lookupClusterSpec.keyspaceName, + table: "t1", + columns: []string{"c2"}, + ownerTable: "t1", + ownerTableKeyspace: lookupClusterSpec.keyspaceName, + ignoreNulls: true, + t: t, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + tc.initQuery = initQuery + tc.runningQuery = runningQuery + tc.postExternalizeQuery = postExternalizeQuery + tc.cleanupQuery = cleanupQuery + testLookupVindex(t, &tc) + }) + } +} + +func testLookupVindex(t *testing.T, tc *lookupTestCase) { + vtgateConn, cancel := getVTGateConn() + defer cancel() + var totalRows int + lv := tc.lv + + t.Run("init data", func(t *testing.T) { + totalRows += getNumRowsInQuery(t, tc.initQuery) + _, err := vtgateConn.ExecuteFetch(tc.initQuery, 1000, false) + require.NoError(t, err) + }) + + t.Run("create", func(t *testing.T) { + tc.lv.create() + + lks := lv.tableKeyspace + vindexName := lv.name + waitForRowCount(t, vtgateConn, lks, vindexName, totalRows) + totalRows += getNumRowsInQuery(t, tc.runningQuery) + _, err := vtgateConn.ExecuteFetch(tc.runningQuery, 1000, false) + require.NoError(t, err) + waitForRowCount(t, vtgateConn, tc.lv.ownerTableKeyspace, lv.name, totalRows) + }) + + t.Run("externalize", func(t *testing.T) { + tc.lv.externalize() + totalRows += getNumRowsInQuery(t, tc.postExternalizeQuery) + _, err := vtgateConn.ExecuteFetch(tc.postExternalizeQuery, 1000, false) + require.NoError(t, err) + waitForRowCount(t, vtgateConn, tc.lv.ownerTableKeyspace, lv.name, totalRows) + }) + + t.Run("cleanup", func(t *testing.T) { + _, err := vtgateConn.ExecuteFetch(tc.cleanupQuery, 1000, false) + require.NoError(t, err) + waitForRowCount(t, vtgateConn, tc.lv.ownerTableKeyspace, lv.name, 0) + }) +}