Skip to content

Commit

Permalink
Move all e2e tests to vtctldclient
Browse files Browse the repository at this point in the history 
Signed-off-by: Matt Lord <[email protected]>
  • Loading branch information
@mattlord
mattlord committed Dec 26, 2024
1 parent 30e1e40 commit 58b309e
Show file tree
Hide file tree
Showing 30 changed files with 289 additions and 619 deletions.
35 changes: 34 additions & 1 deletion go/cmd/vtctldclient/command/topology.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"github.com/spf13/cobra"

"vitess.io/vitess/go/cmd/vtctldclient/cli"
"vitess.io/vitess/go/vt/topo"

vtctldatapb "vitess.io/vitess/go/vt/proto/vtctldata"
)
Expand All @@ -35,12 +36,22 @@ var (
Args: cobra.ExactArgs(1),
RunE: commandGetTopologyPath,
}

// The version of the key/path to get. If not specified, the latest/current
// version is returned.
version int64 = 0
// If true, only the data is output and it is in JSON format rather than prototext.
dataAsJSON bool = false

// WriteTopologyPath makes a WriteTopologyPath gRPC call to a vtctld.
WriteTopologyPath = &cobra.Command{
Use: "WriteTopologyPath <path> <file>",
Short: "Copies a local file structure to the topology server at the given path.",
DisableFlagsInUseLine: true,
Args: cobra.ExactArgs(2),
RunE: commandWriteTopologyPath,
}
// The cell to use for the copy. Defaults to global cell.
cell string
)

func commandGetTopologyPath(cmd *cobra.Command, args []string) error {
Expand Down Expand Up @@ -75,8 +86,30 @@ func commandGetTopologyPath(cmd *cobra.Command, args []string) error {
return nil
}

func commandWriteTopologyPath(cmd *cobra.Command, args []string) error {
/*
path := cmd.Flags().Arg(0)
file := cmd.Flags().Arg(1)
conn, err := ts.ConnForCell(cmd.Context(), cell)
if err != nil {
return err
}
data, err := os.ReadFile(file)
if err != nil {
return err
}
_, err = conn.Update(cmd.Context(), path, data, nil)
return err
*/
return nil
}

func init() {
GetTopologyPath.Flags().Int64Var(&version, "version", version, "The version of the path's key to get. If not specified, the latest version is returned.")
GetTopologyPath.Flags().BoolVar(&dataAsJSON, "data-as-json", dataAsJSON, "If true, only the data is output and it is in JSON format rather than prototext.")
Root.AddCommand(GetTopologyPath)

WriteTopologyPath.Flags().StringVar(&cell, "cell", topo.GlobalCell, "Topology server cell to copy the file to.")
Root.AddCommand(WriteTopologyPath)
}
64 changes: 32 additions & 32 deletions go/test/endtoend/keyspace/keyspace_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -277,24 +277,24 @@ func TestDeleteKeyspace(t *testing.T) {
// TODO: (ajm188) if this test gets fixed, the flags need to be updated to comply with VEP-4 as well.
// tells that in zone2 after deleting shard, there is no shard #264 and in zone1 there is only 1 #269
/*func RemoveKeyspaceCell(t *testing.T) {
_ = clusterForKSTest.VtctlclientProcess.ExecuteCommand("CreateKeyspace", "test_delete_keyspace_removekscell")
_ = clusterForKSTest.VtctlclientProcess.ExecuteCommand("CreateShard", "test_delete_keyspace_removekscell/0")
_ = clusterForKSTest.VtctlclientProcess.ExecuteCommand("CreateShard", "test_delete_keyspace_removekscell/1")
_ = clusterForKSTest.VtctlclientProcess.ExecuteCommand("InitTablet", "--port=1234", "--bind-address=127.0.0.1", "-keyspace=test_delete_keyspace_removekscell", "--shard=0", "zone1-0000000100", "primary")
_ = clusterForKSTest.VtctlclientProcess.ExecuteCommand("InitTablet", "--port=1234", "--bind-address=127.0.0.1", "-keyspace=test_delete_keyspace_removekscell", "--shard=1", "zone1-0000000101", "primary")
_ = clusterForKSTest.VtctlclientProcess.ExecuteCommand("InitTablet", "--port=1234", "--bind-address=127.0.0.1", "-keyspace=test_delete_keyspace_removekscell", "--shard=0", "zone2-0000000100", "replica")
_ = clusterForKSTest.VtctlclientProcess.ExecuteCommand("InitTablet", "--port=1234", "--bind-address=127.0.0.1", "-keyspace=test_delete_keyspace_removekscell", "--shard=1", "zone2-0000000101", "replica")
_ = clusterForKSTest.VtctldClientProcess.ExecuteCommand("CreateKeyspace", "test_delete_keyspace_removekscell")
_ = clusterForKSTest.VtctldClientProcess.ExecuteCommand("CreateShard", "test_delete_keyspace_removekscell/0")
_ = clusterForKSTest.VtctldClientProcess.ExecuteCommand("CreateShard", "test_delete_keyspace_removekscell/1")
_ = clusterForKSTest.VtctldClientProcess.ExecuteCommand("InitTablet", "--port=1234", "--bind-address=127.0.0.1", "-keyspace=test_delete_keyspace_removekscell", "--shard=0", "zone1-0000000100", "primary")
_ = clusterForKSTest.VtctldClientProcess.ExecuteCommand("InitTablet", "--port=1234", "--bind-address=127.0.0.1", "-keyspace=test_delete_keyspace_removekscell", "--shard=1", "zone1-0000000101", "primary")
_ = clusterForKSTest.VtctldClientProcess.ExecuteCommand("InitTablet", "--port=1234", "--bind-address=127.0.0.1", "-keyspace=test_delete_keyspace_removekscell", "--shard=0", "zone2-0000000100", "replica")
_ = clusterForKSTest.VtctldClientProcess.ExecuteCommand("InitTablet", "--port=1234", "--bind-address=127.0.0.1", "-keyspace=test_delete_keyspace_removekscell", "--shard=1", "zone2-0000000101", "replica")
// Create the serving/replication entries and check that they exist, so we can later check they're deleted.
_ = clusterForKSTest.VtctlclientProcess.ExecuteCommand("RebuildKeyspaceGraph", "test_delete_keyspace_removekscell")
_ = clusterForKSTest.VtctlclientProcess.ExecuteCommand("GetShardReplication", "zone2", "test_delete_keyspace_removekscell/0")
_ = clusterForKSTest.VtctlclientProcess.ExecuteCommand("GetShardReplication", "zone2", "test_delete_keyspace_removekscell/1")
_ = clusterForKSTest.VtctlclientProcess.ExecuteCommand("GetSrvKeyspace", "zone2", "test_delete_keyspace_removekscell")
_ = clusterForKSTest.VtctlclientProcess.ExecuteCommand("GetSrvKeyspace", "zone1", "test_delete_keyspace_removekscell")
_ = clusterForKSTest.VtctldClientProcess.ExecuteCommand("RebuildKeyspaceGraph", "test_delete_keyspace_removekscell")
_ = clusterForKSTest.VtctldClientProcess.ExecuteCommand("GetShardReplication", "zone2", "test_delete_keyspace_removekscell/0")
_ = clusterForKSTest.VtctldClientProcess.ExecuteCommand("GetShardReplication", "zone2", "test_delete_keyspace_removekscell/1")
_ = clusterForKSTest.VtctldClientProcess.ExecuteCommand("GetSrvKeyspace", "zone2", "test_delete_keyspace_removekscell")
_ = clusterForKSTest.VtctldClientProcess.ExecuteCommand("GetSrvKeyspace", "zone1", "test_delete_keyspace_removekscell")
// Just remove the shard from one cell (including tablets),
// but leaving the global records and other cells/shards alone.
_ = clusterForKSTest.VtctlclientProcess.ExecuteCommand("RemoveShardCell", "--recursive", "test_delete_keyspace_removekscell/0", "zone2")
_ = clusterForKSTest.VtctldClientProcess.ExecuteCommand("RemoveShardCell", "--recursive", "test_delete_keyspace_removekscell/0", "zone2")
//Check that the shard is gone from zone2.
srvKeyspaceZone2 := getSrvKeyspace(t, cell2, "test_delete_keyspace_removekscell")
Expand All @@ -308,42 +308,42 @@ func TestDeleteKeyspace(t *testing.T) {
assert.Equal(t, len(partition.ShardReferences), 2)
}
_ = clusterForKSTest.VtctlclientProcess.ExecuteCommand("RebuildKeyspaceGraph", "test_delete_keyspace_removekscell")
_ = clusterForKSTest.VtctlclientProcess.ExecuteCommand("GetKeyspace", "test_delete_keyspace_removekscell")
_ = clusterForKSTest.VtctlclientProcess.ExecuteCommand("GetShard", "test_delete_keyspace_removekscell/0")
_ = clusterForKSTest.VtctldClientProcess.ExecuteCommand("RebuildKeyspaceGraph", "test_delete_keyspace_removekscell")
_ = clusterForKSTest.VtctldClientProcess.ExecuteCommand("GetKeyspace", "test_delete_keyspace_removekscell")
_ = clusterForKSTest.VtctldClientProcess.ExecuteCommand("GetShard", "test_delete_keyspace_removekscell/0")
_ = clusterForKSTest.VtctlclientProcess.ExecuteCommand("GetTablet", "zone1-0000000100")
_ = clusterForKSTest.VtctldClientProcess.ExecuteCommand("GetTablet", "zone1-0000000100")
err := clusterForKSTest.VtctlclientProcess.ExecuteCommand("GetTablet", "zone2-0000000100")
err := clusterForKSTest.VtctldClientProcess.ExecuteCommand("GetTablet", "zone2-0000000100")
require.Error(t, err)
_ = clusterForKSTest.VtctlclientProcess.ExecuteCommand("GetTablet", "zone2-0000000101")
_ = clusterForKSTest.VtctlclientProcess.ExecuteCommand("GetShardReplication", "zone1", "test_delete_keyspace_removekscell/0")
_ = clusterForKSTest.VtctldClientProcess.ExecuteCommand("GetTablet", "zone2-0000000101")
_ = clusterForKSTest.VtctldClientProcess.ExecuteCommand("GetShardReplication", "zone1", "test_delete_keyspace_removekscell/0")
err = clusterForKSTest.VtctlclientProcess.ExecuteCommand("GetShardReplication", "zone2", "test_delete_keyspace_removekscell/0")
err = clusterForKSTest.VtctldClientProcess.ExecuteCommand("GetShardReplication", "zone2", "test_delete_keyspace_removekscell/0")
require.Error(t, err)
_ = clusterForKSTest.VtctlclientProcess.ExecuteCommand("GetShardReplication", "zone2", "test_delete_keyspace_removekscell/1")
_ = clusterForKSTest.VtctlclientProcess.ExecuteCommand("GetSrvKeyspace", "zone2", "test_delete_keyspace_removekscell")
_ = clusterForKSTest.VtctldClientProcess.ExecuteCommand("GetShardReplication", "zone2", "test_delete_keyspace_removekscell/1")
_ = clusterForKSTest.VtctldClientProcess.ExecuteCommand("GetSrvKeyspace", "zone2", "test_delete_keyspace_removekscell")
// Add it back to do another test.
_ = clusterForKSTest.VtctlclientProcess.ExecuteCommand("InitTablet", "--port=1234", "--keyspace=test_delete_keyspace_removekscell", "--shard=0", "zone2-0000000100", "replica")
_ = clusterForKSTest.VtctlclientProcess.ExecuteCommand("RebuildKeyspaceGraph", "test_delete_keyspace_removekscell")
_ = clusterForKSTest.VtctlclientProcess.ExecuteCommand("GetShardReplication", "zone2", "test_delete_keyspace_removekscell/0")
_ = clusterForKSTest.VtctldClientProcess.ExecuteCommand("InitTablet", "--port=1234", "--keyspace=test_delete_keyspace_removekscell", "--shard=0", "zone2-0000000100", "replica")
_ = clusterForKSTest.VtctldClientProcess.ExecuteCommand("RebuildKeyspaceGraph", "test_delete_keyspace_removekscell")
_ = clusterForKSTest.VtctldClientProcess.ExecuteCommand("GetShardReplication", "zone2", "test_delete_keyspace_removekscell/0")
// Now use RemoveKeyspaceCell to remove all shards.
_ = clusterForKSTest.VtctlclientProcess.ExecuteCommand("RemoveKeyspaceCell", "-recursive", "test_delete_keyspace_removekscell", "zone2")
_ = clusterForKSTest.VtctlclientProcess.ExecuteCommand("RebuildKeyspaceGraph", "test_delete_keyspace_removekscell")
_ = clusterForKSTest.VtctlclientProcess.ExecuteCommand("GetShardReplication", "zone1", "test_delete_keyspace_removekscell/0")
_ = clusterForKSTest.VtctldClientProcess.ExecuteCommand("RemoveKeyspaceCell", "-recursive", "test_delete_keyspace_removekscell", "zone2")
_ = clusterForKSTest.VtctldClientProcess.ExecuteCommand("RebuildKeyspaceGraph", "test_delete_keyspace_removekscell")
_ = clusterForKSTest.VtctldClientProcess.ExecuteCommand("GetShardReplication", "zone1", "test_delete_keyspace_removekscell/0")
err = clusterForKSTest.VtctlclientProcess.ExecuteCommand("GetShardReplication", "zone2", "test_delete_keyspace_removekscell/0")
err = clusterForKSTest.VtctldClientProcess.ExecuteCommand("GetShardReplication", "zone2", "test_delete_keyspace_removekscell/0")
require.Error(t, err)
err = clusterForKSTest.VtctlclientProcess.ExecuteCommand("GetShardReplication", "zone2", "test_delete_keyspace_removekscell/1")
err = clusterForKSTest.VtctldClientProcess.ExecuteCommand("GetShardReplication", "zone2", "test_delete_keyspace_removekscell/1")
require.Error(t, err)
// Clean up
_ = clusterForKSTest.VtctlclientProcess.ExecuteCommand("DeleteKeyspace", "-recursive", "test_delete_keyspace_removekscell")
_ = clusterForKSTest.VtctldClientProcess.ExecuteCommand("DeleteKeyspace", "-recursive", "test_delete_keyspace_removekscell")
} */

func TestShardCountForAllKeyspaces(t *testing.T) {
Expand Down
2 changes: 1 addition & 1 deletion go/test/endtoend/reparent/utils/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -677,7 +677,7 @@ func CheckReparentFromOutside(t *testing.T, clusterInstance *cluster.LocalProces
assert.Len(t, result[cell1].Nodes, 2)
}
} else {
result, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("GetShardReplication", cell1, KeyspaceShard)
result, err := clusterInstance.VtctldClientProcess.ExecuteCommandWithOutput("GetShardReplication", cell1, KeyspaceShard)
require.Nil(t, err, "error should be Nil")
if !downPrimary {
assertNodeCount(t, result, int(3))
Expand Down
10 changes: 5 additions & 5 deletions go/test/endtoend/sharded/sharded_keyspace_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,13 +147,13 @@ func TestShardedKeyspace(t *testing.T) {
require.Nil(t, err)
assert.Equal(t, `[[INT64(1) VARCHAR("test 1")]]`, fmt.Sprintf("%v", rows.Rows))

err = clusterInstance.VtctlclientProcess.ExecuteCommand("ValidateSchemaShard", fmt.Sprintf("%s/%s", keyspaceName, shard1.Name))
err = clusterInstance.VtctldClientProcess.ExecuteCommand("ValidateSchemaShard", fmt.Sprintf("%s/%s", keyspaceName, shard1.Name))
require.Nil(t, err)

err = clusterInstance.VtctlclientProcess.ExecuteCommand("ValidateSchemaShard", fmt.Sprintf("%s/%s", keyspaceName, shard1.Name))
err = clusterInstance.VtctldClientProcess.ExecuteCommand("ValidateSchemaShard", fmt.Sprintf("%s/%s", keyspaceName, shard1.Name))
require.Nil(t, err)

output, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("ValidateSchemaKeyspace", keyspaceName)
output, err := clusterInstance.VtctldClientProcess.ExecuteCommandWithOutput("ValidateSchemaKeyspace", keyspaceName)
require.Error(t, err)
// We should assert that there is a schema difference and that both the shard primaries are involved in it.
// However, we cannot assert in which order the two primaries will occur since the underlying function does not guarantee that
Expand All @@ -166,9 +166,9 @@ func TestShardedKeyspace(t *testing.T) {
require.Nil(t, err)
err = clusterInstance.VtctldClientProcess.ExecuteCommand("GetPermissions", shard1.Vttablets[1].Alias)
require.Nil(t, err)
err = clusterInstance.VtctlclientProcess.ExecuteCommand("ValidatePermissionsShard", fmt.Sprintf("%s/%s", keyspaceName, shard1.Name))
err = clusterInstance.VtctldClientProcess.ExecuteCommand("ValidatePermissionsShard", fmt.Sprintf("%s/%s", keyspaceName, shard1.Name))
require.Nil(t, err)
err = clusterInstance.VtctlclientProcess.ExecuteCommand("ValidatePermissionsKeyspace", keyspaceName)
err = clusterInstance.VtctldClientProcess.ExecuteCommand("ValidatePermissionsKeyspace", keyspaceName)
require.Nil(t, err)

rows, err = shard1Primary.VttabletProcess.QueryTablet("select id, msg from vt_select_test order by id", keyspaceName, true)
Expand Down
24 changes: 10 additions & 14 deletions go/test/endtoend/vreplication/fk_ext_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,8 +151,7 @@ func TestFKExt(t *testing.T) {
}
sqls := strings.Split(FKExtSourceSchema, "\n")
for _, sql := range sqls {
output, err := vc.VtctlClient.ExecuteCommandWithOutput("ApplySchema", "--",
"--ddl_strategy=direct", "--sql", sql, keyspaceName)
output, err := vc.VtctldClient.ExecuteCommandWithOutput("ApplySchema", "--ddl-strategy=direct", "--sql", sql, keyspaceName)
require.NoErrorf(t, err, output)
}
doReshard(t, fkextConfig.target2KeyspaceName, "reshard2to3", "-80,80-", threeShards, tablets)
Expand All @@ -165,8 +164,7 @@ func TestFKExt(t *testing.T) {
tablets[shard] = vc.Cells[cellName].Keyspaces[keyspaceName].Shards[shard].Tablets[fmt.Sprintf("%s-%d", cellName, tabletID)].Vttablet
sqls := strings.Split(FKExtSourceSchema, "\n")
for _, sql := range sqls {
output, err := vc.VtctlClient.ExecuteCommandWithOutput("ApplySchema", "--",
"--ddl_strategy=direct", "--sql", sql, keyspaceName)
output, err := vc.VtctldClient.ExecuteCommandWithOutput("ApplySchema", "--ddl-strategy=direct", "--sql", sql, keyspaceName)
require.NoErrorf(t, err, output)
}
doReshard(t, fkextConfig.target2KeyspaceName, "reshard3to1", threeShards, "0", tablets)
Expand Down Expand Up @@ -254,7 +252,7 @@ func doReshard(t *testing.T, keyspace, workflowName, sourceShards, targetShards
for _, targetTab := range targetTabs {
catchup(t, targetTab, workflowName, "Reshard")
}
vdiff(t, keyspace, workflowName, fkextConfig.cell, false, true, nil)
vdiff(t, keyspace, workflowName, fkextConfig.cell, nil)
rs.SwitchReadsAndWrites()
//if lg.WaitForAdditionalRows(100) != nil {
// t.Fatal("WaitForAdditionalRows failed")
Expand All @@ -263,7 +261,7 @@ func doReshard(t *testing.T, keyspace, workflowName, sourceShards, targetShards
if compareRowCounts(t, keyspace, strings.Split(sourceShards, ","), strings.Split(targetShards, ",")) != nil {
t.Fatal("Row counts do not match")
}
vdiff(t, keyspace, workflowName+"_reverse", fkextConfig.cell, true, false, nil)
vdiff(t, keyspace, workflowName+"_reverse", fkextConfig.cell, nil)

rs.ReverseReadsAndWrites()
//if lg.WaitForAdditionalRows(100) != nil {
Expand All @@ -273,7 +271,7 @@ func doReshard(t *testing.T, keyspace, workflowName, sourceShards, targetShards
if compareRowCounts(t, keyspace, strings.Split(targetShards, ","), strings.Split(sourceShards, ",")) != nil {
t.Fatal("Row counts do not match")
}
vdiff(t, keyspace, workflowName, fkextConfig.cell, false, true, nil)
vdiff(t, keyspace, workflowName, fkextConfig.cell, nil)
lg.Stop()

rs.SwitchReadsAndWrites()
Expand Down Expand Up @@ -313,12 +311,10 @@ const fkExtMaterializeSpec = `

func materializeTables(t *testing.T) {
wfName := "mat"
err := vc.VtctlClient.ExecuteCommand("ApplySchema", "--", "--ddl_strategy=direct",
"--sql", FKExtMaterializeSchema, fkextConfig.target1KeyspaceName)
err := vc.VtctldClient.ExecuteCommand("ApplySchema", "--ddl-strategy=direct", "--sql", FKExtMaterializeSchema, fkextConfig.target1KeyspaceName)
require.NoError(t, err, fmt.Sprintf("ApplySchema Error: %s", err))
materializeSpec := fmt.Sprintf(fkExtMaterializeSpec, "mat", fkextConfig.target2KeyspaceName, fkextConfig.target1KeyspaceName)
err = vc.VtctlClient.ExecuteCommand("Materialize", materializeSpec)
require.NoError(t, err, "Materialize")
materialize(t, materializeSpec)
tab := vc.getPrimaryTablet(t, fkextConfig.target1KeyspaceName, "0")
catchup(t, tab, wfName, "Materialize")
validateMaterializeRowCounts(t)
Expand Down Expand Up @@ -363,7 +359,7 @@ func doMoveTables(t *testing.T, sourceKeyspace, targetKeyspace, workflowName, ta
for _, targetTab := range targetTabs {
catchup(t, targetTab, workflowName, "MoveTables")
}
vdiff(t, targetKeyspace, workflowName, fkextConfig.cell, false, true, nil)
vdiff(t, targetKeyspace, workflowName, fkextConfig.cell, nil)
lg.Stop()
lg.SetDBStrategy("vtgate", targetKeyspace)
if lg.Start() != nil {
Expand All @@ -377,7 +373,7 @@ func doMoveTables(t *testing.T, sourceKeyspace, targetKeyspace, workflowName, ta
}

waitForLowLag(t, sourceKeyspace, workflowName+"_reverse")
vdiff(t, sourceKeyspace, workflowName+"_reverse", fkextConfig.cell, false, true, nil)
vdiff(t, sourceKeyspace, workflowName+"_reverse", fkextConfig.cell, nil)
if lg.WaitForAdditionalRows(100) != nil {
t.Fatal("WaitForAdditionalRows failed")
}
Expand All @@ -388,7 +384,7 @@ func doMoveTables(t *testing.T, sourceKeyspace, targetKeyspace, workflowName, ta
}
waitForLowLag(t, targetKeyspace, workflowName)
time.Sleep(5 * time.Second)
vdiff(t, targetKeyspace, workflowName, fkextConfig.cell, false, true, nil)
vdiff(t, targetKeyspace, workflowName, fkextConfig.cell, nil)
lg.Stop()
mt.SwitchReadsAndWrites()
mt.Complete()
Expand Down
4 changes: 2 additions & 2 deletions go/test/endtoend/vreplication/fk_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,11 +102,11 @@ func TestFKWorkflow(t *testing.T) {
targetTab := targetKs.Shards["0"].Tablets[fmt.Sprintf("%s-%d", cellName, targetTabletId)].Vttablet
require.NotNil(t, targetTab)
catchup(t, targetTab, workflowName, "MoveTables")
vdiff(t, targetKeyspace, workflowName, cellName, true, false, nil)
vdiff(t, targetKeyspace, workflowName, cellName, nil)
if withLoad {
ls.waitForAdditionalRows(200)
}
vdiff(t, targetKeyspace, workflowName, cellName, true, false, nil)
vdiff(t, targetKeyspace, workflowName, cellName, nil)
if withLoad {
cancel()
<-ch
Expand Down
Loading

0 comments on commit 58b309e

Please sign in to comment.