From 6017496fe802cf28388f087abf35f99c5b5a7b21 Mon Sep 17 00:00:00 2001 From: Philip Offtermatt <57488781+p-offtermatt@users.noreply.github.com> Date: Mon, 27 Nov 2023 10:05:33 +0100 Subject: [PATCH] Bring v0.38.x to the up-to-date status of main (#93) * Update README.md * Refactor: remove isConnected * Add test for Abci query * Enable tags only for semver and default branch (#82) * Add fine-grained control of time (#88) * Start implementing auto-include-tx flag * Split testnet initialization and run * Add script to restart testnet from beginning * Start implementing auto-include-tx flag * Delete backup directory before taking backup * Uncomment killing processes with app binary * Rewrite TxQueue * Fix fire events method * Implement tx filtering and rechecks * Start implementing basic test for Tx effect * Ensure the script is not blocking * Rename Dockerfile-test to Dockerfile.test * Add Contains method * Remove initial setup from Dockerfile * Remove sleep and add -p 1 * Use big.Int for community pool size * Use testnet restart in tests instead of a new setup each time * Add cometmock args to be taken by startup script * Pull time handling into TimeHandler class * Fix CheckTx and time * Add test cases for AutoTx and block production * Add time control to README * Fix auto-tx flag * Add test for starting timestamp * Add test for starting time=system time --- .gitignore | 1 + Dockerfile-test | 26 -- Dockerfile.test | 48 +++ Makefile | 6 +- README.md | 12 +- cometmock/abci_client/client.go | 303 +++++++++++------ cometmock/abci_client/counterparty.go | 4 - cometmock/abci_client/time_handler.go | 119 +++++++ cometmock/main.go | 92 ++++- cometmock/rpc_server/routes.go | 38 ++- cometmock/utils/txs.go | 17 + e2e-tests/.gitignore | 4 + .../local-testnet-singlechain-restart.sh | 27 ++ e2e-tests/local-testnet-singlechain-setup.sh | 228 +++++++++++++ e2e-tests/local-testnet-singlechain-start.sh | 9 + e2e-tests/local-testnet-singlechain.sh | 217 +----------- e2e-tests/main_test.go | 320 +++++++++++++++--- e2e-tests/test_utils.go | 166 +++++++++ 18 files changed, 1213 insertions(+), 424 deletions(-) delete mode 100644 Dockerfile-test create mode 100644 Dockerfile.test create mode 100644 cometmock/abci_client/time_handler.go create mode 100644 cometmock/utils/txs.go create mode 100644 e2e-tests/.gitignore create mode 100755 e2e-tests/local-testnet-singlechain-restart.sh create mode 100755 e2e-tests/local-testnet-singlechain-setup.sh create mode 100755 e2e-tests/local-testnet-singlechain-start.sh create mode 100644 e2e-tests/test_utils.go diff --git a/.gitignore b/.gitignore index 4061428..e71a340 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ # Test binary, built with `go test -c` *.test +!Dockerfile.test # Output of the go coverage tool, specifically when used with LiteIDE *.out diff --git a/Dockerfile-test b/Dockerfile-test deleted file mode 100644 index 256e9db..0000000 --- a/Dockerfile-test +++ /dev/null @@ -1,26 +0,0 @@ -# import simd from ibc-go -FROM ghcr.io/cosmos/simapp:0.50.0-rc.1 AS simapp-builder - -FROM golang:1.21-alpine as cometmock-builder - -ENV PACKAGES curl make git libc-dev bash gcc linux-headers -RUN apk add --no-cache $PACKAGES - -ENV CGO_ENABLED=0 -ENV GOOS=linux -ENV GOFLAGS="-buildvcs=false" - -# cache gomodules for cometmock -ADD ./go.mod /go.mod -ADD ./go.sum /go.sum -RUN go mod download - -# Add CometMock and install it -ADD . /CometMock -WORKDIR /CometMock -RUN go build -o /usr/local/bin/cometmock ./cometmock - -RUN apk update -RUN apk add --no-cache which iputils procps-ng tmux net-tools htop jq gcompat - -COPY --from=simapp-builder /usr/bin/simd /usr/local/bin/simd \ No newline at end of file diff --git a/Dockerfile.test b/Dockerfile.test new file mode 100644 index 0000000..c889496 --- /dev/null +++ b/Dockerfile.test @@ -0,0 +1,48 @@ +# import simd from ibc-go +FROM ghcr.io/cosmos/simapp:0.50.0-rc.1 AS simapp-builder + +FROM golang:1.21-alpine as cometmock-builder + +ENV PACKAGES curl make git libc-dev bash gcc linux-headers +RUN apk add --no-cache $PACKAGES + +ENV CGO_ENABLED=0 +ENV GOOS=linux +ENV GOFLAGS="-buildvcs=false" + +# cache gomodules for cometmock +ADD ./go.mod /go.mod +ADD ./go.sum /go.sum +RUN go mod download + +# Add CometMock and install it +ADD . /CometMock +WORKDIR /CometMock +RUN go build -o /usr/local/bin/cometmock ./cometmock + +RUN apk update +RUN apk add --no-cache which iputils procps-ng tmux net-tools htop jq gcompat + +FROM golang:1.21-alpine as test-env + +ENV PACKAGES curl make git libc-dev bash gcc linux-headers +RUN apk add --no-cache $PACKAGES +RUN apk update +RUN apk add --no-cache which iputils procps-ng tmux net-tools htop jq gcompat + +ENV CGO_ENABLED=0 +ENV GOOS=linux +ENV GOFLAGS="-buildvcs=false" + +ADD ./go.mod /go.mod +ADD ./go.sum /go.sum +RUN go mod download + +ADD ./e2e-tests /CometMock/e2e-tests + +COPY --from=simapp-builder /usr/bin/simd /usr/local/bin/simd + +WORKDIR /CometMock/e2e-tests +RUN /CometMock/e2e-tests/local-testnet-singlechain-setup.sh simd "" + +COPY --from=cometmock-builder /usr/local/bin/cometmock /usr/local/bin/cometmock \ No newline at end of file diff --git a/Makefile b/Makefile index 343ad1d..45ecc32 100644 --- a/Makefile +++ b/Makefile @@ -2,12 +2,12 @@ install: go install ./cometmock test-locally: - go test -timeout 600s ./e2e-tests -test.v + go test -timeout 600s -p 1 ./e2e-tests -test.v test-docker: # Build the Docker image - docker build -f Dockerfile-test -t cometmock-test . + docker build -f Dockerfile.test -t cometmock-test . # Start a container and execute the test command inside docker rm cometmock-test-instance || true - docker run --name cometmock-test-instance --workdir /CometMock cometmock-test go test -timeout 600s ./e2e-tests -test.v \ No newline at end of file + docker run --name cometmock-test-instance --workdir /CometMock cometmock-test go test -p 1 -timeout 600s ./e2e-tests -test.v \ No newline at end of file diff --git a/README.md b/README.md index 6bc8e16..522d73c 100644 --- a/README.md +++ b/README.md @@ -27,11 +27,19 @@ CometMock was tested with `go version go1.20.3 darwin/arm64`. To run CometMock, start your (cosmos-sdk) application instances with the flags ```--with-tendermint=false, --transport=grpc```. After the applications started, start CometMock like this ``` -cometmock [--block-time=XXX] {app_address1,app_address2,...} {genesis_file} {cometmock_listen_address} {home_folder1,home_folder2,...} {connection_mode} +cometmock [--block-time=value] [--auto-tx=] [--block-production-interval=] [--starting-timestamp=] [--starting-timestamp-from-genesis=] {app_address1,app_address2,...} {genesis_file} {cometmock_listen_address} {home_folder1,home_folder2,...} {connection_mode} ``` where: -* The `--block-time` flag is optional and specifies the time in milliseconds between blocks. The default is 1000ms(=1s). Values <= 0 mean that automatic block production is disabled. In this case, blocks can be produced by calling the `advance_blocks` endpoint or by broadcasting transactions (each transaction will be included in a fresh block). Note that all flags have to come before positional arguments. +* The `--block-time` flag is optional and specifies the time in milliseconds between the timestamps of consecutive blocks. +Values <= 0 mean that the timestamps are taken from the system time. The default value is -1. +* The `--auto-tx` flag is optional. If it is set to true, when a transaction is broadcasted, it will be automatically included in the next block. The default value is false. +* The `--block-production-interval` flag is optional and specifies the time (in milliseconds) to sleep between the production of consecutive blocks. +This does not mean that blocks are produced this fast, just that CometMock will sleep by this amount between producing two blocks. +The default value is 1000ms=1s. +* The `--starting-timestamp` flag is optional and specifies the starting timestamp of the blockchain. If not specified, the starting timestamp is taken from the system time. +* The `--starting-timestamp-from-genesis` flag is optional and can be used to override the starting timestamp of the blockchain with the timestamp of the genesis file. +In that case, the first block will have a timestamp of Genesis timestamp + block time or, if block time is <= 0, Genesis timestamp + some small, unspecified amount depending on system time. * The `app_addresses` are the `--address` flags of the applications. This is by default `"tcp://0.0.0.0:26658"` * The `genesis_file` is the genesis json that is also used by apps. * The `cometmock_listen_address` can be freely chosen and will be the address that requests that would normally go to CometBFT rpc endpoints need to be directed to. diff --git a/cometmock/abci_client/client.go b/cometmock/abci_client/client.go index 919c312..12ade8e 100644 --- a/cometmock/abci_client/client.go +++ b/cometmock/abci_client/client.go @@ -14,12 +14,13 @@ import ( "github.com/cometbft/cometbft/crypto/merkle" cometlog "github.com/cometbft/cometbft/libs/log" cmtmath "github.com/cometbft/cometbft/libs/math" - cmttypes "github.com/cometbft/cometbft/proto/tendermint/types" + cmtproto "github.com/cometbft/cometbft/proto/tendermint/types" "github.com/cometbft/cometbft/state" blockindexkv "github.com/cometbft/cometbft/state/indexer/block/kv" "github.com/cometbft/cometbft/state/txindex" indexerkv "github.com/cometbft/cometbft/state/txindex/kv" "github.com/cometbft/cometbft/types" + cmttypes "github.com/cometbft/cometbft/types" "github.com/informalsystems/CometMock/cometmock/storage" "github.com/informalsystems/CometMock/cometmock/utils" ) @@ -44,7 +45,7 @@ const ( // hardcode max data bytes to -1 (unlimited) since we do not utilize a mempool // to pick evidence/txs out of -const maxDataBytes = -1 +const maxDataBytes = cmttypes.MaxBlockSizeBytes // AbciClient facilitates calls to the ABCI interface of multiple nodes. // It also tracks the current state and a common logger. @@ -70,24 +71,34 @@ type AbciClient struct { signingStatus map[string]bool signingStatusMutex sync.RWMutex - // time offset. whenever we qury the time, we add this offset to it - // this means after modifying this, blocks will have the timestamp offset by this value. - // this will look to the app like one block took a long time to be produced. - timeOffset time.Duration + // The TimeHandler that will be queried + // to obtain the block timestamp for each block. + TimeHandler TimeHandler + + // If this is true, then when broadcastTx is called, + // a block will automatically be produced immediately. + // If not, the transaction will be added to the TxQueue + // and consumed when the next block is created. + AutoIncludeTx bool + + // A list of transactions that will be included in the next block that is created. + // When transaction FreshTxQueue[i] is included, it will be removed from the FreshTxQueue, + // and the result will be sent to ResponseChannelQueue[i]. + // + FreshTxQueue []types.Tx + StaleTxQueue []types.Tx } -func (a *AbciClient) GetTimeOffset() time.Duration { - return a.timeOffset +func (a *AbciClient) QueueTx(tx types.Tx) { + // lock the block mutex so txs are not queued while a block is being run + blockMutex.Lock() + a.FreshTxQueue = append(a.FreshTxQueue, tx) + blockMutex.Unlock() } -func (a *AbciClient) IncrementTimeOffset(additionalOffset time.Duration) error { - if additionalOffset < 0 { - a.Logger.Error("time offset cannot be decremented, please provide a positive offset") - return fmt.Errorf("time offset cannot be decremented, please provide a positive offset") - } - a.Logger.Debug("Incrementing time offset", "additionalOffset", additionalOffset.String()) - a.timeOffset = a.timeOffset + additionalOffset - return nil +func (a *AbciClient) ClearTxs() { + a.FreshTxQueue = make([]types.Tx, 0) + a.StaleTxQueue = make([]types.Tx, 0) } func (a *AbciClient) CauseLightClientAttack(address string, misbehaviourType string) error { @@ -111,7 +122,7 @@ func (a *AbciClient) CauseLightClientAttack(address string, misbehaviourType str return fmt.Errorf("unknown misbehaviour type %s, possible types are: Equivocation, Lunatic, Amnesia", misbehaviourType) } - _, _, _, err = a.RunBlockWithEvidence(nil, map[*types.Validator]MisbehaviourType{validator: misbehaviour}) + err = a.RunBlockWithEvidence(map[*types.Validator]MisbehaviourType{validator: misbehaviour}) return err } @@ -123,8 +134,7 @@ func (a *AbciClient) CauseDoubleSign(address string) error { return err } - _, _, _, err = a.RunBlockWithEvidence(nil, map[*types.Validator]MisbehaviourType{validator: DuplicateVote}) - return err + return a.RunBlockWithEvidence(map[*types.Validator]MisbehaviourType{validator: DuplicateVote}) } func (a *AbciClient) GetValidatorFromAddress(address string) (*types.Validator, error) { @@ -203,7 +213,16 @@ func CreateAndStartIndexerService(eventBus *types.EventBus, logger cometlog.Logg return indexerService, txIndexer, blockIndexer, indexerService.Start() } -func NewAbciClient(clients map[string]AbciCounterpartyClient, logger cometlog.Logger, curState state.State, lastBlock *types.Block, lastCommit *types.ExtendedCommit, storage storage.Storage, errorOnUnequalResponses bool) *AbciClient { +func NewAbciClient( + clients map[string]AbciCounterpartyClient, + logger cometlog.Logger, + curState state.State, + lastBlock *types.Block, + lastCommit *types.ExtendedCommit, + storage storage.Storage, + timeHandler TimeHandler, + errorOnUnequalResponses bool, +) *AbciClient { signingStatus := make(map[string]bool) for addr := range clients { signingStatus[addr] = true @@ -232,8 +251,10 @@ func NewAbciClient(clients map[string]AbciCounterpartyClient, logger cometlog.Lo IndexerService: indexerService, TxIndex: txIndex, BlockIndex: blockIndex, + TimeHandler: timeHandler, ErrorOnUnequalResponses: errorOnUnequalResponses, signingStatus: signingStatus, + FreshTxQueue: make([]types.Tx, 0), } } @@ -426,10 +447,11 @@ func (a *AbciClient) SendCommit() (*abcitypes.ResponseCommit, error) { return responses[0], nil } -func (a *AbciClient) SendCheckTx(tx *[]byte) (*abcitypes.ResponseCheckTx, error) { +func (a *AbciClient) SendCheckTx(checkType abcitypes.CheckTxType, tx *[]byte) (*abcitypes.ResponseCheckTx, error) { // build the CheckTx request checkTxRequest := abcitypes.RequestCheckTx{ - Tx: *tx, + Tx: *tx, + Type: checkType, } // send CheckTx to all clients and collect the responses @@ -460,18 +482,6 @@ func (a *AbciClient) SendCheckTx(tx *[]byte) (*abcitypes.ResponseCheckTx, error) } func (a *AbciClient) SendAbciQuery(data []byte, path string, height int64, prove bool) (*abcitypes.ResponseQuery, error) { - // find the first connected client - var client *AbciCounterpartyClient - for _, c := range a.Clients { - if c.isConnected { - client = &c - break - } - } - if client == nil { - return nil, fmt.Errorf("no connected clients") - } - // build the Query request request := abcitypes.RequestQuery{ Data: data, @@ -480,21 +490,36 @@ func (a *AbciClient) SendAbciQuery(data []byte, path string, height int64, prove Prove: prove, } - // send Query to the client and collect the response - ctx, cancel := context.WithTimeout(context.Background(), ABCI_TIMEOUT) - defer cancel() - response, err := client.Client.Query(ctx, &request) - if err != nil { - return nil, err + responses := make([]*abcitypes.ResponseQuery, 0) + + for _, client := range a.Clients { + // send Query to all clients and collect the responses + ctx, cancel := context.WithTimeout(context.Background(), ABCI_TIMEOUT) + defer cancel() + response, err := client.Client.Query(ctx, &request) + if err != nil { + return nil, err + } + + responses = append(responses, response) } - return response, nil + if a.ErrorOnUnequalResponses { + // return an error if the responses are not all equal + for i := 1; i < len(responses); i++ { + if !reflect.DeepEqual(responses[i], responses[0]) { + return nil, fmt.Errorf("responses are not all equal: %v is not equal to %v", responses[i], responses[0]) + } + } + } + + return responses[0], nil } // RunEmptyBlocks runs a specified number of empty blocks through ABCI. func (a *AbciClient) RunEmptyBlocks(numBlocks int) error { for i := 0; i < numBlocks; i++ { - _, _, _, err := a.RunBlock(nil) + err := a.RunBlock() if err != nil { return err } @@ -515,6 +540,8 @@ func (a *AbciClient) decideProposal( // Create a new proposal block from state/txs from the mempool. var err error + numTxs := len(*txs) + _ = numTxs block, err = a.CreateProposalBlock( proposerApp, proposerVal, @@ -605,14 +632,20 @@ func (a *AbciClient) CreateProposalBlock( // RunBlock runs a block with a specified transaction through the ABCI application. // It calls RunBlockWithTimeAndProposer with the current time and the LastValidators.Proposer. -func (a *AbciClient) RunBlock(tx *[]byte) (*abcitypes.ResponseCheckTx, *abcitypes.ResponseFinalizeBlock, *abcitypes.ResponseCommit, error) { - return a.RunBlockWithTimeAndProposer(tx, time.Now().Add(a.timeOffset), a.CurState.LastValidators.Proposer, make(map[*types.Validator]MisbehaviourType, 0)) +func (a *AbciClient) RunBlock() error { + blockTime := a.TimeHandler.GetBlockTime(a.LastBlock.Time) + return a.RunBlockWithTimeAndProposer(blockTime, a.CurState.LastValidators.Proposer, make(map[*types.Validator]MisbehaviourType, 0)) +} + +func (a *AbciClient) RunBlockWithTime(t time.Time) error { + return a.RunBlockWithTimeAndProposer(t, a.CurState.LastValidators.Proposer, make(map[*types.Validator]MisbehaviourType, 0)) } // RunBlockWithEvidence runs a block with a specified transaction through the ABCI application. // It also produces the specified evidence for the specified misbehaving validators. -func (a *AbciClient) RunBlockWithEvidence(tx *[]byte, misbehavingValidators map[*types.Validator]MisbehaviourType) (*abcitypes.ResponseCheckTx, *abcitypes.ResponseFinalizeBlock, *abcitypes.ResponseCommit, error) { - return a.RunBlockWithTimeAndProposer(tx, time.Now().Add(a.timeOffset), a.CurState.LastValidators.Proposer, misbehavingValidators) +func (a *AbciClient) RunBlockWithEvidence(misbehavingValidators map[*types.Validator]MisbehaviourType) error { + blockTime := a.TimeHandler.GetBlockTime(a.LastBlock.Time) + return a.RunBlockWithTimeAndProposer(blockTime, a.CurState.LastValidators.Proposer, misbehavingValidators) } func (a *AbciClient) ConstructDuplicateVoteEvidence(v *types.Validator) (*types.DuplicateVoteEvidence, error) { @@ -632,25 +665,25 @@ func (a *AbciClient) ConstructDuplicateVoteEvidence(v *types.Validator) (*types. index, valInLastState := lastState.Validators.GetByAddress(v.Address) // produce vote A. - voteA := &cmttypes.Vote{ + voteA := &cmtproto.Vote{ ValidatorAddress: v.Address, ValidatorIndex: int32(index), Height: lastBlock.Height, Round: 1, - Timestamp: time.Now().Add(a.timeOffset), - Type: cmttypes.PrecommitType, + Timestamp: lastBlock.Time, + Type: cmtproto.PrecommitType, BlockID: blockId.ToProto(), } // TODO: remove the two votes/create a real difference // produce vote B, which just has a different round. - voteB := &cmttypes.Vote{ + voteB := &cmtproto.Vote{ ValidatorAddress: v.Address, ValidatorIndex: int32(index), Height: lastBlock.Height, Round: 2, // this is what differentiates the votes - Timestamp: time.Now().Add(a.timeOffset), - Type: cmttypes.PrecommitType, + Timestamp: lastBlock.Time, + Type: cmtproto.PrecommitType, BlockID: blockId.ToProto(), } @@ -796,7 +829,7 @@ func (a *AbciClient) ExtendAndSignVote( Height: block.Height, Round: block.LastCommit.Round, Timestamp: block.Time, - Type: cmttypes.PrecommitType, + Type: cmtproto.PrecommitType, BlockID: types.BlockID{ Hash: block.Hash(), PartSetHeader: blockParts.Header(), @@ -877,23 +910,13 @@ func (a *AbciClient) SendFinalizeBlock( return responses[0], nil } -// RunBlock runs a block with a specified transaction through the ABCI application. -// It calls BeginBlock, DeliverTx, EndBlock, Commit and then -// updates the state. -// RunBlock is safe for use by multiple goroutines simultaneously. -func (a *AbciClient) RunBlockWithTimeAndProposer( - tx *[]byte, +// internal method that runs a block. +// Should only be used after locking the blockMutex. +func (a *AbciClient) runBlock_helper( blockTime time.Time, proposer *types.Validator, misbehavingValidators map[*types.Validator]MisbehaviourType, -) (*abcitypes.ResponseCheckTx, *abcitypes.ResponseFinalizeBlock, *abcitypes.ResponseCommit, error) { - // lock mutex to avoid running two blocks at the same time - a.Logger.Debug("Locking mutex") - blockMutex.Lock() - - defer blockMutex.Unlock() - defer a.Logger.Debug("Unlocking mutex") - +) error { a.Logger.Info("Running block") if verbose { a.Logger.Info("State at start of block", "state", a.CurState) @@ -901,17 +924,41 @@ func (a *AbciClient) RunBlockWithTimeAndProposer( newHeight := a.CurState.LastBlockHeight + 1 - txs := make(types.Txs, 0) - if tx != nil { - txs = append(txs, *tx) + var err error + + for index, tx := range a.FreshTxQueue { + txBytes := []byte(tx) + resCheckTx, err := a.SendCheckTx(abcitypes.CheckTxType_New, &txBytes) + if err != nil { + return fmt.Errorf("error from CheckTx: %v", err) + } + // if the CheckTx code is != 0 + if resCheckTx.Code != abcitypes.CodeTypeOK { + // drop the tx by setting the index to empty + a.FreshTxQueue[index] = cmttypes.Tx{} + } } - var resCheckTx *abcitypes.ResponseCheckTx - var err error - if tx != nil { - resCheckTx, err = a.SendCheckTx(tx) + // recheck txs from the stale queue + for index, tx := range a.StaleTxQueue { + txBytes := []byte(tx) + resCheckTx, err := a.SendCheckTx(abcitypes.CheckTxType_Recheck, &txBytes) if err != nil { - return nil, nil, nil, fmt.Errorf("error from CheckTx: %v", err) + return fmt.Errorf("error from CheckTx: %v", err) + } + // if the CheckTx code is != 0 + if resCheckTx.Code != abcitypes.CodeTypeOK { + // drop the tx by setting the index to empty + a.StaleTxQueue[index] = cmttypes.Tx{} + } + } + + // filter all empty txs from the queues + newTxQueue := make([]cmttypes.Tx, 0) + for _, tx := range append(a.FreshTxQueue, a.StaleTxQueue...) { + txBytes := []byte(tx) + if len(txBytes) > 0 { + newTxQueue = append(newTxQueue, tx) } } @@ -935,7 +982,7 @@ func (a *AbciClient) RunBlockWithTimeAndProposer( } if err != nil { - return nil, nil, nil, fmt.Errorf("error constructing evidence: %v", err) + return fmt.Errorf("error constructing evidence: %v", err) } evidences = append(evidences, evidence) @@ -950,10 +997,11 @@ func (a *AbciClient) RunBlockWithTimeAndProposer( } if proposerApp == nil { - return nil, nil, nil, fmt.Errorf("could not find proposer app for address %v", proposerAddress) + return fmt.Errorf("could not find proposer app for address %v", proposerAddress) } // The proposer runs PrepareProposal + txs := cmttypes.Txs(newTxQueue) _, block, err := a.decideProposal( proposerApp, proposer, @@ -962,15 +1010,30 @@ func (a *AbciClient) RunBlockWithTimeAndProposer( &txs, evidences, ) + + // set the block time to the time passed as argument + block.Time = blockTime + + // clear the tx queues + a.ClearTxs() + + // for each tx not included in the block, + // put it in the stale queue + for _, tx := range newTxQueue { + if !utils.Contains(block.Txs, tx) { + a.StaleTxQueue = append(a.StaleTxQueue, tx) + } + } + if err != nil { - return nil, nil, nil, fmt.Errorf("error in decideProposal: %v", err) + return fmt.Errorf("error in decideProposal: %v", err) } var nonProposers []*AbciCounterpartyClient for _, val := range a.CurState.Validators.Validators { client, err := a.GetCounterpartyFromAddress(val.Address.String()) if err != nil { - return nil, nil, nil, fmt.Errorf("error when getting counterparty client from address: address %v, error %v", val.Address.String(), err) + return fmt.Errorf("error when getting counterparty client from address: address %v, error %v", val.Address.String(), err) } if client.ValidatorAddress != proposerAddress.String() { @@ -982,11 +1045,11 @@ func (a *AbciClient) RunBlockWithTimeAndProposer( for _, client := range nonProposers { accepted, err := a.ProcessProposal(client, block) if err != nil { - return nil, nil, nil, fmt.Errorf("error in ProcessProposal for block %v, error %v", block.String(), err) + return fmt.Errorf("error in ProcessProposal for block %v, error %v", block.String(), err) } if !accepted { - return nil, nil, nil, fmt.Errorf("non-proposer %v did not accept the proposal for block %v", client.ValidatorAddress, block.String()) + return fmt.Errorf("non-proposer %v did not accept the proposal for block %v", client.ValidatorAddress, block.String()) } } @@ -997,17 +1060,17 @@ func (a *AbciClient) RunBlockWithTimeAndProposer( shouldSign, err := a.GetSigningStatus(val.Address.String()) if err != nil { - return nil, nil, nil, fmt.Errorf("error getting signing status for validator %v, error %v", val.Address.String(), err) + return fmt.Errorf("error getting signing status for validator %v, error %v", val.Address.String(), err) } if shouldSign { client, ok := a.Clients[val.Address.String()] if !ok { - return nil, nil, nil, fmt.Errorf("did not find privval for address: address %v", val.Address.String()) + return fmt.Errorf("did not find privval for address: address %v", val.Address.String()) } vote, err := a.ExtendAndSignVote(&client, val, int32(index), block) if err != nil { - return nil, nil, nil, fmt.Errorf("error when signing vote for validator %v, error %v", val.Address.String(), err) + return fmt.Errorf("error when signing vote for validator %v, error %v", val.Address.String(), err) } votes = append(votes, vote) @@ -1023,7 +1086,7 @@ func (a *AbciClient) RunBlockWithTimeAndProposer( a.Logger.Info("Verifying vote extension for validator", val.Address.String()) client, err := a.GetCounterpartyFromAddress(val.Address.String()) if err != nil { - return nil, nil, nil, fmt.Errorf("error when getting counterparty client from address: address %v, error %v", val.Address.String(), err) + return fmt.Errorf("error when getting counterparty client from address: address %v, error %v", val.Address.String(), err) } for _, vote := range votes { @@ -1064,7 +1127,7 @@ func (a *AbciClient) RunBlockWithTimeAndProposer( a.CurState.ChainID, block.Height, 0, // round is hardcoded to 0 - cmttypes.PrecommitType, + cmtproto.PrecommitType, a.CurState.Validators, ) } else { @@ -1072,7 +1135,7 @@ func (a *AbciClient) RunBlockWithTimeAndProposer( a.CurState.ChainID, block.Height, 0, // round is hardcoded to 0 - cmttypes.PrecommitType, + cmtproto.PrecommitType, a.CurState.Validators, ) } @@ -1082,10 +1145,10 @@ func (a *AbciClient) RunBlockWithTimeAndProposer( if vote != nil { added, err := voteSet.AddVote(vote) if err != nil { - return nil, nil, nil, fmt.Errorf("error adding vote %v to vote set: %v", vote.String(), err) + return fmt.Errorf("error adding vote %v to vote set: %v", vote.String(), err) } if !added { - return nil, nil, nil, fmt.Errorf("could not add vote %v to vote set", vote.String()) + return fmt.Errorf("could not add vote %v to vote set", vote.String()) } } } @@ -1096,7 +1159,7 @@ func (a *AbciClient) RunBlockWithTimeAndProposer( // sanity check that the commit is signed correctly err = a.CurState.Validators.VerifyCommitLightTrusting(a.CurState.ChainID, a.LastCommit.ToCommit(), cmtmath.Fraction{Numerator: 1, Denominator: 3}) if err != nil { - return nil, nil, nil, fmt.Errorf("error verifying commit %v: %v", a.LastCommit.ToCommit().StringIndented("\t"), err) + return fmt.Errorf("error verifying commit %v: %v", a.LastCommit.ToCommit().StringIndented("\t"), err) } // sanity check that the commit makes a proper light block @@ -1113,13 +1176,13 @@ func (a *AbciClient) RunBlockWithTimeAndProposer( err = lightBlock.ValidateBasic(a.CurState.ChainID) if err != nil { a.Logger.Error("Light block validation failed", "err", err) - return nil, nil, nil, err + return err } lastCommitInfo := utils.BuildLastCommitInfo(block, a.CurState.Validators, a.CurState.InitialHeight) resFinalizeBlock, err := a.SendFinalizeBlock(block, &lastCommitInfo) if err != nil { - return nil, nil, nil, fmt.Errorf("error from FinalizeBlock for block %v: %v", block.String(), err) + return fmt.Errorf("error from FinalizeBlock for block %v: %v", block.String(), err) } // lock the state update mutex while the stores are updated to avoid @@ -1133,29 +1196,47 @@ func (a *AbciClient) RunBlockWithTimeAndProposer( // insert entries into the storage err = a.Storage.UpdateStores(newHeight, block, a.LastCommit.ToCommit(), &state, resFinalizeBlock) if err != nil { - return nil, nil, nil, fmt.Errorf("error updating stores: %v", err) + return fmt.Errorf("error updating stores: %v", err) } blockId, err := utils.GetBlockIdFromBlock(block) if err != nil { - return nil, nil, nil, fmt.Errorf("error getting block id from block %v: %v", block.String(), err) + return fmt.Errorf("error getting block id from block %v: %v", block.String(), err) } // updates state as a side effect. returns an error if the state update fails err = a.UpdateStateFromBlock(blockId, block, resFinalizeBlock) if err != nil { - return nil, nil, nil, fmt.Errorf("error updating state for result %v, block %v: %v", resFinalizeBlock.String(), block.String(), err) + return fmt.Errorf("error updating state for result %v, block %v: %v", resFinalizeBlock.String(), block.String(), err) } // unlock the state mutex, since we are done updating state a.Storage.UnlockAfterStateUpdate() - resCommit, err := a.SendCommit() + _, err = a.SendCommit() if err != nil { - return nil, nil, nil, fmt.Errorf("error from Commit for block %v: %v", block.String(), err) + return fmt.Errorf("error from Commit for block %v: %v", block.String(), err) } a.CurState.AppHash = resFinalizeBlock.AppHash - return resCheckTx, resFinalizeBlock, resCommit, nil + return nil +} + +// RunBlock RunBlockWithTimeAndProposer runs a block through the ABCI application. +// RunBlock is safe for use by multiple goroutines simultaneously. +func (a *AbciClient) RunBlockWithTimeAndProposer( + blockTime time.Time, + proposer *types.Validator, + misbehavingValidators map[*types.Validator]MisbehaviourType, +) error { + // lock mutex to avoid running two blocks at the same time + a.Logger.Debug("Locking mutex") + blockMutex.Lock() + + err := a.runBlock_helper(blockTime, proposer, misbehavingValidators) + + blockMutex.Unlock() + a.Logger.Debug("Unlocking mutex") + return err } // UpdateStateFromBlock updates the AbciClients state @@ -1194,7 +1275,7 @@ func (a *AbciClient) UpdateStateFromBlock( // Events are fired after everything else. // NOTE: if we crash between Commit and Save, events wont be fired during replay - fireEvents(a.Logger, &a.EventBus, block, finalizeBlockRes, validatorUpdates) + fireEvents(a.Logger, &a.EventBus, block, *blockId, finalizeBlockRes, validatorUpdates) return nil } @@ -1291,27 +1372,37 @@ func validateValidatorUpdates( return nil } +// Fire NewBlock, NewBlockHeader. +// Fire TxEvent for every tx. +// NOTE: if CometBFT crashes before commit, some or all of these events may be published again. func fireEvents( logger cometlog.Logger, eventBus types.BlockEventPublisher, block *types.Block, - finalizeBlockRes *abcitypes.ResponseFinalizeBlock, + blockID types.BlockID, + abciResponse *abcitypes.ResponseFinalizeBlock, validatorUpdates []*types.Validator, ) { if err := eventBus.PublishEventNewBlock(types.EventDataNewBlock{ - // TODO: fill in BlockID Block: block, - ResultFinalizeBlock: *finalizeBlockRes, + BlockID: blockID, + ResultFinalizeBlock: *abciResponse, }); err != nil { logger.Error("failed publishing new block", "err", err) } - eventDataNewBlockHeader := types.EventDataNewBlockHeader{ + if err := eventBus.PublishEventNewBlockHeader(types.EventDataNewBlockHeader{ Header: block.Header, + }); err != nil { + logger.Error("failed publishing new block header", "err", err) } - if err := eventBus.PublishEventNewBlockHeader(eventDataNewBlockHeader); err != nil { - logger.Error("failed publishing new block header", "err", err) + if err := eventBus.PublishEventNewBlockEvents(types.EventDataNewBlockEvents{ + Height: block.Height, + Events: abciResponse.Events, + NumTxs: int64(len(block.Txs)), + }); err != nil { + logger.Error("failed publishing new block events", "err", err) } if len(block.Evidence.Evidence) != 0 { @@ -1330,7 +1421,7 @@ func fireEvents( Height: block.Height, Index: uint32(i), Tx: tx, - Result: *(finalizeBlockRes.TxResults[i]), + Result: *(abciResponse.TxResults[i]), }}); err != nil { logger.Error("failed publishing event TX", "err", err) } diff --git a/cometmock/abci_client/counterparty.go b/cometmock/abci_client/counterparty.go index b01e3a8..e8ab7e8 100644 --- a/cometmock/abci_client/counterparty.go +++ b/cometmock/abci_client/counterparty.go @@ -14,19 +14,15 @@ type AbciCounterpartyClient struct { Client abciclient.Client NetworkAddress string ValidatorAddress string - isConnected bool PrivValidator types.PrivValidator } // NewAbciCounterpartyClient creates a new AbciCounterpartyClient. -// It always starts with isConnected = false. -// Call AbciClient.RetryDisconnectedClients to try to connect to the app. func NewAbciCounterpartyClient(client abciclient.Client, networkAddress, validatorAddress string, privValidator types.PrivValidator) *AbciCounterpartyClient { return &AbciCounterpartyClient{ Client: client, NetworkAddress: networkAddress, ValidatorAddress: validatorAddress, - isConnected: false, PrivValidator: privValidator, } } diff --git a/cometmock/abci_client/time_handler.go b/cometmock/abci_client/time_handler.go new file mode 100644 index 0000000..e11a1a9 --- /dev/null +++ b/cometmock/abci_client/time_handler.go @@ -0,0 +1,119 @@ +package abci_client + +import ( + "sync" + "time" +) + +// A TimeHandler is responsible for +// deciding the timestamps of blocks. +// It will be called by AbciClient.RunBlock +// to decide on a block time. +// It may decide the time based on any number of factors, +// and the parameters of its methods might expand over time as needed. +// The TimeHandler does not have a way to decide the time of the first block, +// which is expected to be done externally, e.g. from the Genesis. +type TimeHandler interface { + // CONTRACT: TimeHandler.GetBlockTime will be called + // precisely once for each block after the first. + // It returns the timestamp of the next block. + GetBlockTime(lastBlockTimestamp time.Time) time.Time + + // AdvanceTime advances the timestamp of all following blocks by + // the given duration. + // The duration needs to be non-negative. + // It returns the timestamp that the next block would have if it + // was produced now. + AdvanceTime(duration time.Duration) time.Time +} + +// The SystemClockTimeHandler uses the system clock +// to decide the timestamps of blocks. +// It will return the system time + offset for each block. +// The offset is calculated by the initial timestamp +// + the sum of all durations passed to AdvanceTime. +type SystemClockTimeHandler struct { + // The offset to add to the system time. + curOffset time.Duration + + // A mutex that ensures that there are no concurrent calls + // to AdvanceTime + mutex sync.Mutex +} + +func NewSystemClockTimeHandler(initialTimestamp time.Time) *SystemClockTimeHandler { + return &SystemClockTimeHandler{ + curOffset: time.Since(initialTimestamp), + } +} + +func (s *SystemClockTimeHandler) GetBlockTime(lastBlockTimestamp time.Time) time.Time { + return time.Now().Add(s.curOffset) +} + +func (s *SystemClockTimeHandler) AdvanceTime(duration time.Duration) time.Time { + s.mutex.Lock() + defer s.mutex.Unlock() + + s.curOffset += duration + return time.Now().Add(s.curOffset) +} + +var _ TimeHandler = (*SystemClockTimeHandler)(nil) + +// The FixedBlockTimeHandler uses a fixed duration +// to advance the timestamp of a block compared to the previous block. +// The block timestamps therefore do not at all depend on the system time, +// but on the time of the previous block. +type FixedBlockTimeHandler struct { + // The fixed duration to add to the last block time + // when deciding the next block timestamp. + blockTime time.Duration + + // The offset to add to the last block time. + // This will be cleared after each block, + // but since the block time of the next block depends + // on the last block, + // this will shift the timestamps of all future blocks. + curBlockOffset time.Duration + + // A mutex that ensures that GetBlockTime and AdvanceTime + // are not called concurrently. + // Otherwise, the block offset might be put into a broken state. + mutex sync.Mutex + + // The timestamp of the last block we produced. + // If this is used before the first block is produced, + // it will be the zero time. + lastBlockTimestamp time.Time +} + +func NewFixedBlockTimeHandler(blockTime time.Duration) *FixedBlockTimeHandler { + return &FixedBlockTimeHandler{ + blockTime: blockTime, + curBlockOffset: 0, + } +} + +func (f *FixedBlockTimeHandler) GetBlockTime(lastBlockTimestamp time.Time) time.Time { + f.mutex.Lock() + defer f.mutex.Unlock() + + res := lastBlockTimestamp.Add(f.blockTime + f.curBlockOffset) + f.curBlockOffset = 0 + f.lastBlockTimestamp = res + return res +} + +// FixedBlockTimeHandler.AdvanceTime will only return the correct next block time +// after GetBlockTime has been called once, but it will +// still advance the time correctly before that - only the output will be wrong. +func (f *FixedBlockTimeHandler) AdvanceTime(duration time.Duration) time.Time { + f.mutex.Lock() + defer f.mutex.Unlock() + + f.curBlockOffset += duration + return f.lastBlockTimestamp.Add(f.blockTime + f.curBlockOffset) +} + +var _ TimeHandler = (*FixedBlockTimeHandler)(nil) diff --git a/cometmock/main.go b/cometmock/main.go index 38877ef..1ef551b 100644 --- a/cometmock/main.go +++ b/cometmock/main.go @@ -41,7 +41,7 @@ func GetMockPVsFromNodeHomes(nodeHomes []string) []types.PrivValidator { func main() { logger := cometlog.NewTMLogger(cometlog.NewSyncWriter(os.Stdout)) - argumentString := "[--block-time=value] " + argumentString := "[--block-time=value] [--auto-tx=] [--block-production-interval=] [--starting-timestamp=] [--starting-timestamp-from-genesis=] " app := &cli.App{ Name: "cometmock", @@ -60,15 +60,51 @@ func main() { &cli.Int64Flag{ Name: "block-time", Usage: ` -Time between blocks in milliseconds. +The number of milliseconds by which the block timestamp should advance from one block to the next. +If this is <0, block timestamps will advance with the system time between the block productions. +Even then, it is still possible to shift the block time from the system time, e.g. by setting an initial timestamp +or by using the 'advance_time' endpoint.`, + Value: -1, + }, + &cli.BoolFlag{ + Name: "auto-tx", + Usage: ` +If this is true, transactions are included immediately +after they are received via broadcast_tx, i.e. a new block +is created when a BroadcastTx endpoint is hit. +If this is false, transactions are still included +upon creation of new blocks, but CometMock will not specifically produce +a new block when a transaction is broadcast.`, + Value: true, + }, + &cli.Int64Flag{ + Name: "block-production-interval", + Usage: ` +Time to sleep between blocks in milliseconds. To disable block production, set to 0. This will not necessarily mean block production is this fast - it is just the sleep time between blocks. -Setting this to a value <= 0 disables automatic block production. +Setting this to a value < 0 disables automatic block production. In this case, blocks are only produced when instructed explicitly either by advancing blocks or broadcasting transactions.`, Value: 1000, }, + &cli.Int64Flag{ + Name: "starting-timestamp", + Usage: ` +The timestamp to use for the first block, given in milliseconds since the unix epoch. +If this is < 0, the current system time is used. +If this is >= 0, the system time is ignored and this timestamp is used for the first block instead.`, + Value: -1, + }, + &cli.BoolFlag{ + Name: "starting-timestamp-from-genesis", + Usage: ` +If this is true, it overrides the starting-timestamp, and instead +bases the time for the first block on the genesis time, incremented by the block time +or the system time between creating the genesis request and producing the first block.`, + Value: false, + }, }, ArgsUsage: argumentString, Action: func(c *cli.Context) error { @@ -86,8 +122,8 @@ advancing blocks or broadcasting transactions.`, return cli.Exit(fmt.Sprintf("Invalid connection mode: %s. Connection mode must be either 'socket' or 'grpc'.\nUsage: %s", connectionMode, argumentString), 1) } - blockTime := c.Int("block-time") - fmt.Printf("Block time: %d\n", blockTime) + blockProductionInterval := c.Int("block-production-interval") + fmt.Printf("Block production interval: %d\n", blockProductionInterval) // read node homes from args nodeHomes := strings.Split(nodeHomesString, ",") @@ -112,6 +148,26 @@ advancing blocks or broadcasting transactions.`, panic(err) } + // read starting timestamp from args + // if starting timestamp should be taken from genesis, + // read it from there + var startingTime time.Time + if c.Bool("starting-timestamp-from-genesis") { + startingTime = genesisDoc.GenesisTime + } else { + if c.Int64("starting-timestamp") < 0 { + startingTime = time.Now() + } else { + dur := time.Duration(c.Int64("starting-timestamp")) * time.Millisecond + startingTime = time.Unix(0, 0).Add(dur) + } + } + fmt.Printf("Starting time: %s\n", startingTime.Format(time.RFC3339)) + + // read block time from args + blockTime := time.Duration(c.Int64("block-time")) * time.Millisecond + fmt.Printf("Block time: %d\n", blockTime.Milliseconds()) + clientMap := make(map[string]abci_client.AbciCounterpartyClient) for i, appAddress := range appAddresses { @@ -139,6 +195,13 @@ advancing blocks or broadcasting transactions.`, clientMap[validatorAddress.String()] = *counterpartyClient } + var timeHandler abci_client.TimeHandler + if blockTime < 0 { + timeHandler = abci_client.NewSystemClockTimeHandler(startingTime) + } else { + timeHandler = abci_client.NewFixedBlockTimeHandler(blockTime) + } + abci_client.GlobalClient = abci_client.NewAbciClient( clientMap, logger, @@ -146,9 +209,13 @@ advancing blocks or broadcasting transactions.`, &types.Block{}, &types.ExtendedCommit{}, &storage.MapStorage{}, + timeHandler, true, ) + abci_client.GlobalClient.AutoIncludeTx = c.Bool("auto-tx") + fmt.Printf("Auto include tx: %t\n", abci_client.GlobalClient.AutoIncludeTx) + // initialize chain err = abci_client.GlobalClient.SendInitChain(curState, genesisDoc) if err != nil { @@ -156,8 +223,15 @@ advancing blocks or broadcasting transactions.`, panic(err) } + var firstBlockTime time.Time + if blockTime < 0 { + firstBlockTime = startingTime + } else { + firstBlockTime = startingTime.Add(blockTime) + } + // run an empty block - _, _, _, err = abci_client.GlobalClient.RunBlock(nil) + err = abci_client.GlobalClient.RunBlockWithTime(firstBlockTime) if err != nil { logger.Error(err.Error()) panic(err) @@ -165,15 +239,15 @@ advancing blocks or broadcasting transactions.`, go rpc_server.StartRPCServerWithDefaultConfig(cometMockListenAddress, logger) - if blockTime > 0 { + if blockProductionInterval > 0 { // produce blocks according to blockTime for { - _, _, _, err := abci_client.GlobalClient.RunBlock(nil) + err := abci_client.GlobalClient.RunBlock() if err != nil { logger.Error(err.Error()) panic(err) } - time.Sleep(time.Millisecond * time.Duration(blockTime)) + time.Sleep(time.Millisecond * time.Duration(blockProductionInterval)) } } else { // wait forever diff --git a/cometmock/rpc_server/routes.go b/cometmock/rpc_server/routes.go index 5c48e47..f84613e 100644 --- a/cometmock/rpc_server/routes.go +++ b/cometmock/rpc_server/routes.go @@ -10,6 +10,8 @@ import ( cmtmath "github.com/cometbft/cometbft/libs/math" cmtquery "github.com/cometbft/cometbft/libs/pubsub/query" "github.com/cometbft/cometbft/p2p" + + abcitypes "github.com/cometbft/cometbft/abci/types" ctypes "github.com/cometbft/cometbft/rpc/core/types" rpc "github.com/cometbft/cometbft/rpc/jsonrpc/server" rpctypes "github.com/cometbft/cometbft/rpc/jsonrpc/types" @@ -85,8 +87,8 @@ func AdvanceTime(ctx *rpctypes.Context, duration_in_seconds time.Duration) (*Res return nil, errors.New("duration to advance time by must be greater than 0") } - abci_client.GlobalClient.IncrementTimeOffset(duration_in_seconds * time.Second) - return &ResultAdvanceTime{time.Now().Add(abci_client.GlobalClient.GetTimeOffset())}, nil + res := abci_client.GlobalClient.TimeHandler.AdvanceTime(duration_in_seconds * time.Second) + return &ResultAdvanceTime{res}, nil } type ResultSetSigningStatus struct { @@ -432,16 +434,14 @@ func Health(ctx *rpctypes.Context) (*ctypes.ResultHealth, error) { return &ctypes.ResultHealth{}, nil } +// CURRENTLY UNSUPPORTED - THIS IS BECAUSE IT IS DISCOURAGED TO USE THIS BY COMETBFT +// needs some major changes to work with ABCI++ // BroadcastTxCommit broadcasts a transaction, // and wait until it is included in a block and and comitted. // In our case, this means running a block with just the the transition, // then return. func BroadcastTxCommit(ctx *rpctypes.Context, tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) { - abci_client.GlobalClient.Logger.Info( - "BroadcastTxCommit called", "tx", tx) - - res, err := BroadcastTx(&tx) - return res, err + return nil, errors.New("BroadcastTxCommit is currently not supported. Try BroadcastTxSync or BroadcastTxAsync instead") } // BroadcastTxSync would normally broadcast a transaction and wait until it gets the result from CheckTx. @@ -480,25 +480,26 @@ func BroadcastTxAsync(ctx *rpctypes.Context, tx types.Tx) (*ctypes.ResultBroadca return &ctypes.ResultBroadcastTx{}, nil } -// BroadcastTx delivers a transaction to the ABCI client, includes it in the next block, then returns. func BroadcastTx(tx *types.Tx) (*ctypes.ResultBroadcastTxCommit, error) { abci_client.GlobalClient.Logger.Info( "BroadcastTxs called", "tx", tx) - byteTx := []byte(*tx) - - responseCheckTx, responseFinalizeBlock, _, err := abci_client.GlobalClient.RunBlock(&byteTx) + txBytes := []byte(*tx) + checkTxResponse, err := abci_client.GlobalClient.SendCheckTx(abcitypes.CheckTxType_New, &txBytes) if err != nil { return nil, err } + abci_client.GlobalClient.QueueTx(*tx) + + if abci_client.GlobalClient.AutoIncludeTx { + go abci_client.GlobalClient.RunBlock() + } - // TODO: fill the return value if necessary return &ctypes.ResultBroadcastTxCommit{ - CheckTx: *responseCheckTx, - TxResult: *responseFinalizeBlock.TxResults[0], - Height: abci_client.GlobalClient.LastBlock.Height, - Hash: tx.Hash(), - }, nil + CheckTx: *checkTxResponse, + Hash: tx.Hash(), + Height: abci_client.GlobalClient.CurState.LastBlockHeight, + }, err } func ABCIInfo(ctx *rpctypes.Context) (*ctypes.ResultABCIInfo, error) { @@ -520,6 +521,9 @@ func ABCIQuery( "ABCIQuery called", "path", "data", "height", "prove", path, data, height, prove) response, err := abci_client.GlobalClient.SendAbciQuery(data, path, height, prove) + if err != nil { + return nil, err + } abci_client.GlobalClient.Logger.Info( "Response to ABCI query", response.String()) diff --git a/cometmock/utils/txs.go b/cometmock/utils/txs.go new file mode 100644 index 0000000..92601e3 --- /dev/null +++ b/cometmock/utils/txs.go @@ -0,0 +1,17 @@ +package utils + +import ( + "bytes" + + cmttypes "github.com/cometbft/cometbft/types" +) + +// Contains returns true if txs contains tx, false otherwise. +func Contains(txs cmttypes.Txs, tx cmttypes.Tx) bool { + for _, ttx := range txs { + if bytes.Equal([]byte(ttx), []byte(tx)) { + return true + } + } + return false +} diff --git a/e2e-tests/.gitignore b/e2e-tests/.gitignore new file mode 100644 index 0000000..219563d --- /dev/null +++ b/e2e-tests/.gitignore @@ -0,0 +1,4 @@ +# Scripts that are generated by the testnet setup +start_apps.sh +start_cometmock.sh +cometmock_log \ No newline at end of file diff --git a/e2e-tests/local-testnet-singlechain-restart.sh b/e2e-tests/local-testnet-singlechain-restart.sh new file mode 100755 index 0000000..7edba3a --- /dev/null +++ b/e2e-tests/local-testnet-singlechain-restart.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# After the testnet was started, this script can restart it. +# It does so by killing the existing testnet, +# overwriting the node home directories with backups made +# right after initializatio, and then starting the testnet again. + + +BINARY_NAME=$1 + +set -eux + +ROOT_DIR=${HOME}/nodes/provider +BACKUP_DIR=${ROOT_DIR}_bkup + +if [ -z "$BINARY_NAME" ]; then + echo "Usage: $0 [cometmock_args]" + exit 1 +fi + +# Kill the testnet +pkill -f ^$BINARY_NAME &> /dev/null || true +pkill -f ^cometmock &> /dev/null || true + +# Restore the backup +rm -rf ${ROOT_DIR} +cp -r ${BACKUP_DIR} ${ROOT_DIR} \ No newline at end of file diff --git a/e2e-tests/local-testnet-singlechain-setup.sh b/e2e-tests/local-testnet-singlechain-setup.sh new file mode 100755 index 0000000..70c1044 --- /dev/null +++ b/e2e-tests/local-testnet-singlechain-setup.sh @@ -0,0 +1,228 @@ +#!/bin/bash +## This script sets up the environment to run the single chain local testnet. +## Importantly, it does not actually start nodes (or cometmock) - instead, +## it will produce two scripts, start_apps.sh and start_cometmock.sh. +## After this script is done setting up, simply run these two scripts to run the +## testnet. +## The reason for this is that we want to be able to make the testnet setup +## differentiated from the actual run to allow for better caching in Docker. + +set -eux + +BINARY_NAME=$1 + +# User balance of stake tokens +USER_COINS="100000000000stake" +# Amount of stake tokens staked +STAKE="100000000stake" +# Node IP address +NODE_IP="127.0.0.1" + +# Home directory +HOME_DIR=$HOME + +rm -rf ./start_apps.sh +rm -rf ./start_cometmock.sh + +# Validator moniker +MONIKERS=("coordinator" "alice" "bob") +LEAD_VALIDATOR_MONIKER="coordinator" + +PROV_NODES_ROOT_DIR=${HOME_DIR}/nodes/provider +CONS_NODES_ROOT_DIR=${HOME_DIR}/nodes/consumer + +# Base port. Ports assigned after these ports sequentially by nodes. +RPC_LADDR_BASEPORT=29170 +P2P_LADDR_BASEPORT=29180 +GRPC_LADDR_BASEPORT=29190 +NODE_ADDRESS_BASEPORT=29200 +PPROF_LADDR_BASEPORT=29210 +CLIENT_BASEPORT=29220 + +# keeps a comma separated list of node addresses for provider and consumer +PROVIDER_NODE_LISTEN_ADDR_STR="" +CONSUMER_NODE_LISTEN_ADDR_STR="" + +# Strings that keep the homes of provider nodes and homes of consumer nodes +PROV_NODES_HOME_STR="" +CONS_NODES_HOME_STR="" + +PROVIDER_COMETMOCK_ADDR=tcp://$NODE_IP:22331 +CONSUMER_COMETMOCK_ADDR=tcp://$NODE_IP:22332 + +# Clean start +pkill -f ^$BINARY_NAME &> /dev/null || true +pkill -f ^cometmock &> /dev/null || true +sleep 1 +rm -rf ${PROV_NODES_ROOT_DIR} +rm -rf ${CONS_NODES_ROOT_DIR} + +# Let lead validator create genesis file +LEAD_VALIDATOR_PROV_DIR=${PROV_NODES_ROOT_DIR}/provider-${LEAD_VALIDATOR_MONIKER} +LEAD_VALIDATOR_CONS_DIR=${CONS_NODES_ROOT_DIR}/consumer-${LEAD_VALIDATOR_MONIKER} +LEAD_PROV_KEY=${LEAD_VALIDATOR_MONIKER}-key +LEAD_PROV_LISTEN_ADDR=tcp://${NODE_IP}:${RPC_LADDR_BASEPORT} + +for index in "${!MONIKERS[@]}" +do + MONIKER=${MONIKERS[$index]} + # validator key + PROV_KEY=${MONIKER}-key + + # home directory of this validator on provider + PROV_NODE_DIR=${PROV_NODES_ROOT_DIR}/provider-${MONIKER} + + # home directory of this validator on consumer + CONS_NODE_DIR=${CONS_NODES_ROOT_DIR}/consumer-${MONIKER} + + # Build genesis file and node directory structure + $BINARY_NAME init $MONIKER --chain-id provider --home ${PROV_NODE_DIR} + jq ".app_state.gov.params.voting_period = \"100000s\" | .app_state.staking.params.unbonding_time = \"86400s\" | .app_state.slashing.params.signed_blocks_window=\"1000\" " \ + ${PROV_NODE_DIR}/config/genesis.json > \ + ${PROV_NODE_DIR}/edited_genesis.json && mv ${PROV_NODE_DIR}/edited_genesis.json ${PROV_NODE_DIR}/config/genesis.json + + + sleep 1 + + # Create account keypair + $BINARY_NAME keys add $PROV_KEY --home ${PROV_NODE_DIR} --keyring-backend test --output json > ${PROV_NODE_DIR}/${PROV_KEY}.json 2>&1 + sleep 1 + + # copy genesis in, unless this validator is the lead validator + if [ $MONIKER != $LEAD_VALIDATOR_MONIKER ]; then + cp ${LEAD_VALIDATOR_PROV_DIR}/config/genesis.json ${PROV_NODE_DIR}/config/genesis.json + fi + + # Add stake to user + PROV_ACCOUNT_ADDR=$(jq -r '.address' ${PROV_NODE_DIR}/${PROV_KEY}.json) + $BINARY_NAME genesis add-genesis-account $PROV_ACCOUNT_ADDR $USER_COINS --home ${PROV_NODE_DIR} --keyring-backend test + sleep 1 + + # copy genesis out, unless this validator is the lead validator + if [ $MONIKER != $LEAD_VALIDATOR_MONIKER ]; then + cp ${PROV_NODE_DIR}/config/genesis.json ${LEAD_VALIDATOR_PROV_DIR}/config/genesis.json + fi + + PPROF_LADDR=${NODE_IP}:$(($PPROF_LADDR_BASEPORT + $index)) + P2P_LADDR_PORT=$(($P2P_LADDR_BASEPORT + $index)) + + # adjust configs of this node + sed -i -r 's/timeout_commit = "5s"/timeout_commit = "3s"/g' ${PROV_NODE_DIR}/config/config.toml + sed -i -r 's/timeout_propose = "3s"/timeout_propose = "1s"/g' ${PROV_NODE_DIR}/config/config.toml + + # make address book non-strict. necessary for this setup + sed -i -r 's/addr_book_strict = true/addr_book_strict = false/g' ${PROV_NODE_DIR}/config/config.toml + + # avoid port double binding + sed -i -r "s/pprof_laddr = \"localhost:6060\"/pprof_laddr = \"${PPROF_LADDR}\"/g" ${PROV_NODE_DIR}/config/config.toml + + # allow duplicate IP addresses (all nodes are on the same machine) + sed -i -r 's/allow_duplicate_ip = false/allow_duplicate_ip = true/g' ${PROV_NODE_DIR}/config/config.toml +done + +for MONIKER in "${MONIKERS[@]}" +do + # validator key + PROV_KEY=${MONIKER}-key + + # home directory of this validator on provider + PROV_NODE_DIR=${PROV_NODES_ROOT_DIR}/provider-${MONIKER} + + # copy genesis in, unless this validator is the lead validator + if [ $MONIKER != $LEAD_VALIDATOR_MONIKER ]; then + cp ${LEAD_VALIDATOR_PROV_DIR}/config/genesis.json* ${PROV_NODE_DIR}/config/genesis.json + fi + + # Stake 1/1000 user's coins + $BINARY_NAME genesis gentx $PROV_KEY $STAKE --chain-id provider --home ${PROV_NODE_DIR} --keyring-backend test --moniker $MONIKER + sleep 1 + + # Copy gentxs to the lead validator for possible future collection. + # Obviously we don't need to copy the first validator's gentx to itself + if [ $MONIKER != $LEAD_VALIDATOR_MONIKER ]; then + cp ${PROV_NODE_DIR}/config/gentx/* ${LEAD_VALIDATOR_PROV_DIR}/config/gentx/ + fi +done + +# Collect genesis transactions with lead validator +$BINARY_NAME genesis collect-gentxs --home ${LEAD_VALIDATOR_PROV_DIR} --gentx-dir ${LEAD_VALIDATOR_PROV_DIR}/config/gentx/ + +sleep 1 + +START_COMMANDS="" +for index in "${!MONIKERS[@]}" +do + MONIKER=${MONIKERS[$index]} + + PERSISTENT_PEERS="" + + for peer_index in "${!MONIKERS[@]}" + do + if [ $index == $peer_index ]; then + continue + fi + PEER_MONIKER=${MONIKERS[$peer_index]} + + PEER_PROV_NODE_DIR=${PROV_NODES_ROOT_DIR}/provider-${PEER_MONIKER} + + PEER_NODE_ID=$($BINARY_NAME tendermint show-node-id --home ${PEER_PROV_NODE_DIR}) + + PEER_P2P_LADDR_PORT=$(($P2P_LADDR_BASEPORT + $peer_index)) + PERSISTENT_PEERS="$PERSISTENT_PEERS,$PEER_NODE_ID@${NODE_IP}:${PEER_P2P_LADDR_PORT}" + done + + # remove trailing comma from persistent peers + PERSISTENT_PEERS=${PERSISTENT_PEERS:1} + + # validator key + PROV_KEY=${MONIKER}-key + + # home directory of this validator on provider + PROV_NODE_DIR=${PROV_NODES_ROOT_DIR}/provider-${MONIKER} + + # home directory of this validator on consumer + CONS_NODE_DIR=${PROV_NODES_ROOT_DIR}/consumer-${MONIKER} + + # copy genesis in, unless this validator is already the lead validator and thus it already has its genesis + if [ $MONIKER != $LEAD_VALIDATOR_MONIKER ]; then + cp ${LEAD_VALIDATOR_PROV_DIR}/config/genesis.json ${PROV_NODE_DIR}/config/genesis.json + fi + + # enable vote extensions by setting .consesnsus.params.abci.vote_extensions_enable_height to 1, but 1 does not work currently - set it to 2 instead. see https://github.com/cosmos/cosmos-sdk/issues/18029#issuecomment-1754598598 + jq ".consensus.params.abci.vote_extensions_enable_height = \"2\"" ${PROV_NODE_DIR}/config/genesis.json > ${PROV_NODE_DIR}/edited_genesis.json && mv ${PROV_NODE_DIR}/edited_genesis.json ${PROV_NODE_DIR}/config/genesis.json + + RPC_LADDR_PORT=$(($RPC_LADDR_BASEPORT + $index)) + P2P_LADDR_PORT=$(($P2P_LADDR_BASEPORT + $index)) + GRPC_LADDR_PORT=$(($GRPC_LADDR_BASEPORT + $index)) + NODE_ADDRESS_PORT=$(($NODE_ADDRESS_BASEPORT + $index)) + + PROVIDER_NODE_LISTEN_ADDR_STR="${NODE_IP}:${NODE_ADDRESS_PORT},$PROVIDER_NODE_LISTEN_ADDR_STR" + PROV_NODES_HOME_STR="${PROV_NODE_DIR},$PROV_NODES_HOME_STR" + + rm -rf ${PROV_NODES_ROOT_DIR}_bkup + cp -r ${PROV_NODES_ROOT_DIR} ${PROV_NODES_ROOT_DIR}_bkup + + # Start gaia + echo $BINARY_NAME start \ + --home ${PROV_NODE_DIR} \ + --transport=grpc --with-tendermint=false \ + --p2p.persistent_peers ${PERSISTENT_PEERS} \ + --rpc.laddr $PROVIDER_COMETMOCK_ADDR \ + --grpc.address ${NODE_IP}:${GRPC_LADDR_PORT} \ + --address tcp://${NODE_IP}:${NODE_ADDRESS_PORT} \ + --p2p.laddr tcp://${NODE_IP}:${P2P_LADDR_PORT} \ + --grpc-web.enable=false "&> ${PROV_NODE_DIR}/logs &" | tee -a start_apps.sh + + sleep 5 +done + +PROVIDER_NODE_LISTEN_ADDR_STR=${PROVIDER_NODE_LISTEN_ADDR_STR::${#PROVIDER_NODE_LISTEN_ADDR_STR}-1} +PROV_NODES_HOME_STR=${PROV_NODES_HOME_STR::${#PROV_NODES_HOME_STR}-1} + +echo "Testnet applications are set up! Starting CometMock..." +echo cometmock \$1 $PROVIDER_NODE_LISTEN_ADDR_STR ${LEAD_VALIDATOR_PROV_DIR}/config/genesis.json $PROVIDER_COMETMOCK_ADDR $PROV_NODES_HOME_STR grpc "&> ${LEAD_VALIDATOR_PROV_DIR}/cometmock_log &" | tee -a start_cometmock.sh + +chmod +x start_apps.sh +chmod +x start_cometmock.sh + +# cometmock $PROVIDER_NODE_LISTEN_ADDR_STR ${LEAD_VALIDATOR_PROV_DIR}/config/genesis.json $PROVIDER_COMETMOCK_ADDR $PROV_NODES_HOME_STR grpc $COMETMOCK_ARGS &> ${LEAD_VALIDATOR_PROV_DIR}/cometmock_log & \ No newline at end of file diff --git a/e2e-tests/local-testnet-singlechain-start.sh b/e2e-tests/local-testnet-singlechain-start.sh new file mode 100755 index 0000000..bad0414 --- /dev/null +++ b/e2e-tests/local-testnet-singlechain-start.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +## Starts the testnet, assuming that the scripts generated by local-testnet-singlechain-setup.sh +## are already present in the current directory. + +COMETMOCK_ARGS=$1 + +./start_apps.sh +./start_cometmock.sh "$1" \ No newline at end of file diff --git a/e2e-tests/local-testnet-singlechain.sh b/e2e-tests/local-testnet-singlechain.sh index b3b4b4b..e2acc66 100755 --- a/e2e-tests/local-testnet-singlechain.sh +++ b/e2e-tests/local-testnet-singlechain.sh @@ -1,213 +1,16 @@ #!/bin/bash -set -eux - -BINARY_NAME=$1 - -# User balance of stake tokens -USER_COINS="100000000000stake" -# Amount of stake tokens staked -STAKE="100000000stake" -# Node IP address -NODE_IP="127.0.0.1" - -# Home directory -HOME_DIR=$HOME - -# Validator moniker -MONIKERS=("coordinator" "alice" "bob") -LEAD_VALIDATOR_MONIKER="coordinator" - -PROV_NODES_ROOT_DIR=${HOME_DIR}/nodes/provider -CONS_NODES_ROOT_DIR=${HOME_DIR}/nodes/consumer - -# Base port. Ports assigned after these ports sequentially by nodes. -RPC_LADDR_BASEPORT=29170 -P2P_LADDR_BASEPORT=29180 -GRPC_LADDR_BASEPORT=29190 -NODE_ADDRESS_BASEPORT=29200 -PPROF_LADDR_BASEPORT=29210 -CLIENT_BASEPORT=29220 - -# keeps a comma separated list of node addresses for provider and consumer -PROVIDER_NODE_LISTEN_ADDR_STR="" -CONSUMER_NODE_LISTEN_ADDR_STR="" - -# Strings that keep the homes of provider nodes and homes of consumer nodes -PROV_NODES_HOME_STR="" -CONS_NODES_HOME_STR="" - -PROVIDER_COMETMOCK_ADDR=tcp://$NODE_IP:22331 -CONSUMER_COMETMOCK_ADDR=tcp://$NODE_IP:22332 - -# Clean start -pkill -f ^$BINARY_NAME &> /dev/null || true -pkill -f ^cometmock &> /dev/null || true -sleep 1 -rm -rf ${PROV_NODES_ROOT_DIR} -rm -rf ${CONS_NODES_ROOT_DIR} - -# Let lead validator create genesis file -LEAD_VALIDATOR_PROV_DIR=${PROV_NODES_ROOT_DIR}/provider-${LEAD_VALIDATOR_MONIKER} -LEAD_VALIDATOR_CONS_DIR=${CONS_NODES_ROOT_DIR}/consumer-${LEAD_VALIDATOR_MONIKER} -LEAD_PROV_KEY=${LEAD_VALIDATOR_MONIKER}-key -LEAD_PROV_LISTEN_ADDR=tcp://${NODE_IP}:${RPC_LADDR_BASEPORT} - -for index in "${!MONIKERS[@]}" -do - MONIKER=${MONIKERS[$index]} - # validator key - PROV_KEY=${MONIKER}-key - - # home directory of this validator on provider - PROV_NODE_DIR=${PROV_NODES_ROOT_DIR}/provider-${MONIKER} - - # home directory of this validator on consumer - CONS_NODE_DIR=${CONS_NODES_ROOT_DIR}/consumer-${MONIKER} - - # Build genesis file and node directory structure - $BINARY_NAME init $MONIKER --chain-id provider --home ${PROV_NODE_DIR} - jq ".app_state.gov.params.voting_period = \"100000s\" | .app_state.staking.params.unbonding_time = \"86400s\" | .app_state.slashing.params.signed_blocks_window=\"1000\" " \ - ${PROV_NODE_DIR}/config/genesis.json > \ - ${PROV_NODE_DIR}/edited_genesis.json && mv ${PROV_NODE_DIR}/edited_genesis.json ${PROV_NODE_DIR}/config/genesis.json - - - sleep 1 - - # Create account keypair - $BINARY_NAME keys add $PROV_KEY --home ${PROV_NODE_DIR} --keyring-backend test --output json > ${PROV_NODE_DIR}/${PROV_KEY}.json 2>&1 - sleep 1 - - # copy genesis in, unless this validator is the lead validator - if [ $MONIKER != $LEAD_VALIDATOR_MONIKER ]; then - cp ${LEAD_VALIDATOR_PROV_DIR}/config/genesis.json ${PROV_NODE_DIR}/config/genesis.json - fi - - # Add stake to user - PROV_ACCOUNT_ADDR=$(jq -r '.address' ${PROV_NODE_DIR}/${PROV_KEY}.json) - $BINARY_NAME genesis add-genesis-account $PROV_ACCOUNT_ADDR $USER_COINS --home ${PROV_NODE_DIR} --keyring-backend test - sleep 1 - - # copy genesis out, unless this validator is the lead validator - if [ $MONIKER != $LEAD_VALIDATOR_MONIKER ]; then - cp ${PROV_NODE_DIR}/config/genesis.json ${LEAD_VALIDATOR_PROV_DIR}/config/genesis.json - fi - - PPROF_LADDR=${NODE_IP}:$(($PPROF_LADDR_BASEPORT + $index)) - P2P_LADDR_PORT=$(($P2P_LADDR_BASEPORT + $index)) - - # adjust configs of this node - sed -i -r 's/timeout_commit = "5s"/timeout_commit = "3s"/g' ${PROV_NODE_DIR}/config/config.toml - sed -i -r 's/timeout_propose = "3s"/timeout_propose = "1s"/g' ${PROV_NODE_DIR}/config/config.toml - - # make address book non-strict. necessary for this setup - sed -i -r 's/addr_book_strict = true/addr_book_strict = false/g' ${PROV_NODE_DIR}/config/config.toml - - # avoid port double binding - sed -i -r "s/pprof_laddr = \"localhost:6060\"/pprof_laddr = \"${PPROF_LADDR}\"/g" ${PROV_NODE_DIR}/config/config.toml - # allow duplicate IP addresses (all nodes are on the same machine) - sed -i -r 's/allow_duplicate_ip = false/allow_duplicate_ip = true/g' ${PROV_NODE_DIR}/config/config.toml -done - -for MONIKER in "${MONIKERS[@]}" -do - # validator key - PROV_KEY=${MONIKER}-key - - # home directory of this validator on provider - PROV_NODE_DIR=${PROV_NODES_ROOT_DIR}/provider-${MONIKER} - - # copy genesis in, unless this validator is the lead validator - if [ $MONIKER != $LEAD_VALIDATOR_MONIKER ]; then - cp ${LEAD_VALIDATOR_PROV_DIR}/config/genesis.json* ${PROV_NODE_DIR}/config/genesis.json - fi - - # Stake 1/1000 user's coins - $BINARY_NAME genesis gentx $PROV_KEY $STAKE --chain-id provider --home ${PROV_NODE_DIR} --keyring-backend test --moniker $MONIKER - sleep 1 - - # Copy gentxs to the lead validator for possible future collection. - # Obviously we don't need to copy the first validator's gentx to itself - if [ $MONIKER != $LEAD_VALIDATOR_MONIKER ]; then - cp ${PROV_NODE_DIR}/config/gentx/* ${LEAD_VALIDATOR_PROV_DIR}/config/gentx/ - fi -done - -# Collect genesis transactions with lead validator -$BINARY_NAME genesis collect-gentxs --home ${LEAD_VALIDATOR_PROV_DIR} --gentx-dir ${LEAD_VALIDATOR_PROV_DIR}/config/gentx/ - -sleep 1 - - -for index in "${!MONIKERS[@]}" -do - MONIKER=${MONIKERS[$index]} - - PERSISTENT_PEERS="" - - for peer_index in "${!MONIKERS[@]}" - do - if [ $index == $peer_index ]; then - continue - fi - PEER_MONIKER=${MONIKERS[$peer_index]} - - PEER_PROV_NODE_DIR=${PROV_NODES_ROOT_DIR}/provider-${PEER_MONIKER} - - PEER_NODE_ID=$($BINARY_NAME tendermint show-node-id --home ${PEER_PROV_NODE_DIR}) - - PEER_P2P_LADDR_PORT=$(($P2P_LADDR_BASEPORT + $peer_index)) - PERSISTENT_PEERS="$PERSISTENT_PEERS,$PEER_NODE_ID@${NODE_IP}:${PEER_P2P_LADDR_PORT}" - done - - # remove trailing comma from persistent peers - PERSISTENT_PEERS=${PERSISTENT_PEERS:1} - - # validator key - PROV_KEY=${MONIKER}-key - - # home directory of this validator on provider - PROV_NODE_DIR=${PROV_NODES_ROOT_DIR}/provider-${MONIKER} - - # home directory of this validator on consumer - CONS_NODE_DIR=${PROV_NODES_ROOT_DIR}/consumer-${MONIKER} - - # copy genesis in, unless this validator is already the lead validator and thus it already has its genesis - if [ $MONIKER != $LEAD_VALIDATOR_MONIKER ]; then - cp ${LEAD_VALIDATOR_PROV_DIR}/config/genesis.json ${PROV_NODE_DIR}/config/genesis.json - fi - - # enable vote extensions by setting .consesnsus.params.abci.vote_extensions_enable_height to 1, but 1 does not work currently - set it to 2 instead. see https://github.com/cosmos/cosmos-sdk/issues/18029#issuecomment-1754598598 - jq ".consensus.params.abci.vote_extensions_enable_height = \"2\"" ${PROV_NODE_DIR}/config/genesis.json > ${PROV_NODE_DIR}/edited_genesis.json && mv ${PROV_NODE_DIR}/edited_genesis.json ${PROV_NODE_DIR}/config/genesis.json - - RPC_LADDR_PORT=$(($RPC_LADDR_BASEPORT + $index)) - P2P_LADDR_PORT=$(($P2P_LADDR_BASEPORT + $index)) - GRPC_LADDR_PORT=$(($GRPC_LADDR_BASEPORT + $index)) - NODE_ADDRESS_PORT=$(($NODE_ADDRESS_BASEPORT + $index)) - - PROVIDER_NODE_LISTEN_ADDR_STR="${NODE_IP}:${NODE_ADDRESS_PORT},$PROVIDER_NODE_LISTEN_ADDR_STR" - PROV_NODES_HOME_STR="${PROV_NODE_DIR},$PROV_NODES_HOME_STR" - - # Start gaia - $BINARY_NAME start \ - --home ${PROV_NODE_DIR} \ - --transport=grpc --with-tendermint=false \ - --p2p.persistent_peers ${PERSISTENT_PEERS} \ - --rpc.laddr $PROVIDER_COMETMOCK_ADDR \ - --grpc.address ${NODE_IP}:${GRPC_LADDR_PORT} \ - --address tcp://${NODE_IP}:${NODE_ADDRESS_PORT} \ - --p2p.laddr tcp://${NODE_IP}:${P2P_LADDR_PORT} \ - --grpc-web.enable=false &> ${PROV_NODE_DIR}/logs & - - sleep 5 -done +## This script sets up the local testnet and starts it. +## To run this, both the application binary and cometmock must be installed. +set -eux -PROVIDER_NODE_LISTEN_ADDR_STR=${PROVIDER_NODE_LISTEN_ADDR_STR::${#PROVIDER_NODE_LISTEN_ADDR_STR}-1} -PROV_NODES_HOME_STR=${PROV_NODES_HOME_STR::${#PROV_NODES_HOME_STR}-1} +parent_path=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P ) +pushd "$parent_path" -echo "Testnet applications are set up! Starting CometMock..." -echo cometmock $PROVIDER_NODE_LISTEN_ADDR_STR ${LEAD_VALIDATOR_PROV_DIR}/config/genesis.json $PROVIDER_COMETMOCK_ADDR $PROV_NODES_HOME_STR grpc +BINARY_NAME=$1 -cometmock $PROVIDER_NODE_LISTEN_ADDR_STR ${LEAD_VALIDATOR_PROV_DIR}/config/genesis.json $PROVIDER_COMETMOCK_ADDR $PROV_NODES_HOME_STR grpc &> ${LEAD_VALIDATOR_PROV_DIR}/cometmock_log & +COMETMOCK_ARGS=$2 -sleep 5 \ No newline at end of file +# set up the net +./local-testnet-singlechain-setup.sh $BINARY_NAME "$COMETMOCK_ARGS" +./local-testnet-singlechain-start.sh diff --git a/e2e-tests/main_test.go b/e2e-tests/main_test.go index 57368b6..95c672d 100644 --- a/e2e-tests/main_test.go +++ b/e2e-tests/main_test.go @@ -1,66 +1,32 @@ package main import ( - "bytes" "encoding/json" "fmt" + "math/big" "os/exec" - "strconv" "testing" "time" -) - -func runCommandWithOutput(cmd *exec.Cmd) (string, error) { - var stdout, stderr bytes.Buffer - cmd.Stdout = &stdout - cmd.Stderr = &stderr - - err := cmd.Run() - if err != nil { - return "", fmt.Errorf("error running command: %v\nstdout: %s\nstderr: %s", err, stdout.String(), stderr.String()) - } - - return stdout.String(), nil -} - -// From the output of the AbciInfo command, extract the latest block height. -// The json bytes should look e.g. like this: -// {"jsonrpc":"2.0","id":1,"result":{"response":{"data":"interchain-security-p","last_block_height":"2566","last_block_app_hash":"R4Q3Si7+t7TIidl2oTHcQRDNEz+lP0IDWhU5OI89psg="}}} -func extractHeightFromInfo(jsonBytes []byte) (int, error) { - // Use a generic map to represent the JSON structure - var data map[string]interface{} - - if err := json.Unmarshal(jsonBytes, &data); err != nil { - return -1, fmt.Errorf("Failed to unmarshal JSON %s \n error was %v", string(jsonBytes), err) - } - - // Navigate the map and use type assertions to get the last_block_height - result, ok := data["result"].(map[string]interface{}) - if !ok { - return -1, fmt.Errorf("Failed to navigate abci_info output structure trying to access result: json was %s", string(jsonBytes)) - } - - response, ok := result["response"].(map[string]interface{}) - if !ok { - return -1, fmt.Errorf("Failed to navigate abci_info output structure trying to access response: json was %s", string(jsonBytes)) - } - - lastBlockHeight, ok := response["last_block_height"].(string) - if !ok { - return -1, fmt.Errorf("Failed to navigate abci_info output structure trying to access last_block_height: json was %s", string(jsonBytes)) - } - return strconv.Atoi(lastBlockHeight) -} + "github.com/stretchr/testify/require" +) -// Tests happy path functionality for Abci Info. -func TestAbciInfo(t *testing.T) { +func StartChain( + t *testing.T, + cometmockArgs string, +) error { // execute the local-testnet-singlechain.sh script t.Log("Running local-testnet-singlechain.sh") - cmd := exec.Command("./local-testnet-singlechain.sh", "simd") + cmd := exec.Command("./local-testnet-singlechain-restart.sh", "simd") _, err := runCommandWithOutput(cmd) if err != nil { - t.Fatalf("Error running local-testnet-singlechain.sh: %v", err) + return fmt.Errorf("Error running local-testnet-singlechain.sh: %v", err) + } + + cmd = exec.Command("./local-testnet-singlechain-start.sh", cometmockArgs) + _, err = runCommandWithOutput(cmd) + if err != nil { + return fmt.Errorf("Error running local-testnet-singlechain.sh: %v", err) } t.Log("Done starting testnet") @@ -77,11 +43,22 @@ func TestAbciInfo(t *testing.T) { t.Log("Waiting for blocks to be produced, latest output: ", string(out)) time.Sleep(1 * time.Second) } + time.Sleep(5 * time.Second) + return nil +} + +// Tests happy path functionality for Abci Info. +func TestAbciInfo(t *testing.T) { + // start the chain + err := StartChain(t, "") + if err != nil { + t.Fatalf("Error starting chain: %v", err) + } // call the abci_info command by calling curl on the REST endpoint // curl -H 'Content-Type: application/json' -H 'Accept:application/json' --data '{"jsonrpc":"2.0","method":"abci_info","id":1}' 127.0.0.1:22331 args := []string{"bash", "-c", "curl -H 'Content-Type: application/json' -H 'Accept:application/json' --data '{\"jsonrpc\":\"2.0\",\"method\":\"abci_info\",\"id\":1}' 127.0.0.1:22331"} - cmd = exec.Command(args[0], args[1:]...) + cmd := exec.Command(args[0], args[1:]...) out, err := runCommandWithOutput(cmd) if err != nil { t.Fatalf("Error running curl\ncommand: %v\noutput: %v\nerror: %v", cmd, string(out), err) @@ -114,3 +91,246 @@ func TestAbciInfo(t *testing.T) { t.Fatalf("Expected block height to increase, but it did not. First height was %v, second height was %v", height, height2) } } + +func TestAbciQuery(t *testing.T) { + // start the chain + err := StartChain(t, "") + if err != nil { + t.Fatalf("Error starting chain: %v", err) + } + + // call the abci_query command by submitting a query that hits the AbciQuery endpoint + // for simplicity, we query for the staking params here - any query would work, + // but ones without arguments are easier to construct + args := []string{"bash", "-c", "simd q staking params --node tcp://127.0.0.1:22331 --output json"} + cmd := exec.Command(args[0], args[1:]...) + out, err := runCommandWithOutput(cmd) + if err != nil { + t.Fatalf("Error running command: %v\noutput: %v\nerror: %v", cmd, string(out), err) + } + + // check that the output is valid JSON + var data map[string]interface{} + if err := json.Unmarshal([]byte(out), &data); err != nil { + t.Fatalf("Failed to unmarshal JSON %s \n error was %v", string(out), err) + } + + // check that the output contains the expected params field. its contents are not important + _, ok := data["params"] + if !ok { + t.Fatalf("Expected output to contain params field, but it did not. Output was %s", string(out)) + } +} + +func TestTx(t *testing.T) { + err := StartChain(t, "") + if err != nil { + t.Fatalf("Error starting chain: %v", err) + } + + // check the current amount in the community pool + communityPoolSize, err := getCommunityPoolSize() + require.NoError(t, err) + + // send some tokens to the community pool + err = sendToCommunityPool(50000000000, "coordinator") + require.NoError(t, err) + + // check that the amount in the community pool has increased + communityPoolSize2, err := getCommunityPoolSize() + require.NoError(t, err) + + // cannot check for equality because the community pool gets dust over time + require.True(t, communityPoolSize2.Cmp(communityPoolSize.Add(communityPoolSize, big.NewInt(50000000000))) == +1) +} + +// TestBlockTime checks that the basic behaviour with a specified block-time is as expected, +// i.e. the time increases by the specified block time for each block. +func TestBlockTime(t *testing.T) { + err := StartChain(t, "--block-time=5000") + if err != nil { + t.Fatalf("Error starting chain: %v", err) + } + + // get a block with height+time + blockString, err := QueryBlock() + require.NoError(t, err) + + // get the height and time from the block + height, err := GetHeightFromBlock(blockString) + require.NoError(t, err) + + blockTime, err := GetTimeFromBlock(blockString) + require.NoError(t, err) + + // wait for a couple of blocks to be produced + time.Sleep(10 * time.Second) + + // get the new height and time + blockString2, err := QueryBlock() + require.NoError(t, err) + + height2, err := GetHeightFromBlock(blockString2) + require.NoError(t, err) + + blockTime2, err := GetTimeFromBlock(blockString2) + require.NoError(t, err) + + blockDifference := height2 - height + // we expect that at least one block was produced, otherwise there is a problem + require.True(t, blockDifference >= 1) + + // get the expected time diff between blocks, as block time was set to 5000 millis = 5 seconds + expectedTimeDifference := time.Duration(blockDifference) * 5 * time.Second + + timeDifference := blockTime2.Sub(blockTime) + + require.Equal(t, expectedTimeDifference, timeDifference) +} + +// TestAutoBlockProductionOff checks that the basic behaviour with +// block-production-interval is as expected, i.e. blocks only +// appear when it is manually instructed. +func TestNoAutoBlockProduction(t *testing.T) { + err := StartChain(t, "--block-production-interval=-1 --block-time=0") + if err != nil { + t.Fatalf("Error starting chain: %v", err) + } + + height, blockTime, err := GetHeightAndTime() + require.NoError(t, err) + + // wait a few seconds to detect it blocks are produced automatically + time.Sleep(10 * time.Second) + + // get the new height and time + height2, blockTime2, err := GetHeightAndTime() + require.NoError(t, err) + + // no blocks should have been produced + require.Equal(t, height, height2) + require.Equal(t, blockTime, blockTime2) + + // advance time by 5 seconds + err = AdvanceTime(5 * time.Second) + require.NoError(t, err) + + // get the height and time again, they should not have changed yet + height3, blockTime3, err := GetHeightAndTime() + require.NoError(t, err) + + require.Equal(t, height, height3) + require.Equal(t, blockTime, blockTime3) + + // produce a block + err = AdvanceBlocks(1) + require.NoError(t, err) + + // get the height and time again, they should have changed + height4, blockTime4, err := GetHeightAndTime() + require.NoError(t, err) + + require.Equal(t, height+1, height4) + require.Equal(t, blockTime.Add(5*time.Second), blockTime4) +} + +// TestNoAutoTx checks that without auto-tx, transactions are not included +// in blocks automatically. +func TestNoAutoTx(t *testing.T) { + err := StartChain(t, "--block-production-interval=-1 --auto-tx=false") + if err != nil { + t.Fatalf("Error starting chain: %v", err) + } + + // produce a couple of blocks to initialize the community pool + err = AdvanceBlocks(10) + require.NoError(t, err) + + height, blockTime, err := GetHeightAndTime() + require.NoError(t, err) + + communityPoolBefore, err := getCommunityPoolSize() + require.NoError(t, err) + + // broadcast txs + err = sendToCommunityPool(50000000000, "coordinator") + require.NoError(t, err) + err = sendToCommunityPool(50000000000, "bob") + require.NoError(t, err) + + // get the new height and time + height2, blockTime2, err := GetHeightAndTime() + require.NoError(t, err) + + // no blocks should have been produced + require.Equal(t, height, height2) + require.Equal(t, blockTime, blockTime2) + + // produce a block + err = AdvanceBlocks(1) + require.NoError(t, err) + + // get the height and time again, they should have changed + height3, blockTime3, err := GetHeightAndTime() + require.NoError(t, err) + + require.Equal(t, height+1, height3) + // exact time does not matter, just that it is after the previous block + require.True(t, blockTime.Before(blockTime3)) + + // check that the community pool was increased + communityPoolAfter, err := getCommunityPoolSize() + require.NoError(t, err) + + // cannot check for equality because the community pool gets dust over time + require.True(t, communityPoolAfter.Cmp(communityPoolBefore.Add(communityPoolBefore, big.NewInt(100000000000))) == +1) +} + +func TestStartingTimestamp(t *testing.T) { + err := StartChain(t, "--block-production-interval=-1 --auto-tx=false --starting-timestamp=0 --block-time=1") + if err != nil { + t.Fatalf("Error starting chain: %v", err) + } + + // produce a couple of blocks + err = AdvanceBlocks(10) + require.NoError(t, err) + + // get the time + _, blockTime, err := GetHeightAndTime() + require.NoError(t, err) + + // the time should be starting-timestamp + 10 * blockTime + 1 (for the first block needed after Genesis) + startingTimestamp := time.Unix(0, 0) + expectedTime := startingTimestamp.Add(11 * time.Millisecond) + + require.True(t, expectedTime.Compare(blockTime) == 0, "expectedTime: %v, blockTime: %v", expectedTime, blockTime) +} + +func TestSystemStartingTime(t *testing.T) { + err := StartChain(t, "--block-production-interval=-1 --auto-tx=false --starting-timestamp=-1 --block-time=1") + if err != nil { + t.Fatalf("Error starting chain: %v", err) + } + startingTime := time.Now() + + // produce a couple of blocks + err = AdvanceBlocks(10) + require.NoError(t, err) + + // get the time + _, blockTime, err := GetHeightAndTime() + require.NoError(t, err) + + // the time should be starting-timestamp + 10 * blockTime + 1 (for the first block needed after Genesis) + expectedTime := startingTime.Add(11 * time.Millisecond) + + // since the starting timestamp is taken from the system time, + // we can only check that the time is close to the expected time + // since the chain startup is hard to time exactly + delta := 30 * time.Second + + diff := expectedTime.Sub(blockTime).Abs() + + require.True(t, diff <= delta, "expectedTime: %v, blockTime: %v", expectedTime, blockTime) +} diff --git a/e2e-tests/test_utils.go b/e2e-tests/test_utils.go new file mode 100644 index 0000000..0f92b56 --- /dev/null +++ b/e2e-tests/test_utils.go @@ -0,0 +1,166 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "math/big" + "os/exec" + "strconv" + "strings" + "time" +) + +// From the output of the AbciInfo command, extract the latest block height. +// The json bytes should look e.g. like this: +// {"jsonrpc":"2.0","id":1,"result":{"response":{"data":"interchain-security-p","last_block_height":"2566","last_block_app_hash":"R4Q3Si7+t7TIidl2oTHcQRDNEz+lP0IDWhU5OI89psg="}}} +func extractHeightFromInfo(jsonBytes []byte) (int, error) { + // Use a generic map to represent the JSON structure + var data map[string]interface{} + + if err := json.Unmarshal(jsonBytes, &data); err != nil { + return -1, fmt.Errorf("failed to unmarshal JSON %s \n error was %v", string(jsonBytes), err) + } + + // Navigate the map and use type assertions to get the last_block_height + result, ok := data["result"].(map[string]interface{}) + if !ok { + return -1, fmt.Errorf("failed to navigate abci_info output structure trying to access result: json was %s", string(jsonBytes)) + } + + response, ok := result["response"].(map[string]interface{}) + if !ok { + return -1, fmt.Errorf("failed to navigate abci_info output structure trying to access response: json was %s", string(jsonBytes)) + } + + lastBlockHeight, ok := response["last_block_height"].(string) + if !ok { + return -1, fmt.Errorf("failed to navigate abci_info output structure trying to access last_block_height: json was %s", string(jsonBytes)) + } + + return strconv.Atoi(lastBlockHeight) +} + +// Queries simd for the latest block. +func QueryBlock() (string, error) { + // execute the query command + cmd := exec.Command("bash", "-c", "simd q block --type height 0 --output json --node tcp://127.0.0.1:22331 --output json") + out, err := runCommandWithOutput(cmd) + if err != nil { + return "", fmt.Errorf("error running query command: %v", err) + } + + return out, nil +} + +type BlockInfo struct { + Header struct { + Height string `json:"height"` + Time string `json:"time"` + } `json:"header"` +} + +func GetHeightFromBlock(blockString string) (int, error) { + var block BlockInfo + err := json.Unmarshal([]byte(blockString), &block) + if err != nil { + return 0, err + } + + res, err := strconv.Atoi(block.Header.Height) + if err != nil { + return 0, err + } + + return res, nil +} + +func GetTimeFromBlock(blockBytes string) (time.Time, error) { + var block BlockInfo + err := json.Unmarshal([]byte(blockBytes), &block) + if err != nil { + return time.Time{}, err + } + + res, err := time.Parse(time.RFC3339, block.Header.Time) + if err != nil { + return time.Time{}, err + } + + return res, nil +} + +func GetHeightAndTime() (int, time.Time, error) { + blockBytes, err := QueryBlock() + if err != nil { + return 0, time.Time{}, err + } + + height, err := GetHeightFromBlock(blockBytes) + if err != nil { + return 0, time.Time{}, err + } + + timestamp, err := GetTimeFromBlock(blockBytes) + if err != nil { + return 0, time.Time{}, err + } + + return height, timestamp, nil +} + +// Queries the size of the community pool. +// For this, it will just check the number of tokens of the first denom in the community pool. +func getCommunityPoolSize() (*big.Int, error) { + // execute the query command + cmd := exec.Command("bash", "-c", "simd q distribution community-pool --output json --node tcp://127.0.0.1:22331 | jq -r '.pool[0].amount'") + out, err := runCommandWithOutput(cmd) + if err != nil { + return big.NewInt(-1), fmt.Errorf("error running query command: %v", err) + } + + res := new(big.Int) + + res, ok := res.SetString(strings.TrimSpace(out), 10) + if !ok { + return big.NewInt(-1), fmt.Errorf("error parsing community pool size: %v", err) + } + return res, err +} + +func sendToCommunityPool(amount int, sender string) error { + // execute the tx command + stringCmd := fmt.Sprintf("simd tx distribution fund-community-pool %vstake --chain-id provider --from %v-key --keyring-backend test --node tcp://127.0.0.1:22331 --home ~/nodes/provider/provider-%v -y", amount, sender, sender) + cmd := exec.Command("bash", "-c", stringCmd) + _, err := runCommandWithOutput(cmd) + return err +} + +func runCommandWithOutput(cmd *exec.Cmd) (string, error) { + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + err := cmd.Run() + if err != nil { + return "", fmt.Errorf("error running command: %v\nstdout: %s\nstderr: %s", err, stdout.String(), stderr.String()) + } + + return stdout.String(), nil +} + +func AdvanceTime(duration time.Duration) error { + stringCmd := fmt.Sprintf("curl -H 'Content-Type: application/json' -H 'Accept:application/json' --data '{\"jsonrpc\":\"2.0\",\"method\":\"advance_time\",\"params\":{\"duration_in_seconds\": \"%v\"},\"id\":1}' 127.0.0.1:22331", duration.Seconds()) + + cmd := exec.Command("bash", "-c", stringCmd) + _, err := runCommandWithOutput(cmd) + return err +} + +func AdvanceBlocks(numBlocks int) error { + stringCmd := fmt.Sprintf("curl -H 'Content-Type: application/json' -H 'Accept:application/json' --data '{\"jsonrpc\":\"2.0\",\"method\":\"advance_blocks\",\"params\":{\"num_blocks\": \"%v\"},\"id\":1}' 127.0.0.1:22331", numBlocks) + + cmd := exec.Command("bash", "-c", stringCmd) + _, err := runCommandWithOutput(cmd) + return err +}