Add mTLS grpcserver (#5154)

## Motivation
Closes #5131

do not merge before spacemeshos/api#268 and spacemeshos/post#245

## Changes
- setup for gRPC servers has been moved from node startup into the `grpcserver` package
- `NewPublic`, `NewPrivate` and `NewTLS` create servers for the given purposes based on the configuration passed to them
- replaced more instances of `go-spacemesh/log` with `zap`

## Test Plan
- existing tests pass
- TODO: add new tests for mTLS connection

## TODO
<!-- This section should be removed when all items are complete -->
- [x] Explain motivation or link existing issue(s)
- [x] Test changes and document test plan
- [x] Update documentation as needed
- [x] Update [changelog](../CHANGELOG.md) as needed

CHANGELOG.md

@@ -18,10 +18,12 @@ See [RELEASE](./RELEASE.md) for workflow instructions.
 
   > 2023-10-02T15:28:14.002+0200 WARN fd68b.sync mesh failed to process layer from sync {"node_id": "fd68b9397572556c2f329f3e5af2faf23aef85dbbbb7e38447fae2f4ef38899f", "module": "sync", "sessionId": "29422935-68d6-47d1-87a8-02293aa181f3", "layer_id": 23104, "errmsg": "requested layer 8063 is before evicted 13102", "name": "sync"}
 
-* [#5091](https://github.com/spacemeshos/go-spacemesh/pull/5091) First stage of separating PoST from the node into its own service.
+* [#5091](https://github.com/spacemeshos/go-spacemesh/pull/5091) Separating PoST from the node into its own service.
 * [#5061](https://github.com/spacemeshos/go-spacemesh/pull/5061) Proof generation is now done via a dedicated service instead of the node.
+* [#5154](https://github.com/spacemeshos/go-spacemesh/pull/5154) Enable TLS connections between node and PoST service.
 
-  Operating a node doesn't require any changes at the moment. The service will be automatically started by the node if needed and will be stopped when the node is stopped.
+  PoST proofs are now done via a dedicated process / service that the node communicates with via gRPC. Smapp users can continue to smesh as they used to. The node will
+  automatically start the PoST service when it starts and will shut it down when it shuts down.
 
 * [#5138](https://github.com/spacemeshos/go-spacemesh/pull/5138) Bump poet to v0.9.7
 

Makefile-libs.Inc

@@ -50,7 +50,7 @@ else
 	endif
 endif
 
-POSTRS_SETUP_REV = 0.5.0-alpha2
+POSTRS_SETUP_REV = 0.5.0
 POSTRS_SETUP_ZIP = libpost-$(platform)-v$(POSTRS_SETUP_REV).zip
 POSTRS_SETUP_URL_ZIP ?= https://github.com/spacemeshos/post-rs/releases/download/v$(POSTRS_SETUP_REV)/$(POSTRS_SETUP_ZIP)
 POSTRS_PROFILER_ZIP = profiler-$(platform)-v$(POSTRS_SETUP_REV).zip

README.md

@@ -235,6 +235,56 @@ on Windows you can use Intel OpenAPI:
 choco install opencl-intel-cpu-runtime
 ```
 
+#### Using a remote machine as provider for PoST proofs
+
+To disable the internal PoST service and disable smeshing on your node you can use the following config:
+
+```json
+"smeshing": {
+    "smeshing-start": false,
+}
+```
+
+or use the `--smeshing-start=false` flag. This will disable smeshing on your node causing it not generate any PoST proofs until a remote post
+service connects.
+
+By default the node listens for the PoST service on `grpc-private-listener` (defaults to 127.0.0.1:9093). This endpoint does not require authentication and
+should only be accessible from the same machine. If you want to allow connections from post services on other hosts to your node, you should do so via the
+`grpc-tls-listener` (defaults to 0.0.0.0:9094) and setup TLS for the connection.
+
+This is useful for example if you want to run a node on a cloud provider with fewer resources and run PoST on a local machine with more resources. The post
+service only needs to be online for the initial proof (i.e. when joining the network for the first time) and during the cyclegap in every epoch.
+
+To setup TLS-secured public connections the API config has been extended with the following options:
+
+```json
+"api": {
+    "grpc-private-services": ["admin", "smesher"], // remove "post" from the list of services only exposed to the local machine
+    "grpc-tls-services": ["post"],                 // add "post" to the list of services that should be exposed via TLS
+    "grpc-tls-listener": "0.0.0.0:9094",           // listen address for TLS connections
+    "grpc-tls-ca-cert": "/path/to/ca.pem",         // CA certificate that signed the node's and the PoST service's certificates
+    "grpc-tls-cert": "/path/to/cert.pem",          // certificate for the node
+    "grpc-tls-key": "/path/to/key.pem",            // private key for the node
+}
+```
+
+Ensure that remote PoST services are setup to connect to your node via TLS, that they trust your node's certificate and use a certificate that is signed by the
+same CA as your node's certificate.
+
+The local (supervised) PoST service can also be configured to connect to your node via TLS if needed. The following config options are available:
+
+```json
+"post-service": {
+    "post-opts-post-service": "/path/to/service-binary", // defaults to service in the same directory as the node binary
+    "post-opts-node-address": "http://domain:port",      // defaults to 127.0.0.1:9093 - the same default value as for "grpc-private-listener"
+
+    // the following settings are mandatory when connecting to the node via TLS - when connecting via the private listener they are not needed
+    "post-opts-tls-ca-cert": "/path/to/ca.pem",  // CA certificate that signed the node's and the PoST service's certificates
+    "post-opts-tls-cert": "/path/to/cert.pem",   // certificate for the PoST service
+    "post-opts-tls-key": "/path/to/key.pem",     // private key for the PoST service
+}
+```
+
 ---
 
 ### Testing

activation/e2e/nipost_test.go

@@ -78,19 +78,19 @@ func launchServer(tb testing.TB, services ...grpcserver.ServiceAPI) (grpcserver.
 	cfg := grpcserver.DefaultTestConfig()
 
 	// run on random ports
-	grpcService := grpcserver.New("127.0.0.1:0", logtest.New(tb).Named("grpc"))
+	server := grpcserver.New("127.0.0.1:0", zaptest.NewLogger(tb).Named("grpc"), cfg)
 
 	// attach services
 	for _, svc := range services {
-		svc.RegisterService(grpcService)
+		svc.RegisterService(server.GrpcServer)
 	}
 
-	require.NoError(tb, grpcService.Start())
+	require.NoError(tb, server.Start())
 
 	// update config with bound addresses
-	cfg.PublicListener = grpcService.BoundAddress
+	cfg.PublicListener = server.BoundAddress
 
-	return cfg, func() { assert.NoError(tb, grpcService.Close()) }
+	return cfg, func() { assert.NoError(tb, server.Close()) }
 }
 
 func initPost(tb testing.TB, logger *zap.Logger, mgr *activation.PostSetupManager, opts activation.PostSetupOpts) {

activation/post_supervisor.go

@@ -47,8 +47,11 @@ func DefaultTestPostServiceConfig() PostSupervisorConfig {
 
 type PostSupervisorConfig struct {
 	PostServiceCmd string `mapstructure:"post-opts-post-service"`
+	NodeAddress    string `mapstructure:"post-opts-node-address"`
 
-	NodeAddress string `mapstructure:"post-opts-node-address"`
+	CACert string `mapstructure:"post-opts-ca-cert"`
+	Cert   string `mapstructure:"post-opts-cert"`
+	Key    string `mapstructure:"post-opts-key"`
 }
 
 // PostSupervisor manages a local post service.
@@ -153,6 +156,15 @@ func (ps *PostSupervisor) runCmd(ctx context.Context, cmdCfg PostSupervisorConfi
 			"--nonces", strconv.FormatUint(uint64(provingOpts.Nonces), 10),
 			"--randomx-mode", provingOpts.RandomXMode.String(),
 		}
+		if cmdCfg.CACert != "" {
+			args = append(args, "--ca-cert", cmdCfg.CACert)
+		}
+		if cmdCfg.Cert != "" {
+			args = append(args, "--cert", cmdCfg.Cert)
+		}
+		if cmdCfg.Key != "" {
+			args = append(args, "--key", cmdCfg.Key)
+		}
 
 		cmd := exec.CommandContext(
 			ctx,

api/grpcserver/activation_service.go

@@ -6,8 +6,10 @@ import (
 	"fmt"
 
 	"github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap"
+	"github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
 	pb "github.com/spacemeshos/api/release/go/spacemesh/v1"
 	"go.uber.org/zap"
+	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 	"google.golang.org/protobuf/types/known/emptypb"
@@ -30,8 +32,17 @@ func NewActivationService(atxProvider atxProvider, goldenAtx types.ATXID) *activ
 }
 
 // RegisterService implements ServiceAPI.
-func (s *activationService) RegisterService(server *Server) {
-	pb.RegisterActivationServiceServer(server.GrpcServer, s)
+func (s *activationService) RegisterService(server *grpc.Server) {
+	pb.RegisterActivationServiceServer(server, s)
+}
+
+func (s *activationService) RegisterHandlerService(mux *runtime.ServeMux) error {
+	return pb.RegisterActivationServiceHandlerServer(context.Background(), mux, s)
+}
+
+// String returns the service name.
+func (s *activationService) String() string {
+	return "ActivationService"
 }
 
 // Get implements v1.ActivationServiceServer.

api/grpcserver/admin_service.go

@@ -9,8 +9,10 @@ import (
 	"time"
 
 	"github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap"
+	"github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
 	pb "github.com/spacemeshos/api/release/go/spacemesh/v1"
 	"github.com/spf13/afero"
+	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/status"
@@ -53,8 +55,17 @@ func NewAdminService(db *sql.Database, dataDir string, p peers) *AdminService {
 }
 
 // RegisterService registers this service with a grpc server instance.
-func (a AdminService) RegisterService(server *Server) {
-	pb.RegisterAdminServiceServer(server.GrpcServer, a)
+func (a AdminService) RegisterService(server *grpc.Server) {
+	pb.RegisterAdminServiceServer(server, a)
+}
+
+func (s AdminService) RegisterHandlerService(mux *runtime.ServeMux) error {
+	return pb.RegisterAdminServiceHandlerServer(context.Background(), mux, s)
+}
+
+// String returns the name of this service.
+func (a AdminService) String() string {
+	return "AdminService"
 }
 
 func (a AdminService) CheckpointStream(req *pb.CheckpointStreamRequest, stream pb.AdminService_CheckpointStreamServer) error {

api/grpcserver/admin_service_test.go

@@ -19,7 +19,7 @@ import (
 
 const snapshot uint32 = 15
 
-func newatx(tb testing.TB, db *sql.Database) {
+func newAtx(tb testing.TB, db *sql.Database) {
 	atx := &types.ActivationTx{
 		InnerActivationTx: types.InnerActivationTx{
 			NIPostChallenge: types.NIPostChallenge{
@@ -32,8 +32,8 @@ func newatx(tb testing.TB, db *sql.Database) {
 		},
 	}
 	atx.SetID(types.RandomATXID())
-	vrfnonce := types.VRFPostIndex(11)
-	atx.VRFNonce = &vrfnonce
+	vrfNonce := types.VRFPostIndex(11)
+	atx.VRFNonce = &vrfNonce
 	atx.SmesherID = types.BytesToNodeID(types.RandomBytes(20))
 	atx.NodeID = &atx.SmesherID
 	atx.SetEffectiveNumUnits(atx.NumUnits)
@@ -45,7 +45,7 @@ func newatx(tb testing.TB, db *sql.Database) {
 
 func createMesh(tb testing.TB, db *sql.Database) {
 	for i := 0; i < 10; i++ {
-		newatx(tb, db)
+		newAtx(tb, db)
 	}
 	acct := &types.Account{
 		Layer: types.LayerID(0), Address: types.Address{1, 1}, NextNonce: 1, Balance: 1300, TemplateAddress: &types.Address{2}, State: []byte("state10"),
@@ -62,7 +62,7 @@ func TestAdminService_Checkpoint(t *testing.T) {
 
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
-	conn := dialGrpc(ctx, t, cfg.PublicListener)
+	conn := dialGrpc(ctx, t, cfg)
 	c := pb.NewAdminServiceClient(conn)
 
 	stream, err := c.CheckpointStream(ctx, &pb.CheckpointStreamRequest{SnapshotLayer: snapshot})
@@ -98,7 +98,7 @@ func TestAdminService_CheckpointError(t *testing.T) {
 
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
-	conn := dialGrpc(ctx, t, cfg.PublicListener)
+	conn := dialGrpc(ctx, t, cfg)
 	c := pb.NewAdminServiceClient(conn)
 
 	stream, err := c.CheckpointStream(ctx, &pb.CheckpointStreamRequest{SnapshotLayer: snapshot})
@@ -118,7 +118,7 @@ func TestAdminService_Recovery(t *testing.T) {
 
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
-	conn := dialGrpc(ctx, t, cfg.PublicListener)
+	conn := dialGrpc(ctx, t, cfg)
 	c := pb.NewAdminServiceClient(conn)
 
 	_, err := c.Recover(ctx, &pb.RecoverRequest{})

api/grpcserver/config.go

@@ -10,6 +10,11 @@ type Config struct {
 	PublicListener  string    `mapstructure:"grpc-public-listener"`
 	PrivateServices []Service `mapstructure:"grpc-private-services"`
 	PrivateListener string    `mapstructure:"grpc-private-listener"`
+	TLSServices     []Service `mapstructure:"grpc-tls-services"`
+	TLSListener     string    `mapstructure:"grpc-tls-listener"`
+	TLSCACert       string    `mapstructure:"gprc-tls-ca-cert"`
+	TLSCert         string    `mapstructure:"grpc-tls-cert"`
+	TLSKey          string    `mapstructure:"grpc-tls-key"`
 	GrpcSendMsgSize int       `mapstructure:"grpc-send-msg-size"`
 	GrpcRecvMsgSize int       `mapstructure:"grpc-recv-msg-size"`
 	JSONListener    string    `mapstructure:"grpc-json-listener"`
@@ -36,8 +41,10 @@ func DefaultConfig() Config {
 	return Config{
 		PublicServices:        []Service{Debug, GlobalState, Mesh, Transaction, Node, Activation},
 		PublicListener:        "0.0.0.0:9092",
-		PrivateServices:       []Service{Admin, Smesher, Post}, // TODO(mafa): move from private to public with authentication (probably new service category)
+		PrivateServices:       []Service{Admin, Smesher, Post},
 		PrivateListener:       "127.0.0.1:9093",
+		TLSServices:           []Service{},
+		TLSListener:           "0.0.0.0:9094",
 		JSONListener:          "",
 		GrpcSendMsgSize:       1024 * 1024 * 10,
 		GrpcRecvMsgSize:       1024 * 1024 * 10,
@@ -51,5 +58,6 @@ func DefaultTestConfig() Config {
 	conf.PublicListener = "127.0.0.1:0"
 	conf.PrivateListener = "127.0.0.1:0"
 	conf.JSONListener = "127.0.0.1:0"
+	conf.TLSListener = "127.0.0.1:0"
 	return conf
 }

api/grpcserver/debug_service.go

@@ -5,8 +5,10 @@ import (
 	"fmt"
 
 	"github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap"
+	"github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
 	pb "github.com/spacemeshos/api/release/go/spacemesh/v1"
 	"go.uber.org/zap"
+	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/status"
@@ -27,8 +29,17 @@ type DebugService struct {
 }
 
 // RegisterService registers this service with a grpc server instance.
-func (d DebugService) RegisterService(server *Server) {
-	pb.RegisterDebugServiceServer(server.GrpcServer, d)
+func (d DebugService) RegisterService(server *grpc.Server) {
+	pb.RegisterDebugServiceServer(server, d)
+}
+
+func (s DebugService) RegisterHandlerService(mux *runtime.ServeMux) error {
+	return pb.RegisterDebugServiceHandlerServer(context.Background(), mux, s)
+}
+
+// String returns the name of this service.
+func (d DebugService) String() string {
+	return "DebugService"
 }
 
 // NewDebugService creates a new grpc service using config data.

api/grpcserver/globalstate_service.go

@@ -5,8 +5,10 @@ import (
 	"fmt"
 
 	"github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap"
+	"github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
 	pb "github.com/spacemeshos/api/release/go/spacemesh/v1"
 	"go.uber.org/zap"
+	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/status"
@@ -22,8 +24,17 @@ type GlobalStateService struct {
 }
 
 // RegisterService registers this service with a grpc server instance.
-func (s GlobalStateService) RegisterService(server *Server) {
-	pb.RegisterGlobalStateServiceServer(server.GrpcServer, s)
+func (s GlobalStateService) RegisterService(server *grpc.Server) {
+	pb.RegisterGlobalStateServiceServer(server, s)
+}
+
+func (s GlobalStateService) RegisterHandlerService(mux *runtime.ServeMux) error {
+	return pb.RegisterGlobalStateServiceHandlerServer(context.Background(), mux, s)
+}
+
+// String returns the name of the service.
+func (s GlobalStateService) String() string {
+	return "GlobalStateService"
 }
 
 // NewGlobalStateService creates a new grpc service using config data.