From f2e2621be5c794f2a08dbfebc6c40d393ac43ff5 Mon Sep 17 00:00:00 2001 From: Anton Korotkov Date: Sun, 11 Aug 2024 09:34:17 +0300 Subject: [PATCH 01/20] use scan limiter --- client.go | 56 ++++++++++++++----- handler_backup.go | 8 ++- handler_backup_records.go | 24 ++++---- internal/asinfo/mocks/aerospikeClient_mock.go | 1 - internal/asinfo/mocks/infoGetter_mock.go | 1 - io/aerospike/mocks/dbWriter_mock.go | 1 - io/aerospike/mocks/scanner_mock.go | 1 - io/aerospike/record_reader.go | 21 ++++++- io/aerospike/record_reader_test.go | 6 ++ mocks/AerospikeClient_mock.go | 1 - tests/integration/integration_test.go | 22 ++++---- 11 files changed, 98 insertions(+), 44 deletions(-) diff --git a/client.go b/client.go index cee7de0b..763b495b 100644 --- a/client.go +++ b/client.go @@ -22,6 +22,7 @@ import ( a "github.com/aerospike/aerospike-client-go/v7" "github.com/aerospike/backup-go/internal/logging" + "golang.org/x/sync/semaphore" ) const ( @@ -35,7 +36,7 @@ const ( // AerospikeClient describes aerospike client interface for easy mocking. // -// go:generate mockery --name AerospikeClient +//go:generate mockery --name AerospikeClient type AerospikeClient interface { GetDefaultScanPolicy() *a.ScanPolicy GetDefaultInfoPolicy() *a.InfoPolicy @@ -90,33 +91,60 @@ type AerospikeClient interface { type Client struct { aerospikeClient AerospikeClient logger *slog.Logger + scanLimiter *semaphore.Weighted id string } +type Option func(*Client) + +// WithID sets the ID for the Client. +func WithID(id string) Option { + return func(c *Client) { + c.id = id + } +} + +// WithLogger sets the logger for the Client. +func WithLogger(logger *slog.Logger) Option { + return func(c *Client) { + c.logger = logger + } +} + +// WithScanLimiter sets the scanLimiter for the Client. +func WithScanLimiter(sem *semaphore.Weighted) Option { + return func(c *Client) { + c.scanLimiter = sem + } +} + // NewClient creates a new backup client. +// Options: // - ac is the aerospike client to use for backup and restore operations. // - id is an identifier for the client. // - logger is the logger that this client will log to. -func NewClient(ac AerospikeClient, id string, logger *slog.Logger) (*Client, error) { +// - scan limiter semaphore that is used to limit number of concurrent scans. +func NewClient(ac AerospikeClient, opts ...Option) (*Client, error) { if ac == nil { return nil, errors.New("aerospike client pointer is nil") } - if logger == nil { - logger = slog.Default() + // Initialize the Client with default values + client := &Client{ + aerospikeClient: ac, + logger: slog.Default(), // Default logger } - // qualify the logger with a backup lib group - logger = logger.WithGroup("backup") + // Apply all options to the Client + for _, opt := range opts { + opt(client) + } - // add a client group to the logger - logger = logging.WithClient(logger, id) + // Further customization after applying options + client.logger = client.logger.WithGroup("backup") + client.logger = logging.WithClient(client.logger, client.id) - return &Client{ - aerospikeClient: ac, - id: id, - logger: logger, - }, nil + return client, nil } func (c *Client) getUsableInfoPolicy(p *a.InfoPolicy) a.InfoPolicy { @@ -167,7 +195,7 @@ func (c *Client) Backup( return nil, err } - handler := newBackupHandler(ctx, config, c.aerospikeClient, c.logger, writer) + handler := newBackupHandler(ctx, config, c.aerospikeClient, c.logger, writer, c.scanLimiter) handler.run(ctx) return handler, nil diff --git a/handler_backup.go b/handler_backup.go index c41cd4eb..c00e37cb 100644 --- a/handler_backup.go +++ b/handler_backup.go @@ -31,6 +31,7 @@ import ( "github.com/aerospike/backup-go/models" "github.com/aerospike/backup-go/pipeline" "github.com/google/uuid" + "golang.org/x/sync/semaphore" "golang.org/x/time/rate" ) @@ -59,6 +60,7 @@ type BackupHandler struct { limiter *rate.Limiter errors chan error infoClient *asinfo.InfoClient + scanLimiter *semaphore.Weighted id string stats models.BackupStats } @@ -70,6 +72,7 @@ func newBackupHandler( ac AerospikeClient, logger *slog.Logger, writer Writer, + scanLimiter *semaphore.Weighted, ) *BackupHandler { id := uuid.NewString() logger = logging.WithHandler(logger, id, logging.HandlerTypeBackup, writer.GetType()) @@ -86,6 +89,7 @@ func newBackupHandler( encoder: NewEncoder(config.EncoderType, config.Namespace), limiter: limiter, infoClient: asinfo.NewInfoClientFromAerospike(ac, config.InfoPolicy), + scanLimiter: scanLimiter, } } @@ -125,9 +129,9 @@ func (bh *BackupHandler) backupSync(ctx context.Context) error { } writeWorkers := bh.makeWriteWorkers(backupWriters) - handler := newBackupRecordsHandler(bh.config, bh.aerospikeClient, bh.logger) + handler := newBackupRecordsHandler(bh.config, bh.aerospikeClient, bh.logger, bh.scanLimiter) - bh.stats.TotalRecords, err = handler.countRecords(bh.infoClient) + bh.stats.TotalRecords, err = handler.countRecords(ctx, bh.infoClient) if err != nil { return err } diff --git a/handler_backup_records.go b/handler_backup_records.go index 254e8156..53b092ca 100644 --- a/handler_backup_records.go +++ b/handler_backup_records.go @@ -27,18 +27,21 @@ import ( "github.com/aerospike/backup-go/io/aerospike" "github.com/aerospike/backup-go/models" "github.com/aerospike/backup-go/pipeline" + "golang.org/x/sync/semaphore" ) type backupRecordsHandler struct { config *BackupConfig aerospikeClient AerospikeClient logger *slog.Logger + scanLimiter *semaphore.Weighted } func newBackupRecordsHandler( config *BackupConfig, ac AerospikeClient, logger *slog.Logger, + scanLimiter *semaphore.Weighted, ) *backupRecordsHandler { logger.Debug("created new backup records handler") @@ -46,6 +49,7 @@ func newBackupRecordsHandler( config: config, aerospikeClient: ac, logger: logger, + scanLimiter: scanLimiter, } } @@ -54,7 +58,7 @@ func (bh *backupRecordsHandler) run( writers []pipeline.Worker[*models.Token], recordsReadTotal *atomic.Uint64, ) error { - readWorkers, err := bh.makeAerospikeReadWorkers(bh.config.Parallel) + readWorkers, err := bh.makeAerospikeReadWorkers(ctx, bh.config.Parallel) if err != nil { return err } @@ -82,25 +86,23 @@ func (bh *backupRecordsHandler) run( return job.Run(ctx) } -func (bh *backupRecordsHandler) countRecords(infoClient *asinfo.InfoClient) (uint64, error) { +func (bh *backupRecordsHandler) countRecords(ctx context.Context, infoClient *asinfo.InfoClient) (uint64, error) { if bh.config.isFullBackup() { return infoClient.GetRecordCount(bh.config.Namespace, bh.config.SetList) } - return bh.countRecordsUsingScan() + return bh.countRecordsUsingScan(ctx) } -func (bh *backupRecordsHandler) countRecordsUsingScan() (uint64, error) { +func (bh *backupRecordsHandler) countRecordsUsingScan(ctx context.Context) (uint64, error) { scanPolicy := *bh.config.ScanPolicy scanPolicy.IncludeBinData = false scanPolicy.MaxRecords = 0 - recordReader := aerospike.NewRecordReader( - bh.aerospikeClient, - bh.recordReaderConfigForPartition(PartitionRangeAll(), &scanPolicy), - bh.logger, - ) + readerConfig := bh.recordReaderConfigForPartition(PartitionRangeAll(), &scanPolicy) + recordReader := aerospike.NewRecordReader(ctx, bh.aerospikeClient, readerConfig, bh.logger) + defer recordReader.Close() var count uint64 @@ -121,7 +123,7 @@ func (bh *backupRecordsHandler) countRecordsUsingScan() (uint64, error) { } func (bh *backupRecordsHandler) makeAerospikeReadWorkers( - n int, + ctx context.Context, n int, ) ([]pipeline.Worker[*models.Token], error) { partitionRanges, err := splitPartitions( bh.config.Partitions.Begin, @@ -141,6 +143,7 @@ func (bh *backupRecordsHandler) makeAerospikeReadWorkers( for i := 0; i < n; i++ { recordReader := aerospike.NewRecordReader( + ctx, bh.aerospikeClient, bh.recordReaderConfigForPartition(partitionRanges[i], &scanPolicy), bh.logger, @@ -166,5 +169,6 @@ func (bh *backupRecordsHandler) recordReaderConfigForPartition( FromTime: bh.config.ModAfter, ToTime: bh.config.ModBefore, }, + bh.scanLimiter, ) } diff --git a/internal/asinfo/mocks/aerospikeClient_mock.go b/internal/asinfo/mocks/aerospikeClient_mock.go index 953e766a..0f7381a7 100644 --- a/internal/asinfo/mocks/aerospikeClient_mock.go +++ b/internal/asinfo/mocks/aerospikeClient_mock.go @@ -4,7 +4,6 @@ package mocks import ( aerospike "github.com/aerospike/aerospike-client-go/v7" - mock "github.com/stretchr/testify/mock" ) diff --git a/internal/asinfo/mocks/infoGetter_mock.go b/internal/asinfo/mocks/infoGetter_mock.go index 25aa130b..4f93bbe6 100644 --- a/internal/asinfo/mocks/infoGetter_mock.go +++ b/internal/asinfo/mocks/infoGetter_mock.go @@ -4,7 +4,6 @@ package mocks import ( aerospike "github.com/aerospike/aerospike-client-go/v7" - mock "github.com/stretchr/testify/mock" ) diff --git a/io/aerospike/mocks/dbWriter_mock.go b/io/aerospike/mocks/dbWriter_mock.go index 99c38856..b08a4ddb 100644 --- a/io/aerospike/mocks/dbWriter_mock.go +++ b/io/aerospike/mocks/dbWriter_mock.go @@ -4,7 +4,6 @@ package mocks import ( aerospike "github.com/aerospike/aerospike-client-go/v7" - mock "github.com/stretchr/testify/mock" ) diff --git a/io/aerospike/mocks/scanner_mock.go b/io/aerospike/mocks/scanner_mock.go index 7546a453..c89566e2 100644 --- a/io/aerospike/mocks/scanner_mock.go +++ b/io/aerospike/mocks/scanner_mock.go @@ -4,7 +4,6 @@ package mocks import ( aerospike "github.com/aerospike/aerospike-client-go/v7" - mock "github.com/stretchr/testify/mock" ) diff --git a/io/aerospike/record_reader.go b/io/aerospike/record_reader.go index e2416491..bddd64a8 100644 --- a/io/aerospike/record_reader.go +++ b/io/aerospike/record_reader.go @@ -15,6 +15,7 @@ package aerospike import ( + "context" "fmt" "io" "log/slog" @@ -23,6 +24,7 @@ import ( "github.com/aerospike/backup-go/internal/logging" "github.com/aerospike/backup-go/models" "github.com/google/uuid" + "golang.org/x/sync/semaphore" ) // RecordReaderConfig represents the configuration for scanning Aerospike records. @@ -30,6 +32,7 @@ type RecordReaderConfig struct { timeBounds models.TimeBounds partitionFilter *a.PartitionFilter scanPolicy *a.ScanPolicy + scanLimiter *semaphore.Weighted namespace string setList []string binList []string @@ -41,7 +44,9 @@ func NewRecordReaderConfig(namespace string, partitionFilter *a.PartitionFilter, scanPolicy *a.ScanPolicy, binList []string, - timeBounds models.TimeBounds) *RecordReaderConfig { + timeBounds models.TimeBounds, + scanLimiter *semaphore.Weighted, +) *RecordReaderConfig { return &RecordReaderConfig{ namespace: namespace, setList: setList, @@ -49,6 +54,7 @@ func NewRecordReaderConfig(namespace string, scanPolicy: scanPolicy, binList: binList, timeBounds: timeBounds, + scanLimiter: scanLimiter, } } @@ -69,6 +75,7 @@ type scanner interface { // It reads records from an Aerospike database and returns them as // *models.Record. type RecordReader struct { + ctx context.Context client scanner logger *slog.Logger config *RecordReaderConfig @@ -76,7 +83,9 @@ type RecordReader struct { } // NewRecordReader creates a new RecordReader. -func NewRecordReader(client scanner, +func NewRecordReader( + ctx context.Context, + client scanner, cfg *RecordReaderConfig, logger *slog.Logger, ) *RecordReader { @@ -85,6 +94,7 @@ func NewRecordReader(client scanner, logger.Debug("created new aerospike record reader") return &RecordReader{ + ctx: ctx, config: cfg, client: client, logger: logger, @@ -124,6 +134,8 @@ func (r *RecordReader) Read() (*models.Token, error) { // Close cancels the Aerospike scan used to read records // if it was started. func (r *RecordReader) Close() { + r.config.scanLimiter.Release(int64(len(r.scanResult.data))) + if r.scanResult != nil { r.scanResult.Close() } @@ -142,6 +154,11 @@ func (r *RecordReader) startScan() (*recordSets, error) { setsToScan = []string{""} } + err := r.config.scanLimiter.Acquire(r.ctx, int64(len(setsToScan))) + if err != nil { + return nil, err + } + scans := make([]*a.Recordset, 0, len(setsToScan)) for _, set := range setsToScan { diff --git a/io/aerospike/record_reader_test.go b/io/aerospike/record_reader_test.go index 7f1a6b49..bc85cfe2 100644 --- a/io/aerospike/record_reader_test.go +++ b/io/aerospike/record_reader_test.go @@ -15,6 +15,7 @@ package aerospike import ( + "context" "fmt" "io" "log/slog" @@ -70,6 +71,7 @@ func (suite *readersTestSuite) TestAerospikeRecordReader() { ) reader := NewRecordReader( + context.TODO(), mockScanner, &RecordReaderConfig{ namespace: namespace, @@ -124,6 +126,7 @@ func (suite *readersTestSuite) TestAerospikeRecordReaderRecordResError() { ) reader := NewRecordReader( + context.TODO(), mockScanner, &RecordReaderConfig{ namespace: namespace, @@ -163,6 +166,7 @@ func (suite *readersTestSuite) TestAerospikeRecordReaderClosedChannel() { ) reader := NewRecordReader( + context.TODO(), mockScanner, &RecordReaderConfig{ namespace: namespace, @@ -196,6 +200,7 @@ func (suite *readersTestSuite) TestAerospikeRecordReaderReadFailed() { ) reader := NewRecordReader( + context.TODO(), mockScanner, &RecordReaderConfig{ namespace: namespace, @@ -254,6 +259,7 @@ func (suite *readersTestSuite) TestAerospikeRecordReaderWithPolicy() { ) reader := NewRecordReader( + context.TODO(), mockScanner, &RecordReaderConfig{ namespace: namespace, diff --git a/mocks/AerospikeClient_mock.go b/mocks/AerospikeClient_mock.go index f4e72fde..1497fd3f 100644 --- a/mocks/AerospikeClient_mock.go +++ b/mocks/AerospikeClient_mock.go @@ -4,7 +4,6 @@ package mocks import ( aerospike "github.com/aerospike/aerospike-client-go/v7" - mock "github.com/stretchr/testify/mock" ) diff --git a/tests/integration/integration_test.go b/tests/integration/integration_test.go index 4c59a4c5..f105c056 100644 --- a/tests/integration/integration_test.go +++ b/tests/integration/integration_test.go @@ -138,7 +138,7 @@ func (suite *backupRestoreTestSuite) SetupSuite() { testClient := tests.NewTestClient(testAeroClient) suite.testClient = testClient - backupClient, err := backup.NewClient(testAeroClient, "test_client", slog.Default()) + backupClient, err := backup.NewClient(testAeroClient, backup.WithID("test_client")) if err != nil { suite.FailNow(err.Error()) } @@ -203,7 +203,7 @@ func (suite *backupRestoreTestSuite) TestBackupRestoreIO() { restoreConfig *backup.RestoreConfig bins a.BinMap } - var tests = []struct { + var testsCases = []struct { args args name string }{ @@ -216,7 +216,7 @@ func (suite *backupRestoreTestSuite) TestBackupRestoreIO() { }, }, } - for _, tt := range tests { + for _, tt := range testsCases { expectedRecs := genRecords(suite.namespace, suite.set, 1000, tt.args.bins) suite.SetupTest(expectedRecs) suite.Run(tt.name, func() { @@ -277,7 +277,7 @@ func (suite *backupRestoreTestSuite) TestBackupRestoreDirectory() { configWithFileLimit := backup.NewDefaultBackupConfig() configWithFileLimit.FileLimit = 1024 * 1024 - var tests = []struct { + var testsCases = []struct { name string args args }{ @@ -349,7 +349,7 @@ func (suite *backupRestoreTestSuite) TestBackupRestoreDirectory() { }, }, } - for _, tt := range tests { + for _, tt := range testsCases { var initialRecords = genRecords(suite.namespace, suite.set, 20_000, tt.args.bins) suite.SetupTest(initialRecords) suite.Run(tt.name, func() { @@ -619,7 +619,7 @@ func (suite *backupRestoreTestSuite) TestBackupRestoreIOEncryptionFile() { Mode: backup.EncryptAES128, } - var tests = []struct { + var testsCases = []struct { args args name string }{ @@ -632,7 +632,7 @@ func (suite *backupRestoreTestSuite) TestBackupRestoreIOEncryptionFile() { }, }, } - for _, tt := range tests { + for _, tt := range testsCases { expectedRecs := genRecords(suite.namespace, suite.set, 1000, tt.args.bins) suite.SetupTest(expectedRecs) suite.Run(tt.name, func() { @@ -657,7 +657,7 @@ func (suite *backupRestoreTestSuite) TestBackupRestoreIONamespace() { Destination: &destination, } - var tests = []struct { + var testsCases = []struct { args args name string }{ @@ -670,7 +670,7 @@ func (suite *backupRestoreTestSuite) TestBackupRestoreIONamespace() { }, }, } - for _, tt := range tests { + for _, tt := range testsCases { expectedRecs := genRecords(suite.namespace, suite.set, 1000, tt.args.bins) suite.SetupTest(expectedRecs) suite.Run(tt.name, func() { @@ -697,7 +697,7 @@ func (suite *backupRestoreTestSuite) TestBackupRestoreIOCompression() { Mode: backup.CompressZSTD, } - var tests = []struct { + var testsCases = []struct { args args name string }{ @@ -710,7 +710,7 @@ func (suite *backupRestoreTestSuite) TestBackupRestoreIOCompression() { }, }, } - for _, tt := range tests { + for _, tt := range testsCases { expectedRecs := genRecords(suite.namespace, suite.set, 1000, tt.args.bins) suite.SetupTest(expectedRecs) suite.Run(tt.name, func() { From eb41d06c00a644ce75f48bae533f43d5327e88de Mon Sep 17 00:00:00 2001 From: Anton Korotkov Date: Sun, 11 Aug 2024 09:48:50 +0300 Subject: [PATCH 02/20] update dependencies --- go.mod | 8 ++++---- go.sum | 14 ++++++++------ 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 0b4a2936..d7a0b30b 100644 --- a/go.mod +++ b/go.mod @@ -7,16 +7,17 @@ require ( github.com/aerospike/tools-common-go v0.0.0-20240701164814-36eec593d9c6 github.com/aws/aws-sdk-go-v2 v1.30.3 github.com/aws/aws-sdk-go-v2/config v1.27.27 - github.com/aws/aws-sdk-go-v2/service/s3 v1.58.2 + github.com/aws/aws-sdk-go-v2/service/s3 v1.58.3 github.com/aws/smithy-go v1.20.3 - github.com/docker/docker v27.1.0+incompatible + github.com/docker/docker v27.1.1+incompatible github.com/docker/go-connections v0.5.0 github.com/google/go-cmp v0.6.0 github.com/google/uuid v1.6.0 github.com/klauspost/compress v1.17.9 github.com/minio/minio-go/v7 v7.0.74 github.com/stretchr/testify v1.9.0 - golang.org/x/time v0.5.0 + golang.org/x/sync v0.8.0 + golang.org/x/time v0.6.0 ) require ( @@ -70,7 +71,6 @@ require ( go.opentelemetry.io/proto/otlp v1.3.1 // indirect golang.org/x/crypto v0.24.0 // indirect golang.org/x/net v0.26.0 // indirect - golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.21.0 // indirect golang.org/x/text v0.16.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240711142825-46eb208f015d // indirect diff --git a/go.sum b/go.sum index c03d89c0..a6766d1b 100644 --- a/go.sum +++ b/go.sum @@ -32,8 +32,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 h1:HGErhhrx github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17/go.mod h1:RkZEx4l0EHYDJpWppMJ3nD9wZJAa8/0lq9aVC+r2UII= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.15 h1:246A4lSTXWJw/rmlQI+TT2OcqeDMKBdyjEQrafMaQdA= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.15/go.mod h1:haVfg3761/WF7YPuJOER2MP0k4UAXyHaLclKXB6usDg= -github.com/aws/aws-sdk-go-v2/service/s3 v1.58.2 h1:sZXIzO38GZOU+O0C+INqbH7C2yALwfMWpd64tONS/NE= -github.com/aws/aws-sdk-go-v2/service/s3 v1.58.2/go.mod h1:Lcxzg5rojyVPU/0eFwLtcyTaek/6Mtic5B1gJo7e/zE= +github.com/aws/aws-sdk-go-v2/service/s3 v1.58.3 h1:hT8ZAZRIfqBqHbzKTII+CIiY8G2oC9OpLedkZ51DWl8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.58.3/go.mod h1:Lcxzg5rojyVPU/0eFwLtcyTaek/6Mtic5B1gJo7e/zE= github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 h1:BXx0ZIxvrJdSgSvKTZ+yRBeSqqgPM89VPlulEcl37tM= github.com/aws/aws-sdk-go-v2/service/sso v1.22.4/go.mod h1:ooyCOXjvJEsUw7x+ZDHeISPMhtwI3ZCB7ggFMcFfWLU= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 h1:yiwVzJW2ZxZTurVbYWA7QOrAaCYQR72t0wrSBfoesUE= @@ -53,8 +53,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/docker v27.1.0+incompatible h1:rEHVQc4GZ0MIQKifQPHSFGV/dVgaZafgRf8fCPtDYBs= -github.com/docker/docker v27.1.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v27.1.1+incompatible h1:hO/M4MtV36kzKldqnA37IWhebRA+LnqqcqDja6kVaKY= +github.com/docker/docker v27.1.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -167,6 +167,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -178,8 +180,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= -golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= -golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= +golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= From 5d7cdfb161e93259e027123564b4081f91cd7d9f Mon Sep 17 00:00:00 2001 From: Anton Korotkov Date: Sun, 11 Aug 2024 10:03:04 +0300 Subject: [PATCH 03/20] fix main.go --- examples/aws/s3/main.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/examples/aws/s3/main.go b/examples/aws/s3/main.go index 05f8ae64..9d918ba3 100644 --- a/examples/aws/s3/main.go +++ b/examples/aws/s3/main.go @@ -17,12 +17,10 @@ package main import ( "context" "fmt" - "log" - "log/slog" - "github.com/aerospike/aerospike-client-go/v7" "github.com/aerospike/backup-go" "github.com/aerospike/backup-go/io/aws/s3" + "log" ) const ( @@ -58,7 +56,7 @@ func initBackupClient() *backup.Client { panic(aerr) } - backupClient, err := backup.NewClient(aerospikeClient, "client_id", slog.Default()) + backupClient, err := backup.NewClient(aerospikeClient, backup.WithID("client_id")) if err != nil { panic(err) } From 55af222125c26befeb4467e766e95c5c150cfcf0 Mon Sep 17 00:00:00 2001 From: Anton Korotkov Date: Sun, 11 Aug 2024 10:04:21 +0300 Subject: [PATCH 04/20] optional limiter --- io/aerospike/record_reader.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/io/aerospike/record_reader.go b/io/aerospike/record_reader.go index bddd64a8..d8cd4f1d 100644 --- a/io/aerospike/record_reader.go +++ b/io/aerospike/record_reader.go @@ -134,7 +134,9 @@ func (r *RecordReader) Read() (*models.Token, error) { // Close cancels the Aerospike scan used to read records // if it was started. func (r *RecordReader) Close() { - r.config.scanLimiter.Release(int64(len(r.scanResult.data))) + if r.config.scanLimiter != nil { + r.config.scanLimiter.Release(int64(len(r.scanResult.data))) + } if r.scanResult != nil { r.scanResult.Close() @@ -154,9 +156,11 @@ func (r *RecordReader) startScan() (*recordSets, error) { setsToScan = []string{""} } - err := r.config.scanLimiter.Acquire(r.ctx, int64(len(setsToScan))) - if err != nil { - return nil, err + if r.config.scanLimiter != nil { + err := r.config.scanLimiter.Acquire(r.ctx, int64(len(setsToScan))) + if err != nil { + return nil, err + } } scans := make([]*a.Recordset, 0, len(setsToScan)) From 297b1098e7b328112604c75fb74ac55452793560 Mon Sep 17 00:00:00 2001 From: Anton Korotkov Date: Sun, 11 Aug 2024 10:06:00 +0300 Subject: [PATCH 05/20] fix build in readme --- examples/readme/main.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/examples/readme/main.go b/examples/readme/main.go index 5dfd8465..d231dee4 100644 --- a/examples/readme/main.go +++ b/examples/readme/main.go @@ -16,11 +16,9 @@ package main import ( "context" - "log" - "log/slog" - "github.com/aerospike/aerospike-client-go/v7" "github.com/aerospike/backup-go" + "log" ) func main() { @@ -29,7 +27,7 @@ func main() { panic(aerr) } - backupClient, err := backup.NewClient(aerospikeClient, "client_id", slog.Default()) + backupClient, err := backup.NewClient(aerospikeClient, backup.WithID("client_id")) if err != nil { panic(err) } From 3606da18054d8d519508dd94a8527795dff79efd Mon Sep 17 00:00:00 2001 From: yrizhkov Date: Sun, 11 Aug 2024 10:29:27 +0300 Subject: [PATCH 06/20] update the golangci-lint workflow --- .github/workflows/golangci-lint.yaml | 14 ++++++++------ go.sum | 2 -- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/golangci-lint.yaml b/.github/workflows/golangci-lint.yaml index c00a0b1c..674f7ff1 100644 --- a/.github/workflows/golangci-lint.yaml +++ b/.github/workflows/golangci-lint.yaml @@ -1,14 +1,15 @@ name: golangci-lint on: push: - tags: - - v* branches: - '*' pull_request: branches: - main +permissions: + contents: read + jobs: golangci: name: lint @@ -16,17 +17,18 @@ jobs: steps: - name: Checkout sources uses: actions/checkout@v4 + - name: Get go version from go.mod run: | echo "GO_VERSION=$(grep '^go ' go.mod | cut -d " " -f 2)" >> $GITHUB_ENV + - name: Setup-go uses: actions/setup-go@v5 with: go-version: ${{ env.GO_VERSION }} - - name: "generate-mocks" - run: make mocks-generate + - name: Run golangci-lint - uses: golangci/golangci-lint-action@v4 + uses: golangci/golangci-lint-action@v6 with: version: v1.56.2 - args: --timeout=5m \ No newline at end of file + args: --timeout=5m diff --git a/go.sum b/go.sum index a6766d1b..b13520c7 100644 --- a/go.sum +++ b/go.sum @@ -165,8 +165,6 @@ golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= From ec9edd15b357d6c404060a850757bbfd2c0c7c35 Mon Sep 17 00:00:00 2001 From: Anton Korotkov <106995168+korotkov-aerospike@users.noreply.github.com> Date: Sun, 11 Aug 2024 11:00:48 +0300 Subject: [PATCH 07/20] Update client.go Co-authored-by: Eugene R. --- client.go | 1 + 1 file changed, 1 insertion(+) diff --git a/client.go b/client.go index 763b495b..9a33c7dc 100644 --- a/client.go +++ b/client.go @@ -95,6 +95,7 @@ type Client struct { id string } +// Option is a functional option that allows configuring the [Client]. type Option func(*Client) // WithID sets the ID for the Client. From 581cb080b9e0ef697a0a2a40c07eea191a55ee97 Mon Sep 17 00:00:00 2001 From: Anton Korotkov <106995168+korotkov-aerospike@users.noreply.github.com> Date: Sun, 11 Aug 2024 11:01:01 +0300 Subject: [PATCH 08/20] Update client.go Co-authored-by: Eugene R. --- client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client.go b/client.go index 9a33c7dc..7089553d 100644 --- a/client.go +++ b/client.go @@ -105,7 +105,7 @@ func WithID(id string) Option { } } -// WithLogger sets the logger for the Client. +// WithLogger sets the logger for the [Client]. func WithLogger(logger *slog.Logger) Option { return func(c *Client) { c.logger = logger From 58ae895eab1e7710e992d69cd569da543e1bb0c7 Mon Sep 17 00:00:00 2001 From: Anton Korotkov <106995168+korotkov-aerospike@users.noreply.github.com> Date: Sun, 11 Aug 2024 11:01:10 +0300 Subject: [PATCH 09/20] Update client.go Co-authored-by: Eugene R. --- client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client.go b/client.go index 7089553d..bfdddb14 100644 --- a/client.go +++ b/client.go @@ -112,7 +112,7 @@ func WithLogger(logger *slog.Logger) Option { } } -// WithScanLimiter sets the scanLimiter for the Client. +// WithScanLimiter sets the scan limiter for the [Client]. func WithScanLimiter(sem *semaphore.Weighted) Option { return func(c *Client) { c.scanLimiter = sem From fda1fd9d8fd38eba2754c73337fe1b90e36cfed2 Mon Sep 17 00:00:00 2001 From: Anton Korotkov <106995168+korotkov-aerospike@users.noreply.github.com> Date: Sun, 11 Aug 2024 11:01:27 +0300 Subject: [PATCH 10/20] Update client.go Co-authored-by: Eugene R. --- client.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/client.go b/client.go index bfdddb14..ae0732c3 100644 --- a/client.go +++ b/client.go @@ -120,10 +120,13 @@ func WithScanLimiter(sem *semaphore.Weighted) Option { } // NewClient creates a new backup client. -// Options: // - ac is the aerospike client to use for backup and restore operations. -// - id is an identifier for the client. -// - logger is the logger that this client will log to. +// +// Options: +// - [WithID] to set an identifier for the client. +// - [WithLogger] to set a logger that this client will log to. +// - [WithScanLimiter] to set a semaphore that is used to limit number of +// concurrent scans. // - scan limiter semaphore that is used to limit number of concurrent scans. func NewClient(ac AerospikeClient, opts ...Option) (*Client, error) { if ac == nil { From 3944c15d4432751aaf61ad8d430b40ca272c4401 Mon Sep 17 00:00:00 2001 From: Anton Korotkov Date: Sun, 11 Aug 2024 11:03:35 +0300 Subject: [PATCH 11/20] codereview comments --- client.go | 12 ++++++------ examples/aws/s3/main.go | 3 ++- examples/readme/main.go | 3 ++- io/aerospike/record_reader_test.go | 10 +++++----- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/client.go b/client.go index ae0732c3..206ab30d 100644 --- a/client.go +++ b/client.go @@ -95,25 +95,25 @@ type Client struct { id string } -// Option is a functional option that allows configuring the [Client]. -type Option func(*Client) +// ClientOpt is a functional option that allows configuring the [Client]. +type ClientOpt func(*Client) // WithID sets the ID for the Client. -func WithID(id string) Option { +func WithID(id string) ClientOpt { return func(c *Client) { c.id = id } } // WithLogger sets the logger for the [Client]. -func WithLogger(logger *slog.Logger) Option { +func WithLogger(logger *slog.Logger) ClientOpt { return func(c *Client) { c.logger = logger } } // WithScanLimiter sets the scan limiter for the [Client]. -func WithScanLimiter(sem *semaphore.Weighted) Option { +func WithScanLimiter(sem *semaphore.Weighted) ClientOpt { return func(c *Client) { c.scanLimiter = sem } @@ -128,7 +128,7 @@ func WithScanLimiter(sem *semaphore.Weighted) Option { // - [WithScanLimiter] to set a semaphore that is used to limit number of // concurrent scans. // - scan limiter semaphore that is used to limit number of concurrent scans. -func NewClient(ac AerospikeClient, opts ...Option) (*Client, error) { +func NewClient(ac AerospikeClient, opts ...ClientOpt) (*Client, error) { if ac == nil { return nil, errors.New("aerospike client pointer is nil") } diff --git a/examples/aws/s3/main.go b/examples/aws/s3/main.go index 9d918ba3..a5cf0bf9 100644 --- a/examples/aws/s3/main.go +++ b/examples/aws/s3/main.go @@ -17,10 +17,11 @@ package main import ( "context" "fmt" + "log" + "github.com/aerospike/aerospike-client-go/v7" "github.com/aerospike/backup-go" "github.com/aerospike/backup-go/io/aws/s3" - "log" ) const ( diff --git a/examples/readme/main.go b/examples/readme/main.go index d231dee4..fb95943f 100644 --- a/examples/readme/main.go +++ b/examples/readme/main.go @@ -16,9 +16,10 @@ package main import ( "context" + "log" + "github.com/aerospike/aerospike-client-go/v7" "github.com/aerospike/backup-go" - "log" ) func main() { diff --git a/io/aerospike/record_reader_test.go b/io/aerospike/record_reader_test.go index bc85cfe2..3c11d076 100644 --- a/io/aerospike/record_reader_test.go +++ b/io/aerospike/record_reader_test.go @@ -71,7 +71,7 @@ func (suite *readersTestSuite) TestAerospikeRecordReader() { ) reader := NewRecordReader( - context.TODO(), + context.Background(), mockScanner, &RecordReaderConfig{ namespace: namespace, @@ -126,7 +126,7 @@ func (suite *readersTestSuite) TestAerospikeRecordReaderRecordResError() { ) reader := NewRecordReader( - context.TODO(), + context.Background(), mockScanner, &RecordReaderConfig{ namespace: namespace, @@ -166,7 +166,7 @@ func (suite *readersTestSuite) TestAerospikeRecordReaderClosedChannel() { ) reader := NewRecordReader( - context.TODO(), + context.Background(), mockScanner, &RecordReaderConfig{ namespace: namespace, @@ -200,7 +200,7 @@ func (suite *readersTestSuite) TestAerospikeRecordReaderReadFailed() { ) reader := NewRecordReader( - context.TODO(), + context.Background(), mockScanner, &RecordReaderConfig{ namespace: namespace, @@ -259,7 +259,7 @@ func (suite *readersTestSuite) TestAerospikeRecordReaderWithPolicy() { ) reader := NewRecordReader( - context.TODO(), + context.Background(), mockScanner, &RecordReaderConfig{ namespace: namespace, From 46dff4a2f733e78c048d96948699bc98afcd956c Mon Sep 17 00:00:00 2001 From: Anton Korotkov <106995168+korotkov-aerospike@users.noreply.github.com> Date: Sun, 11 Aug 2024 11:48:08 +0300 Subject: [PATCH 12/20] Update client.go Co-authored-by: Eugene R. --- client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client.go b/client.go index 206ab30d..79527d9c 100644 --- a/client.go +++ b/client.go @@ -98,7 +98,7 @@ type Client struct { // ClientOpt is a functional option that allows configuring the [Client]. type ClientOpt func(*Client) -// WithID sets the ID for the Client. +// WithID sets the ID for the [Client]. func WithID(id string) ClientOpt { return func(c *Client) { c.id = id From 08e1b9d20a10b19b52d495c1596ac36214f5cc43 Mon Sep 17 00:00:00 2001 From: Anton Korotkov Date: Sun, 11 Aug 2024 12:15:43 +0300 Subject: [PATCH 13/20] add default client --- client.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/client.go b/client.go index 206ab30d..278d6f19 100644 --- a/client.go +++ b/client.go @@ -19,6 +19,8 @@ import ( "errors" "fmt" "log/slog" + "math/rand/v2" + "strconv" a "github.com/aerospike/aerospike-client-go/v7" "github.com/aerospike/backup-go/internal/logging" @@ -63,7 +65,7 @@ type AerospikeClient interface { // // handle error // } // -// backupClient, err := backup.NewClient(asc, "id", nil) // create a backup client +// backupClient, err := backup.NewClient(asc, backup.WithID("id")) // create a backup client // if err != nil { // // handle error // } @@ -127,7 +129,6 @@ func WithScanLimiter(sem *semaphore.Weighted) ClientOpt { // - [WithLogger] to set a logger that this client will log to. // - [WithScanLimiter] to set a semaphore that is used to limit number of // concurrent scans. -// - scan limiter semaphore that is used to limit number of concurrent scans. func NewClient(ac AerospikeClient, opts ...ClientOpt) (*Client, error) { if ac == nil { return nil, errors.New("aerospike client pointer is nil") @@ -136,7 +137,8 @@ func NewClient(ac AerospikeClient, opts ...ClientOpt) (*Client, error) { // Initialize the Client with default values client := &Client{ aerospikeClient: ac, - logger: slog.Default(), // Default logger + logger: slog.Default(), + id: strconv.Itoa(rand.IntN(1000)), } // Apply all options to the Client From 5d510ec12f580527394cf47d3073c8139f8de1b8 Mon Sep 17 00:00:00 2001 From: Anton Korotkov Date: Sun, 11 Aug 2024 12:29:02 +0300 Subject: [PATCH 14/20] test for options --- client_test.go | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/client_test.go b/client_test.go index 8b910fc0..1ea6f844 100644 --- a/client_test.go +++ b/client_test.go @@ -15,7 +15,13 @@ package backup import ( + "log/slog" + "strings" "testing" + + "github.com/aerospike/backup-go/mocks" + "github.com/stretchr/testify/assert" + "golang.org/x/sync/semaphore" ) func TestPartitionRange_validate(t *testing.T) { @@ -81,3 +87,28 @@ func TestPartitionRange_validate(t *testing.T) { }) } } + +func TestNilClient(t *testing.T) { + _, err := NewClient(nil) + assert.Error(t, err, "aerospike client is required") +} + +func TestClientOptions(t *testing.T) { + var logBuffer strings.Builder + logger := slog.New(slog.NewTextHandler(&logBuffer, nil)) + sem := semaphore.NewWeighted(10) + id := "ID" + + client, err := NewClient(&mocks.MockAerospikeClient{}, + WithID(id), + WithLogger(logger), + WithScanLimiter(sem), + ) + + assert.NoError(t, err) + assert.Equal(t, id, client.id) + assert.Equal(t, sem, client.scanLimiter) + + client.logger.Info("test") + assert.Contains(t, logBuffer.String(), "level=INFO msg=test backup.client.id=ID") +} From 17ac9a2a62244000fb774eeced6a891011955020 Mon Sep 17 00:00:00 2001 From: Anton Korotkov Date: Sun, 11 Aug 2024 12:34:26 +0300 Subject: [PATCH 15/20] random int linter warning --- client.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client.go b/client.go index 0cf10220..d5ca632f 100644 --- a/client.go +++ b/client.go @@ -138,7 +138,8 @@ func NewClient(ac AerospikeClient, opts ...ClientOpt) (*Client, error) { client := &Client{ aerospikeClient: ac, logger: slog.Default(), - id: strconv.Itoa(rand.IntN(1000)), + // #nosec G404 + id: strconv.Itoa(rand.IntN(1000)), } // Apply all options to the Client From c57e5dabb60d6f756d0460f946de0c5f6036f5ab Mon Sep 17 00:00:00 2001 From: Anton Korotkov Date: Sun, 11 Aug 2024 12:35:31 +0300 Subject: [PATCH 16/20] use rand package --- client.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client.go b/client.go index d5ca632f..9f359c6c 100644 --- a/client.go +++ b/client.go @@ -19,7 +19,7 @@ import ( "errors" "fmt" "log/slog" - "math/rand/v2" + "math/rand" "strconv" a "github.com/aerospike/aerospike-client-go/v7" @@ -139,7 +139,7 @@ func NewClient(ac AerospikeClient, opts ...ClientOpt) (*Client, error) { aerospikeClient: ac, logger: slog.Default(), // #nosec G404 - id: strconv.Itoa(rand.IntN(1000)), + id: strconv.Itoa(rand.Intn(1000)), } // Apply all options to the Client From a7f0f056c1e35580d51646b50bf51a2bde34e709 Mon Sep 17 00:00:00 2001 From: yrizhkov Date: Sun, 11 Aug 2024 13:17:02 +0300 Subject: [PATCH 17/20] update the example in readme --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index ee796f94..294966a3 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,6 @@ package main import ( "context" "log" - "log/slog" "github.com/aerospike/aerospike-client-go/v7" "github.com/aerospike/backup-go" @@ -35,7 +34,7 @@ func main() { panic(aerr) } - backupClient, err := backup.NewClient(aerospikeClient, "client_id", slog.Default()) + backupClient, err := backup.NewClient(aerospikeClient, backup.WithID("client_id")) if err != nil { panic(err) } From 3ea8e8e2c4e51d7c49ab36c72f65939683f13d6e Mon Sep 17 00:00:00 2001 From: Anton Korotkov Date: Sun, 11 Aug 2024 15:45:59 +0300 Subject: [PATCH 18/20] aerospike client getter --- client.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/client.go b/client.go index 9f359c6c..155f7b72 100644 --- a/client.go +++ b/client.go @@ -54,6 +54,7 @@ type AerospikeClient interface { Cluster() *a.Cluster ScanPartitions(scanPolicy *a.ScanPolicy, partitionFilter *a.PartitionFilter, namespace string, setName string, binNames ...string) (*a.Recordset, a.Error) + Close() } // Client is the main entry point for the backup package. @@ -238,3 +239,8 @@ func (c *Client) Restore( return handler, nil } + +// AerospikeClient returns the underlying aerospike client. +func (c *Client) AerospikeClient() AerospikeClient { + return c.aerospikeClient +} From e95de98cb474f5dbbd967f2d4a4354eff39d8029 Mon Sep 17 00:00:00 2001 From: Anton Korotkov Date: Sun, 11 Aug 2024 15:51:24 +0300 Subject: [PATCH 19/20] aerospike client getter --- internal/asinfo/mocks/aerospikeClient_mock.go | 2 +- internal/asinfo/mocks/infoGetter_mock.go | 2 +- .../processors/mocks/DataProcessor_mock.go | 2 +- io/aerospike/mocks/dbWriter_mock.go | 2 +- io/aerospike/mocks/recordWriter_mock.go | 2 +- io/aerospike/mocks/scanner_mock.go | 2 +- io/aerospike/mocks/sindexGetter_mock.go | 2 +- io/aerospike/mocks/udfGetter_mock.go | 2 +- io/encoding/asb/mocks/UserKeyTypesInt_mock.go | 2 +- io/encoding/asb/mocks/binTypesInt_mock.go | 2 +- io/local/mocks/validator_mock.go | 2 +- mocks/AerospikeClient_mock.go | 34 ++++++++++++++++++- mocks/Decoder_mock.go | 2 +- mocks/Encoder_mock.go | 2 +- mocks/StreamingReader_mock.go | 2 +- mocks/Writer_mock.go | 2 +- mocks/statsSetterToken_mock.go | 2 +- pipeline/mocks/DataWriter_mock.go | 2 +- pipeline/mocks/Worker_mock.go | 2 +- pipeline/mocks/dataReader_mock.go | 2 +- .../connection/mocks/connector_mock.go | 2 +- 21 files changed, 53 insertions(+), 21 deletions(-) diff --git a/internal/asinfo/mocks/aerospikeClient_mock.go b/internal/asinfo/mocks/aerospikeClient_mock.go index 0f7381a7..416d6366 100644 --- a/internal/asinfo/mocks/aerospikeClient_mock.go +++ b/internal/asinfo/mocks/aerospikeClient_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. +// Code generated by mockery v2.42.1. DO NOT EDIT. package mocks diff --git a/internal/asinfo/mocks/infoGetter_mock.go b/internal/asinfo/mocks/infoGetter_mock.go index 4f93bbe6..ca7df9fa 100644 --- a/internal/asinfo/mocks/infoGetter_mock.go +++ b/internal/asinfo/mocks/infoGetter_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. +// Code generated by mockery v2.42.1. DO NOT EDIT. package mocks diff --git a/internal/processors/mocks/DataProcessor_mock.go b/internal/processors/mocks/DataProcessor_mock.go index 93032ccc..e30972b4 100644 --- a/internal/processors/mocks/DataProcessor_mock.go +++ b/internal/processors/mocks/DataProcessor_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. +// Code generated by mockery v2.42.1. DO NOT EDIT. package mocks diff --git a/io/aerospike/mocks/dbWriter_mock.go b/io/aerospike/mocks/dbWriter_mock.go index b08a4ddb..b86ee8f6 100644 --- a/io/aerospike/mocks/dbWriter_mock.go +++ b/io/aerospike/mocks/dbWriter_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. +// Code generated by mockery v2.42.1. DO NOT EDIT. package mocks diff --git a/io/aerospike/mocks/recordWriter_mock.go b/io/aerospike/mocks/recordWriter_mock.go index 3a5c5286..a5530984 100644 --- a/io/aerospike/mocks/recordWriter_mock.go +++ b/io/aerospike/mocks/recordWriter_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. +// Code generated by mockery v2.42.1. DO NOT EDIT. package mocks diff --git a/io/aerospike/mocks/scanner_mock.go b/io/aerospike/mocks/scanner_mock.go index c89566e2..cfb2a1c2 100644 --- a/io/aerospike/mocks/scanner_mock.go +++ b/io/aerospike/mocks/scanner_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. +// Code generated by mockery v2.42.1. DO NOT EDIT. package mocks diff --git a/io/aerospike/mocks/sindexGetter_mock.go b/io/aerospike/mocks/sindexGetter_mock.go index cfa468cb..db5b975a 100644 --- a/io/aerospike/mocks/sindexGetter_mock.go +++ b/io/aerospike/mocks/sindexGetter_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. +// Code generated by mockery v2.42.1. DO NOT EDIT. package mocks diff --git a/io/aerospike/mocks/udfGetter_mock.go b/io/aerospike/mocks/udfGetter_mock.go index c789265d..81b88e88 100644 --- a/io/aerospike/mocks/udfGetter_mock.go +++ b/io/aerospike/mocks/udfGetter_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. +// Code generated by mockery v2.42.1. DO NOT EDIT. package mocks diff --git a/io/encoding/asb/mocks/UserKeyTypesInt_mock.go b/io/encoding/asb/mocks/UserKeyTypesInt_mock.go index 4de1e31f..cdd41417 100644 --- a/io/encoding/asb/mocks/UserKeyTypesInt_mock.go +++ b/io/encoding/asb/mocks/UserKeyTypesInt_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. +// Code generated by mockery v2.42.1. DO NOT EDIT. package mocks diff --git a/io/encoding/asb/mocks/binTypesInt_mock.go b/io/encoding/asb/mocks/binTypesInt_mock.go index e0529366..27273d55 100644 --- a/io/encoding/asb/mocks/binTypesInt_mock.go +++ b/io/encoding/asb/mocks/binTypesInt_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. +// Code generated by mockery v2.42.1. DO NOT EDIT. package mocks diff --git a/io/local/mocks/validator_mock.go b/io/local/mocks/validator_mock.go index 2cf510d4..4622b981 100644 --- a/io/local/mocks/validator_mock.go +++ b/io/local/mocks/validator_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. +// Code generated by mockery v2.42.1. DO NOT EDIT. package mocks diff --git a/mocks/AerospikeClient_mock.go b/mocks/AerospikeClient_mock.go index 1497fd3f..a215a739 100644 --- a/mocks/AerospikeClient_mock.go +++ b/mocks/AerospikeClient_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. +// Code generated by mockery v2.42.1. DO NOT EDIT. package mocks @@ -69,6 +69,38 @@ func (_c *MockAerospikeClient_BatchOperate_Call) RunAndReturn(run func(*aerospik return _c } +// Close provides a mock function with given fields: +func (_m *MockAerospikeClient) Close() { + _m.Called() +} + +// MockAerospikeClient_Close_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Close' +type MockAerospikeClient_Close_Call struct { + *mock.Call +} + +// Close is a helper method to define mock.On call +func (_e *MockAerospikeClient_Expecter) Close() *MockAerospikeClient_Close_Call { + return &MockAerospikeClient_Close_Call{Call: _e.mock.On("Close")} +} + +func (_c *MockAerospikeClient_Close_Call) Run(run func()) *MockAerospikeClient_Close_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *MockAerospikeClient_Close_Call) Return() *MockAerospikeClient_Close_Call { + _c.Call.Return() + return _c +} + +func (_c *MockAerospikeClient_Close_Call) RunAndReturn(run func()) *MockAerospikeClient_Close_Call { + _c.Call.Return(run) + return _c +} + // Cluster provides a mock function with given fields: func (_m *MockAerospikeClient) Cluster() *aerospike.Cluster { ret := _m.Called() diff --git a/mocks/Decoder_mock.go b/mocks/Decoder_mock.go index a470bed5..863cca1b 100644 --- a/mocks/Decoder_mock.go +++ b/mocks/Decoder_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. +// Code generated by mockery v2.42.1. DO NOT EDIT. package mocks diff --git a/mocks/Encoder_mock.go b/mocks/Encoder_mock.go index 89ab850a..133c5055 100644 --- a/mocks/Encoder_mock.go +++ b/mocks/Encoder_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. +// Code generated by mockery v2.42.1. DO NOT EDIT. package mocks diff --git a/mocks/StreamingReader_mock.go b/mocks/StreamingReader_mock.go index 7a87dcd5..f2828a48 100644 --- a/mocks/StreamingReader_mock.go +++ b/mocks/StreamingReader_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. +// Code generated by mockery v2.42.1. DO NOT EDIT. package mocks diff --git a/mocks/Writer_mock.go b/mocks/Writer_mock.go index a01e8417..29013d10 100644 --- a/mocks/Writer_mock.go +++ b/mocks/Writer_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. +// Code generated by mockery v2.42.1. DO NOT EDIT. package mocks diff --git a/mocks/statsSetterToken_mock.go b/mocks/statsSetterToken_mock.go index a4009de6..dd4dc8e4 100644 --- a/mocks/statsSetterToken_mock.go +++ b/mocks/statsSetterToken_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. +// Code generated by mockery v2.42.1. DO NOT EDIT. package mocks diff --git a/pipeline/mocks/DataWriter_mock.go b/pipeline/mocks/DataWriter_mock.go index 07ea57ef..d626233a 100644 --- a/pipeline/mocks/DataWriter_mock.go +++ b/pipeline/mocks/DataWriter_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. +// Code generated by mockery v2.42.1. DO NOT EDIT. package mocks diff --git a/pipeline/mocks/Worker_mock.go b/pipeline/mocks/Worker_mock.go index 44c76455..47a7610d 100644 --- a/pipeline/mocks/Worker_mock.go +++ b/pipeline/mocks/Worker_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. +// Code generated by mockery v2.42.1. DO NOT EDIT. package mocks diff --git a/pipeline/mocks/dataReader_mock.go b/pipeline/mocks/dataReader_mock.go index f38d1cc9..6f464f47 100644 --- a/pipeline/mocks/dataReader_mock.go +++ b/pipeline/mocks/dataReader_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. +// Code generated by mockery v2.42.1. DO NOT EDIT. package mocks diff --git a/pkg/secret-agent/connection/mocks/connector_mock.go b/pkg/secret-agent/connection/mocks/connector_mock.go index 28ecc2a8..b6271a81 100644 --- a/pkg/secret-agent/connection/mocks/connector_mock.go +++ b/pkg/secret-agent/connection/mocks/connector_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. +// Code generated by mockery v2.42.1. DO NOT EDIT. package mocks From 17428e56aa12d6f5327991651ae997be2fbe51aa Mon Sep 17 00:00:00 2001 From: Anton Korotkov Date: Mon, 12 Aug 2024 09:37:36 +0300 Subject: [PATCH 20/20] add getNodes --- client.go | 1 + mocks/AerospikeClient_mock.go | 47 +++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/client.go b/client.go index 155f7b72..d4530871 100644 --- a/client.go +++ b/client.go @@ -55,6 +55,7 @@ type AerospikeClient interface { ScanPartitions(scanPolicy *a.ScanPolicy, partitionFilter *a.PartitionFilter, namespace string, setName string, binNames ...string) (*a.Recordset, a.Error) Close() + GetNodes() []*a.Node } // Client is the main entry point for the backup package. diff --git a/mocks/AerospikeClient_mock.go b/mocks/AerospikeClient_mock.go index a215a739..a94ccae4 100644 --- a/mocks/AerospikeClient_mock.go +++ b/mocks/AerospikeClient_mock.go @@ -421,6 +421,53 @@ func (_c *MockAerospikeClient_GetDefaultWritePolicy_Call) RunAndReturn(run func( return _c } +// GetNodes provides a mock function with given fields: +func (_m *MockAerospikeClient) GetNodes() []*aerospike.Node { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for GetNodes") + } + + var r0 []*aerospike.Node + if rf, ok := ret.Get(0).(func() []*aerospike.Node); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*aerospike.Node) + } + } + + return r0 +} + +// MockAerospikeClient_GetNodes_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetNodes' +type MockAerospikeClient_GetNodes_Call struct { + *mock.Call +} + +// GetNodes is a helper method to define mock.On call +func (_e *MockAerospikeClient_Expecter) GetNodes() *MockAerospikeClient_GetNodes_Call { + return &MockAerospikeClient_GetNodes_Call{Call: _e.mock.On("GetNodes")} +} + +func (_c *MockAerospikeClient_GetNodes_Call) Run(run func()) *MockAerospikeClient_GetNodes_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *MockAerospikeClient_GetNodes_Call) Return(_a0 []*aerospike.Node) *MockAerospikeClient_GetNodes_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *MockAerospikeClient_GetNodes_Call) RunAndReturn(run func() []*aerospike.Node) *MockAerospikeClient_GetNodes_Call { + _c.Call.Return(run) + return _c +} + // Put provides a mock function with given fields: policy, key, bins func (_m *MockAerospikeClient) Put(policy *aerospike.WritePolicy, key *aerospike.Key, bins aerospike.BinMap) aerospike.Error { ret := _m.Called(policy, key, bins)