diff --git a/api/v1/registry/client/cache/cache.go b/api/v1/registry/client/cache/cache.go new file mode 100644 index 0000000..01442a9 --- /dev/null +++ b/api/v1/registry/client/cache/cache.go @@ -0,0 +1,51 @@ +package cache + +import ( + "sync" + "time" + + log "github.com/sirupsen/logrus" + + "github.com/ivanilves/lstags/api/v1/registry/client/auth" +) + +// WaitBetween defines how much we will wait between batches of requests +var WaitBetween time.Duration + +// Token is a structure to hold already obtained tokens +// Prevents excess HTTP requests to be made (error 429) +var Token = token{items: make(map[string]auth.Token)} + +type token struct { + items map[string]auth.Token + mux sync.Mutex +} + +// Exists tells if passed key is already present in cache +func (t *token) Exists(key string) bool { + t.mux.Lock() + defer t.mux.Unlock() + + _, defined := t.items[key] + + if !defined && WaitBetween != 0 { + log.Debugf("Locking token operations for %v (key: %s)", WaitBetween, key) + time.Sleep(WaitBetween) + } + + return defined +} + +// Get gets token for a passed key +func (t *token) Get(key string) auth.Token { + return t.items[key] +} + +// Get sets token for a passed key +func (t *token) Set(key string, value auth.Token) { + t.mux.Lock() + + t.items[key] = value + + t.mux.Unlock() +} diff --git a/api/v1/registry/client/client.go b/api/v1/registry/client/client.go index 3f57645..2a26860 100644 --- a/api/v1/registry/client/client.go +++ b/api/v1/registry/client/client.go @@ -11,6 +11,7 @@ import ( log "github.com/sirupsen/logrus" "github.com/ivanilves/lstags/api/v1/registry/client/auth" + "github.com/ivanilves/lstags/api/v1/registry/client/cache" "github.com/ivanilves/lstags/api/v1/registry/client/request" "github.com/ivanilves/lstags/tag" "github.com/ivanilves/lstags/tag/manifest" @@ -20,7 +21,7 @@ import ( var DefaultConcurrentRequests = 16 // DefaultRetryDelay will be used if no explicit RetryDelay configured -var DefaultRetryDelay = 30 * time.Second +var DefaultRetryDelay = 2 * time.Second // MaxConcurrentRequests is a hard limit for simultaneous registry requests const MaxConcurrentRequests = 256 @@ -110,23 +111,40 @@ func (cli *RegistryClient) Ping() error { return nil } -// Login logs in to the registry (returns error, if failed) -func (cli *RegistryClient) Login(username, password string) error { +func (cli *RegistryClient) registryToken(username, password string) (auth.Token, error) { tk, err := auth.NewToken(cli.URL(), username, password, "registry:catalog:*") if err != nil { - log.Debugf("Try to login with less permissions (repository:catalog:*)") + if cli.Config.WaitBetween == 0 { + log.Debugf("Try to login with less permissions (repository:catalog:*)") + } else { + log.Debugf("Try to login with less permissions (repository:catalog:*) [after waiting %v]", cli.Config.WaitBetween) + time.Sleep(cli.Config.WaitBetween) + } tk, err = auth.NewToken(cli.URL(), username, password, "repository:catalog:*") - if err != nil { if username == "" && password == "" { - return nil + return tk, nil } + return tk, err + } + } + + return tk, nil +} + +// Login logs in to the registry (returns error, if failed) +func (cli *RegistryClient) Login(username, password string) error { + if !cache.Token.Exists(cli.registry) { + tk, err := cli.registryToken(username, password) + if err != nil { return err } + + cache.Token.Set(cli.registry, tk) } - cli.Token = tk + cli.Token = cache.Token.Get(cli.registry) cli.username = username cli.password = password @@ -169,19 +187,23 @@ func (cli *RegistryClient) repoToken(repoPath string) (auth.Token, error) { return cli.RepoTokens[repoPath], nil } - repoToken, err := auth.NewToken( - cli.URL(), - cli.username, - cli.password, - "repository:"+repoPath+":pull", - ) - if err != nil { - return nil, err + if !cache.Token.Exists(repoPath) { + repoToken, err := auth.NewToken( + cli.URL(), + cli.username, + cli.password, + "repository:"+repoPath+":pull", + ) + if err != nil { + return nil, err + } + + cache.Token.Set(repoPath, repoToken) } - cli.RepoTokens[repoPath] = repoToken + cli.RepoTokens[repoPath] = cache.Token.Get(repoPath) - return repoToken, nil + return cli.RepoTokens[repoPath], nil } // TagData gets list of all tag names and all additional data for the repository path specified diff --git a/api/v1/registry/client/request/request.go b/api/v1/registry/client/request/request.go index de90d8e..b59f0d0 100644 --- a/api/v1/registry/client/request/request.go +++ b/api/v1/registry/client/request/request.go @@ -89,7 +89,7 @@ func Perform(url, auth, mode string, trace bool, retries int, delay time.Duratio } if resp != nil { - if resp.StatusCode >= 400 && resp.StatusCode < 500 { + if resp.StatusCode != 429 && resp.StatusCode >= 400 && resp.StatusCode < 500 { return nil, "", err } } diff --git a/api/v1/v1.go b/api/v1/v1.go index 6f29a2b..5bef0ce 100644 --- a/api/v1/v1.go +++ b/api/v1/v1.go @@ -18,6 +18,7 @@ import ( log "github.com/sirupsen/logrus" "github.com/ivanilves/lstags/api/v1/collection" + "github.com/ivanilves/lstags/api/v1/registry/client/cache" dockerclient "github.com/ivanilves/lstags/docker/client" dockerconfig "github.com/ivanilves/lstags/docker/config" "github.com/ivanilves/lstags/repository" @@ -552,6 +553,8 @@ func New(config Config) (*API, error) { remote.RetryRequests = config.RetryRequests remote.RetryDelay = config.RetryDelay + cache.WaitBetween = config.WaitBetween + if config.InsecureRegistryEx != "" { repository.InsecureRegistryEx = config.InsecureRegistryEx } diff --git a/main.go b/main.go index 72b4d72..6b63290 100644 --- a/main.go +++ b/main.go @@ -31,7 +31,7 @@ type Options struct { ConcurrentRequests int `short:"c" long:"concurrent-requests" default:"16" description:"Limit of concurrent requests to the registry" env:"CONCURRENT_REQUESTS"` WaitBetween time.Duration `short:"w" long:"wait-between" default:"0" description:"Time to wait between batches of requests (incl. pulls and pushes)" env:"WAIT_BETWEEN"` RetryRequests int `short:"y" long:"retry-requests" default:"2" description:"Number of retries for failed Docker registry requests" env:"RETRY_REQUESTS"` - RetryDelay time.Duration `short:"D" long:"retry-delay" default:"30s" description:"Delay between retries of failed registry requests" env:"RETRY_DELAY"` + RetryDelay time.Duration `short:"D" long:"retry-delay" default:"2s" description:"Delay between retries of failed registry requests" env:"RETRY_DELAY"` InsecureRegistryEx string `short:"I" long:"insecure-registry-ex" description:"Expression to match insecure registry hostnames" env:"INSECURE_REGISTRY_EX"` TraceRequests bool `short:"T" long:"trace-requests" description:"Trace Docker registry HTTP requests" env:"TRACE_REQUESTS"` DoNotFail bool `short:"N" long:"do-not-fail" description:"Do not fail on non-critical errors (could be dangerous!)" env:"DO_NOT_FAIL"`