Skip to content

Commit

Permalink
Merge pull request #198 from ivanilves/v1-2-7
Browse files Browse the repository at this point in the history
v1-2-7: Some refactoring, fixes and change of defaults
  • Loading branch information
vonrabbe authored Aug 27, 2019
2 parents 6ced3b6 + 3f642ce commit 751ff21
Show file tree
Hide file tree
Showing 7 changed files with 142 additions and 57 deletions.
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ CHANGED sha256:aa96c8dc3815c44d4aceaf1ee7903ce58 37c7be7a096b 2017-10-25T2
To maximize our collaboration efficiency we would humbly ask you to follow these recommendations:
* Please add reasonable description (what?/why?/etc) to your pull request :exclamation:
* Your code should pass CI (Travis) and a [pretty liberal] code review :mag:
* Your code should pass CI (CircleCI) and a [pretty liberal] code review :mag:
* If code adds or changes some logic, it should be covered by a unit test :neckbeard:
* Please, put meaningful and [semantic](https://github.com/fteem/git-semantic-commits) messages on your commits :pray:
Expand Down
65 changes: 21 additions & 44 deletions api/v1/registry/client/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,18 @@ import (
"fmt"
"io"
"net/http"
"strconv"
"time"

log "github.com/sirupsen/logrus"

"github.com/ivanilves/lstags/api/v1/registry/client/auth"
"github.com/ivanilves/lstags/api/v1/registry/client/request"
"github.com/ivanilves/lstags/tag"
"github.com/ivanilves/lstags/tag/manifest"
)

// DefaultConcurrentRequests will be used if no explicit ConcurrentRequests configured
var DefaultConcurrentRequests = 32
var DefaultConcurrentRequests = 16

// DefaultRetryDelay will be used if no explicit RetryDelay configured
var DefaultRetryDelay = 30 * time.Second
Expand Down Expand Up @@ -139,26 +139,24 @@ func (cli *RegistryClient) IsLoggedIn() bool {
return cli.Token != nil
}

func decodeTagData(body io.ReadCloser) ([]string, map[string]tag.Manifest, error) {
func decodeTagData(body io.ReadCloser) ([]string, map[string]manifest.Manifest, error) {
tagData := struct {
TagNames []string `json:"tags"`
TagManifests map[string]tag.Manifest `json:"manifest,omitempty"`
RawManifests map[string]manifest.Raw `json:"manifest,omitempty"`
}{}

err := json.NewDecoder(body).Decode(&tagData)
if err != nil {
if err := json.NewDecoder(body).Decode(&tagData); err != nil {
return nil, nil, err
}

tagManifests := make(map[string]tag.Manifest)

for _, manifest := range tagData.TagManifests {
for _, tagName := range manifest.Tags {
tagManifests[tagName] = manifest
}
tagManifests, err := manifest.ParseMap(tagData.RawManifests)
if err != nil {
return nil, nil, err
}

return tagData.TagNames, tagManifests, nil
tagManifests = manifest.MapByTag(tagManifests)

return tagData.TagNames, manifest.MapByTag(tagManifests), nil
}

func (cli *RegistryClient) repoToken(repoPath string) (auth.Token, error) {
Expand Down Expand Up @@ -186,27 +184,15 @@ func (cli *RegistryClient) repoToken(repoPath string) (auth.Token, error) {
return repoToken, nil
}

func mergeTagManifests(a, b map[string]tag.Manifest) map[string]tag.Manifest {
if b == nil {
return a
}

for k, v := range b {
a[k] = v
}

return a
}

// TagData gets list of all tag names and all additional data for the repository path specified
func (cli *RegistryClient) TagData(repoPath string) ([]string, map[string]tag.Manifest, error) {
func (cli *RegistryClient) TagData(repoPath string) ([]string, map[string]manifest.Manifest, error) {
repoToken, err := cli.repoToken(repoPath)
if err != nil {
return nil, nil, err
}

allTagNames := make([]string, 0)
allTagManifests := make(map[string]tag.Manifest)
allTagManifests := make(map[string]manifest.Manifest)

link := "/tags/list"
for {
Expand All @@ -228,7 +214,7 @@ func (cli *RegistryClient) TagData(repoPath string) ([]string, map[string]tag.Ma
}

allTagNames = append(allTagNames, tagNames...)
allTagManifests = mergeTagManifests(allTagManifests, tagManifests)
allTagManifests = manifest.MergeMaps(allTagManifests, tagManifests)

if nextlink == "" {
break
Expand Down Expand Up @@ -258,6 +244,11 @@ func (cli *RegistryClient) tagDigest(repoPath, tagName string) (string, error) {
return "", err
}

digests, defined := resp.Header["Docker-Content-Digest"]
if defined {
return digests[0], nil
}

type configField struct {
Digest string `json:"digest"`
}
Expand Down Expand Up @@ -328,7 +319,7 @@ func (cli *RegistryClient) v1TagOptions(repoPath, tagName string) (*tag.Options,
}

// Tag gets information about specified repository tag
func (cli *RegistryClient) Tag(repoPath, tagName string, tagManifest tag.Manifest) (*tag.Tag, error) {
func (cli *RegistryClient) Tag(repoPath, tagName string, tagManifest manifest.Manifest) (*tag.Tag, error) {
dc := make(chan string, 0)
ec := make(chan error, 0)

Expand Down Expand Up @@ -357,22 +348,8 @@ func (cli *RegistryClient) Tag(repoPath, tagName string, tagManifest tag.Manifes
}

if options.Created == 0 {
options.Created = extractCreated(tagManifest.TimeCreatedMs, tagManifest.TimeUploadedMs)
options.Created = tagManifest.Created()
}

return tag.New(tagName, *options)
}

func extractCreated(c, u string) int64 {
created, err := strconv.ParseInt(c, 10, 64)
if err == nil && created != 0 {
return created / 1000
}

updated, err := strconv.ParseInt(u, 10, 64)
if err == nil && updated != 0 {
return updated / 1000
}

return 0
}
2 changes: 1 addition & 1 deletion main.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ type Options struct {
PushPrefix string `short:"R" long:"push-prefix" description:"[Re]Push pulled images with a specified repo path prefix" env:"PUSH_PREFIX"`
PushUpdate bool `short:"U" long:"push-update" description:"Update our pushed images if remote image digest changes" env:"PUSH_UPDATE"`
PathSeparator string `short:"s" long:"path-separator" default:"/" description:"Configure path separator for registries that only allow single folder depth" env:"PATH_SEPARATOR"`
ConcurrentRequests int `short:"c" long:"concurrent-requests" default:"32" description:"Limit of concurrent requests to the registry" env:"CONCURRENT_REQUESTS"`
ConcurrentRequests int `short:"c" long:"concurrent-requests" default:"16" description:"Limit of concurrent requests to the registry" env:"CONCURRENT_REQUESTS"`
WaitBetween time.Duration `short:"w" long:"wait-between" default:"0" description:"Time to wait between batches of requests (incl. pulls and pushes)" env:"WAIT_BETWEEN"`
RetryRequests int `short:"y" long:"retry-requests" default:"2" description:"Number of retries for failed Docker registry requests" env:"RETRY_REQUESTS"`
RetryDelay time.Duration `short:"D" long:"retry-delay" default:"30s" description:"Delay between retries of failed registry requests" env:"RETRY_DELAY"`
Expand Down
13 changes: 12 additions & 1 deletion tag/local/local.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ func FetchTags(repo *repository.Repository, dc *dockerclient.DockerClient) (map[
tags := make(map[string]*tag.Tag)

for _, imageSummary := range imageSummaries {
repoDigest := imageSummary.ID
repoDigest := extractRepoDigest(imageSummary.RepoDigests, imageSummary.ID)
tagNames := extractTagNames(imageSummary.RepoTags, repo.Name())

if repoDigest == "" {
Expand All @@ -44,6 +44,17 @@ func FetchTags(repo *repository.Repository, dc *dockerclient.DockerClient) (map[
return tags, nil
}

func extractRepoDigest(repoDigests []string, defaultValue string) string {
if len(repoDigests) == 0 {
return defaultValue
}

digestString := repoDigests[0]
digestFields := strings.Split(digestString, "@")

return digestFields[1]
}

func extractTagNames(repoTags []string, repoName string) []string {
tagNames := make([]string, 0)

Expand Down
105 changes: 105 additions & 0 deletions tag/manifest/manifest.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
package manifest

import (
"strconv"
)

// Manifest is an additional tag information presented by some registries (e.g. GCR)
type Manifest struct {
ID string
ImageSizeBytes int64
MediaType string
Tags []string
TimeCreated int64
TimeUploaded int64
}

// Created gets image/tag creation date
func (m Manifest) Created() int64 {
if m.TimeCreated != 0 {
return m.TimeCreated
}

return m.TimeUploaded
}

// Raw embodies raw, unprocessed manifest structure
type Raw struct {
ImageSizeBytes string
MediaType string
Tags []string `json:"tag"`
TimeCreatedMs string
TimeUploadedMs string
}

// Parse parses raw manifest and returns a normal one
func Parse(id string, r Raw) (*Manifest, error) {
imageSizeBytes, err := strconv.ParseInt(r.ImageSizeBytes, 10, 64)
if err != nil {
return nil, err
}

timeCreated, err := strconv.ParseInt(r.TimeCreatedMs, 10, 64)
if err != nil {
return nil, err
}
timeCreated = timeCreated / 1000

timeUploaded, err := strconv.ParseInt(r.TimeUploadedMs, 10, 64)
if err != nil {
return nil, err
}
timeUploaded = timeUploaded / 1000

return &Manifest{
ID: id,
ImageSizeBytes: imageSizeBytes,
MediaType: r.MediaType,
Tags: r.Tags,
TimeCreated: timeCreated,
TimeUploaded: timeUploaded,
}, nil
}

// ParseMap does Parse() over a map of passed raw manifests
func ParseMap(rs map[string]Raw) (map[string]Manifest, error) {
manifests := make(map[string]Manifest)

for k, v := range rs {
m, err := Parse(k, v)
if err != nil {
return nil, err
}

manifests[k] = *m
}

return manifests, nil
}

// MapByTag maps passed manifests by their tag names
// By default manifests are mapped by their digests.
func MapByTag(manifests map[string]Manifest) map[string]Manifest {
mappedByTag := make(map[string]Manifest)

for _, v := range manifests {
for _, tagName := range v.Tags {
mappedByTag[tagName] = v
}
}

return mappedByTag
}

// MergeMaps merges two passed manifest maps into a single one
func MergeMaps(mapA, mapB map[string]Manifest) map[string]Manifest {
if mapB == nil {
return mapA
}

for k, v := range mapB {
mapA[k] = v
}

return mapA
}
3 changes: 2 additions & 1 deletion tag/remote/remote.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (

"github.com/ivanilves/lstags/repository"
"github.com/ivanilves/lstags/tag"
"github.com/ivanilves/lstags/tag/manifest"

"github.com/ivanilves/lstags/api/v1/registry/client"
)
Expand Down Expand Up @@ -97,7 +98,7 @@ func FetchTags(repo *repository.Repository, username, password string) (map[stri
go func(
repo *repository.Repository,
tagName string,
tagManifest tag.Manifest,
tagManifest manifest.Manifest,
rc chan response,
) {
tg, err := cli.Tag(repo.Path(), tagName, tagManifest)
Expand Down
9 changes: 0 additions & 9 deletions tag/tag.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,15 +28,6 @@ type Options struct {
Created int64
}

// Manifest represents additional tag information structure presented by some registries
type Manifest struct {
ImageSizeBytes string
MediaType string
Tags []string `json:"tag"`
TimeCreatedMs string
TimeUploadedMs string
}

// SortKey returns a sort key (used to sort tags before process or display them)
func (tg *Tag) SortKey() string {
return tg.GetCreatedKey() + tg.name
Expand Down

0 comments on commit 751ff21

Please sign in to comment.