Merge pull request #196 from ivanilves/v1-2-5

Bugfix release v1.2.5

Makefile

@@ -95,7 +95,7 @@ lint: fail-on-errors
 vet: ERRORS=$(shell go vet)
 vet: fail-on-errors
 
-semantic: REGEX:="^(feat|fix|docs|style|refactor|test|chore|localize)(\([a-zA-Z0-9\_\-\/]+\))?: [a-zA-Z]"
+semantic: REGEX:="^(feat|fix|docs|style|refactor|test|chore|localize)(\([a-zA-Z0-9\/_-]+\))?: [a-zA-Z]"
 semantic:
 	@if [[ -n "${RANGE}" ]]; then \
 		git log --pretty="format:%s" ${RANGE} | grep -v "Merge pull request" \

README.md

@@ -44,8 +44,8 @@ lstags -P /quay -r registry.company.io quay.io/coreos/hyperkube quay.io/coreos/f
 * `ABSENT` - present in registry, but absent locally
 * `PRESENT` -  present in registry, present locally, with local and remote digests being equal
 * `CHANGED` - present in registry, present locally, but with **different** local and remote digests
-* `ASSUMED` - **maybe** present in registry, not discovered by search, its presence assumed by user
-* `LOCAL-ONLY` - present locally, absent in registry
+* `LOCAL_ONLY` - present locally, absent in registry
+* `NOT_FOUND` - absent in registry, absent locally, probably does not exist at all
 
 ## Authentication
 You can either:

api/v1/registry/client/client.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"strconv"
 	"time"
 
 	log "github.com/sirupsen/logrus"
@@ -138,17 +139,26 @@ func (cli *RegistryClient) IsLoggedIn() bool {
 	return cli.Token != nil
 }
 
-func decodeTagNames(body io.ReadCloser) ([]string, error) {
+func decodeTagData(body io.ReadCloser) ([]string, map[string]tag.Manifest, error) {
 	tagData := struct {
-		TagNames []string `json:"tags"`
+		TagNames     []string                `json:"tags"`
+		TagManifests map[string]tag.Manifest `json:"manifest,omitempty"`
 	}{}
 
 	err := json.NewDecoder(body).Decode(&tagData)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
-	return tagData.TagNames, nil
+	tagManifests := make(map[string]tag.Manifest)
+
+	for _, manifest := range tagData.TagManifests {
+		for _, tagName := range manifest.Tags {
+			tagManifests[tagName] = manifest
+		}
+	}
+
+	return tagData.TagNames, tagManifests, nil
 }
 
 func (cli *RegistryClient) repoToken(repoPath string) (auth.Token, error) {
@@ -176,14 +186,27 @@ func (cli *RegistryClient) repoToken(repoPath string) (auth.Token, error) {
 	return repoToken, nil
 }
 
-// TagNames gets list of all tag names for the repository path specified
-func (cli *RegistryClient) TagNames(repoPath string) ([]string, error) {
+func mergeTagManifests(a, b map[string]tag.Manifest) map[string]tag.Manifest {
+	if b == nil {
+		return a
+	}
+
+	for k, v := range b {
+		a[k] = v
+	}
+
+	return a
+}
+
+// TagData gets list of all tag names and all additional data for the repository path specified
+func (cli *RegistryClient) TagData(repoPath string) ([]string, map[string]tag.Manifest, error) {
 	repoToken, err := cli.repoToken(repoPath)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
-	var allTagNames []string
+	allTagNames := make([]string, 0)
+	allTagManifests := make(map[string]tag.Manifest)
 
 	link := "/tags/list"
 	for {
@@ -196,15 +219,16 @@ func (cli *RegistryClient) TagNames(repoPath string) ([]string, error) {
 			cli.Config.RetryDelay,
 		)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 
-		tagNames, err := decodeTagNames(resp.Body)
+		tagNames, tagManifests, err := decodeTagData(resp.Body)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 
 		allTagNames = append(allTagNames, tagNames...)
+		allTagManifests = mergeTagManifests(allTagManifests, tagManifests)
 
 		if nextlink == "" {
 			break
@@ -213,7 +237,7 @@ func (cli *RegistryClient) TagNames(repoPath string) ([]string, error) {
 		link = "/tags/list?" + nextlink
 	}
 
-	return allTagNames, nil
+	return allTagNames, allTagManifests, nil
 }
 
 func (cli *RegistryClient) tagDigest(repoPath, tagName string) (string, error) {
@@ -234,12 +258,21 @@ func (cli *RegistryClient) tagDigest(repoPath, tagName string) (string, error) {
 		return "", err
 	}
 
-	digests, defined := resp.Header["Docker-Content-Digest"]
-	if !defined {
-		return "", fmt.Errorf("header 'Docker-Content-Digest' not found in HTTP response")
+	type configField struct {
+		Digest string `json:"digest"`
+	}
+	var values struct {
+		Config configField `json:"config"`
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&values); err != nil {
+		return "", err
+	}
+
+	if values.Config.Digest == "" {
+		values.Config.Digest = "this.image.is.bad.it.has.no.digest.fuuu!"
 	}
 
-	return digests[0], nil
+	return values.Config.Digest, nil
 }
 
 func (cli *RegistryClient) v1TagHistory(s string) (*tag.Options, error) {
@@ -295,7 +328,7 @@ func (cli *RegistryClient) v1TagOptions(repoPath, tagName string) (*tag.Options,
 }
 
 // Tag gets information about specified repository tag
-func (cli *RegistryClient) Tag(repoPath, tagName string) (*tag.Tag, error) {
+func (cli *RegistryClient) Tag(repoPath, tagName string, tagManifest tag.Manifest) (*tag.Tag, error) {
 	dc := make(chan string, 0)
 	ec := make(chan error, 0)
 
@@ -311,7 +344,7 @@ func (cli *RegistryClient) Tag(repoPath, tagName string) (*tag.Tag, error) {
 
 	options, err := cli.v1TagOptions(repoPath, tagName)
 	if err != nil {
-		log.Warnf("%s\n", err.Error())
+		log.Debugf("%s\n", err.Error())
 
 		options = &tag.Options{}
 	}
@@ -323,5 +356,23 @@ func (cli *RegistryClient) Tag(repoPath, tagName string) (*tag.Tag, error) {
 		return nil, err
 	}
 
+	if options.Created == 0 {
+		options.Created = extractCreated(tagManifest.TimeCreatedMs, tagManifest.TimeUploadedMs)
+	}
+
 	return tag.New(tagName, *options)
 }
+
+func extractCreated(c, u string) int64 {
+	created, err := strconv.ParseInt(c, 10, 64)
+	if err == nil && created != 0 {
+		return created / 1000
+	}
+
+	updated, err := strconv.ParseInt(u, 10, 64)
+	if err == nil && updated != 0 {
+		return updated / 1000
+	}
+
+	return 0
+}

api/v1/registry/client/request/request.go

@@ -54,7 +54,7 @@ func perform(url, auth, mode string, trace bool) (resp *http.Response, err error
 	if err != nil {
 		return nil, err
 	}
-	if resp.StatusCode != 200 {
+	if resp.StatusCode != 200 && resp.StatusCode != 404 {
 		return resp, errors.New("Bad response status: " + resp.Status + " >> " + url)
 	}
 

tag/local/local.go

@@ -18,7 +18,7 @@ func FetchTags(repo *repository.Repository, dc *dockerclient.DockerClient) (map[
 	tags := make(map[string]*tag.Tag)
 
 	for _, imageSummary := range imageSummaries {
-		repoDigest := extractRepoDigest(imageSummary.RepoDigests)
+		repoDigest := imageSummary.ID
 		tagNames := extractTagNames(imageSummary.RepoTags, repo.Name())
 
 		if repoDigest == "" {
@@ -44,17 +44,6 @@ func FetchTags(repo *repository.Repository, dc *dockerclient.DockerClient) (map[
 	return tags, nil
 }
 
-func extractRepoDigest(repoDigests []string) string {
-	if len(repoDigests) == 0 {
-		return ""
-	}
-
-	digestString := repoDigests[0]
-	digestFields := strings.Split(digestString, "@")
-
-	return digestFields[1]
-}
-
 func extractTagNames(repoTags []string, repoName string) []string {
 	tagNames := make([]string, 0)
 

tag/remote/remote.go

@@ -65,7 +65,7 @@ func FetchTags(repo *repository.Repository, username, password string) (map[stri
 		return nil, err
 	}
 
-	allTagNames, err := cli.TagNames(repo.Path())
+	allTagNames, allTagManifests, err := cli.TagData(repo.Path())
 	if err != nil {
 		return nil, err
 	}
@@ -97,12 +97,13 @@ func FetchTags(repo *repository.Repository, username, password string) (map[stri
 			go func(
 				repo *repository.Repository,
 				tagName string,
+				tagManifest tag.Manifest,
 				rc chan response,
 			) {
-				tg, err := cli.Tag(repo.Path(), tagName)
+				tg, err := cli.Tag(repo.Path(), tagName, tagManifest)
 
 				rc <- response{Tag: tg, Err: err}
-			}(repo, tagNames[tagIndex], rc)
+			}(repo, tagNames[tagIndex], allTagManifests[tagNames[tagIndex]], rc)
 
 			tagIndex++
 

tag/tag.go

@@ -28,6 +28,15 @@ type Options struct {
 	Created int64
 }
 
+// Manifest represents additional tag information structure presented by some registries
+type Manifest struct {
+	ImageSizeBytes string
+	MediaType      string
+	Tags           []string `json:"tag"`
+	TimeCreatedMs  string
+	TimeUploadedMs string
+}
+
 // SortKey returns a sort key (used to sort tags before process or display them)
 func (tg *Tag) SortKey() string {
 	return tg.GetCreatedKey() + tg.name
@@ -98,7 +107,7 @@ func (tg *Tag) GetState() string {
 
 // NeedsPull tells us if tag/image needs pull
 func (tg *Tag) NeedsPull() bool {
-	if tg.state == "ABSENT" || tg.state == "CHANGED" || tg.state == "ASSUMED" {
+	if tg.state == "ABSENT" || tg.state == "CHANGED" {
 		return true
 	}
 
@@ -107,7 +116,7 @@ func (tg *Tag) NeedsPull() bool {
 
 // NeedsPush tells us if tag/image needs push to a registry
 func (tg *Tag) NeedsPush(doUpdate bool) bool {
-	if tg.state == "ABSENT" || tg.state == "ASSUMED" || (tg.state == "CHANGED" && doUpdate) {
+	if tg.state == "ABSENT" || (tg.state == "CHANGED" && doUpdate) {
 		return true
 	}
 
@@ -161,7 +170,7 @@ func calculateState(name string, remoteTags, localTags map[string]*Tag) string {
 	}
 
 	if !definedInRegistry && definedLocally {
-		return "LOCAL-ONLY"
+		return "LOCAL_ONLY"
 	}
 
 	if definedInRegistry && definedLocally {
@@ -172,7 +181,7 @@ func calculateState(name string, remoteTags, localTags map[string]*Tag) string {
 		return "CHANGED"
 	}
 
-	return "ASSUMED"
+	return "NOT_FOUND"
 }
 
 // Join joins local tags with ones from registry, performs state processing and returns:

tag/tag_test.go

@@ -194,7 +194,7 @@ func TestJoin_ImageID(t *testing.T) {
 func TestJoin_State(t *testing.T) {
 	examples := map[string]string{
 		"latest": "CHANGED",
-		"v1.0":   "LOCAL-ONLY",
+		"v1.0":   "LOCAL_ONLY",
 		"v1.1":   "ABSENT",
 		"v1.2":   "PRESENT",
 		"v1.3.1": "CHANGED",
@@ -217,12 +217,12 @@ func TestJoin_State(t *testing.T) {
 	}
 }
 
-func TestJoin_State_WithAssumedTagNames(t *testing.T) {
+func TestJoin_State_WithNotFoundTagNames(t *testing.T) {
 	assumedTagNames := []string{"v1.3.2", "v1.4.1"}
 
 	examples := map[string]string{
 		"v1.3.2": "PRESENT",
-		"v1.4.1": "ASSUMED",
+		"v1.4.1": "NOT_FOUND",
 	}
 
 	_, _, tags := Join(getRemoteTags(), getLocalTags(), assumedTagNames)
@@ -337,10 +337,10 @@ func TestCutImageID(t *testing.T) {
 	testCases := map[string]string{
 		"sha256:249aca4f9e076c53d9fa7cb591cbc0d013f54da93c393f054f5d70c8705c8e6c": "249aca4f9e07",
 		"5bef08742407efd622d243692b79ba0055383bbce12900324f75e56f589aedb0":        "5bef08742407",
-		"sha256:031e148a88a3":                                                     "031e148a88a3",
-		"131e158a88a3":                                                            "131e158a88a3",
-		"csum:something":                                                          "something",
-		"948995":                                                                  "948995",
+		"sha256:031e148a88a3": "031e148a88a3",
+		"131e158a88a3":        "131e158a88a3",
+		"csum:something":      "something",
+		"948995":              "948995",
 	}
 
 	for passed, expected := range testCases {