Merge branch 'master' into print-lint-warnings

.github/workflows/.test.yml

@@ -194,6 +194,7 @@ jobs:
         with:
           name: test-reports-${{ env.TEST_REPORT_NAME }}
           path: ./bin/testreports
+          retention-days: 1
       -
         name: Dump context
         if: failure()

.github/workflows/buildkit.yml

@@ -113,6 +113,7 @@ jobs:
           name: buildkit-${{ env.PLATFORM_PAIR }}
           path: ${{ env.DESTDIR }}/*
           if-no-files-found: error
+          retention-days: 1
 
   test:
     uses: ./.github/workflows/.test.yml

.github/workflows/dockerd.yml

@@ -68,6 +68,7 @@ jobs:
           name: dockerd
           path: /tmp/moby/dockerd
           if-no-files-found: error
+          retention-days: 1
 
   test:
     runs-on: ubuntu-22.04

.github/workflows/test-os.yml

@@ -160,6 +160,7 @@ jobs:
         with:
           name: test-reports-${{ env.TEST_REPORT_NAME }}
           path: ./bin/testreports
+          retention-days: 1
       -
         name: Dump context
         if: failure()

Dockerfile

@@ -22,19 +22,20 @@ ARG ALPINE_VERSION=3.19
 ARG XX_VERSION=1.4.0
 ARG BUILDKIT_DEBUG
 
+ARG ALPINE_ARCH=${TARGETARCH#riscv64}
+ARG ALPINE_ARCH=${ALPINE_ARCH:+"default"}
+ARG ALPINE_ARCH=${ALPINE_ARCH:-$TARGETARCH}
+
 # minio for s3 integration tests
 FROM minio/minio:${MINIO_VERSION} AS minio
 FROM minio/mc:${MINIO_MC_VERSION} AS minio-mc
 
 # alpine base for buildkit image
 # TODO: remove this when alpine image supports riscv64
-FROM alpine:${ALPINE_VERSION} AS alpine-amd64
-FROM alpine:${ALPINE_VERSION} AS alpine-arm
-FROM alpine:${ALPINE_VERSION} AS alpine-arm64
-FROM alpine:${ALPINE_VERSION} AS alpine-s390x
-FROM alpine:${ALPINE_VERSION} AS alpine-ppc64le
+FROM alpine:${ALPINE_VERSION} AS alpine-default
 FROM alpine:edge@sha256:2d01a16bab53a8405876cec4c27235d47455a7b72b75334c614f2fb0968b3f90 AS alpine-riscv64
-FROM alpine-$TARGETARCH AS alpinebase
+FROM alpine-${ALPINE_ARCH} AS alpinebase
+
 
 # xx is a helper for cross-compilation
 FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx

cache/remote.go

@@ -313,10 +313,22 @@ func (p lazyRefProvider) Info(ctx context.Context, dgst digest.Digest) (content.
 	if dgst != p.desc.Digest {
 		return content.Info{}, errdefs.ErrNotFound
 	}
-	if err := p.Unlazy(ctx); err != nil {
-		return content.Info{}, errdefs.ErrNotFound
+	info, err := p.ref.cm.ContentStore.Info(ctx, dgst)
+	if err == nil {
+		return info, nil
 	}
-	return p.ref.cm.ContentStore.Info(ctx, dgst)
+
+	if isLazy, err1 := p.ref.isLazy(ctx); err1 != nil {
+		return content.Info{}, err1
+	} else if !isLazy {
+		return content.Info{}, err
+	}
+
+	// for lazy records don't unlazy without read request
+	return content.Info{
+		Digest: p.desc.Digest,
+		Size:   p.desc.Size,
+	}, nil
 }
 
 func (p lazyRefProvider) Unlazy(ctx context.Context) error {

cache/remotecache/azblob/importer.go

@@ -18,6 +18,7 @@ import (
 	"github.com/moby/buildkit/util/contentutil"
 	"github.com/moby/buildkit/util/progress"
 	"github.com/moby/buildkit/worker"
+	digest "github.com/opencontainers/go-digest"
 	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
@@ -162,14 +163,16 @@ func (ci *importer) makeDescriptorProviderPair(l v1.CacheLayer) (*v1.DescriptorP
 		Size:        l.Annotations.Size,
 		Annotations: annotations,
 	}
+	p := &ciProvider{
+		desc:            desc,
+		containerClient: ci.containerClient,
+		Provider:        contentutil.FromFetcher(&fetcher{containerClient: ci.containerClient, config: ci.config}),
+		config:          ci.config,
+	}
 	return &v1.DescriptorProviderPair{
-		Descriptor: desc,
-		Provider: &ciProvider{
-			desc:            desc,
-			containerClient: ci.containerClient,
-			Provider:        contentutil.FromFetcher(&fetcher{containerClient: ci.containerClient, config: ci.config}),
-			config:          ci.config,
-		},
+		Descriptor:   desc,
+		Provider:     p,
+		InfoProvider: p,
 	}, nil
 }
 
@@ -213,28 +216,34 @@ type ciProvider struct {
 	checked         bool
 }
 
-func (p *ciProvider) CheckDescriptor(ctx context.Context, desc ocispecs.Descriptor) error {
-	if desc.Digest != p.desc.Digest {
-		return nil
+func (p *ciProvider) Info(ctx context.Context, dgst digest.Digest) (content.Info, error) {
+	if dgst != p.desc.Digest {
+		return content.Info{}, errors.Errorf("content not found %s", dgst)
 	}
 
 	if p.checked {
-		return nil
+		return content.Info{
+			Digest: p.desc.Digest,
+			Size:   p.desc.Size,
+		}, nil
 	}
 
 	p.checkMutex.Lock()
 	defer p.checkMutex.Unlock()
 
-	key := blobKey(p.config, desc.Digest.String())
+	key := blobKey(p.config, dgst.String())
 	exists, err := blobExists(ctx, p.containerClient, key)
 	if err != nil {
-		return err
+		return content.Info{}, err
 	}
 
 	if !exists {
-		return errors.Errorf("blob %s not found", desc.Digest)
+		return content.Info{}, errors.Errorf("blob %s not found", dgst)
 	}
 
 	p.checked = true
-	return nil
+	return content.Info{
+		Digest: p.desc.Digest,
+		Size:   p.desc.Size,
+	}, nil
 }

cache/remotecache/gha/gha.go

@@ -33,20 +33,24 @@ func init() {
 }
 
 const (
-	attrScope   = "scope"
-	attrTimeout = "timeout"
-	attrToken   = "token"
-	attrURL     = "url"
-	version     = "1"
+	attrScope      = "scope"
+	attrTimeout    = "timeout"
+	attrToken      = "token"
+	attrURL        = "url"
+	attrRepository = "repository"
+	attrGHToken    = "ghtoken"
+	version        = "1"
 
 	defaultTimeout = 10 * time.Minute
 )
 
 type Config struct {
-	Scope   string
-	URL     string
-	Token   string
-	Timeout time.Duration
+	Scope      string
+	URL        string
+	Token      string // token for the Github Cache runtime API
+	GHToken    string // token for the Github REST API
+	Repository string
+	Timeout    time.Duration
 }
 
 func getConfig(attrs map[string]string) (*Config, error) {
@@ -62,6 +66,7 @@ func getConfig(attrs map[string]string) (*Config, error) {
 	if !ok {
 		return nil, errors.Errorf("token not set for github actions cache")
 	}
+
 	timeout := defaultTimeout
 	if v, ok := attrs[attrTimeout]; ok {
 		var err error
@@ -71,10 +76,12 @@ func getConfig(attrs map[string]string) (*Config, error) {
 		}
 	}
 	return &Config{
-		Scope:   scope,
-		URL:     url,
-		Token:   token,
-		Timeout: timeout,
+		Scope:      scope,
+		URL:        url,
+		Token:      token,
+		Timeout:    timeout,
+		GHToken:    attrs[attrGHToken],
+		Repository: attrs[attrRepository],
 	}, nil
 }
 
@@ -91,9 +98,11 @@ func ResolveCacheExporterFunc() remotecache.ResolveCacheExporterFunc {
 
 type exporter struct {
 	solver.CacheExporterTarget
-	chains *v1.CacheChains
-	cache  *actionscache.Cache
-	config *Config
+	chains     *v1.CacheChains
+	cache      *actionscache.Cache
+	config     *Config
+	keyMapOnce sync.Once
+	keyMap     map[string]struct{}
 }
 
 func NewExporter(c *Config) (remotecache.Exporter, error) {
@@ -118,8 +127,12 @@ func (ce *exporter) Config() remotecache.Config {
 	}
 }
 
+func (ce *exporter) blobKeyPrefix() string {
+	return "buildkit-blob-" + version + "-"
+}
+
 func (ce *exporter) blobKey(dgst digest.Digest) string {
-	return "buildkit-blob-" + version + "-" + dgst.String()
+	return ce.blobKeyPrefix() + dgst.String()
 }
 
 func (ce *exporter) indexKey() string {
@@ -133,6 +146,35 @@ func (ce *exporter) indexKey() string {
 	return "index-" + ce.config.Scope + "-" + version + "-" + scope
 }
 
+func (ce *exporter) initActiveKeyMap(ctx context.Context) {
+	ce.keyMapOnce.Do(func() {
+		if ce.config.Repository == "" || ce.config.GHToken == "" {
+			return
+		}
+		m, err := ce.initActiveKeyMapOnce(ctx)
+		if err != nil {
+			bklog.G(ctx).Errorf("error initializing active key map: %v", err)
+			return
+		}
+		ce.keyMap = m
+	})
+}
+
+func (ce *exporter) initActiveKeyMapOnce(ctx context.Context) (map[string]struct{}, error) {
+	api, err := actionscache.NewRestAPI(ce.config.Repository, ce.config.GHToken, actionscache.Opt{
+		Client:  tracing.DefaultClient,
+		Timeout: ce.config.Timeout,
+	})
+	if err != nil {
+		return nil, err
+	}
+	keys, err := ce.cache.AllKeys(ctx, api, ce.blobKeyPrefix())
+	if err != nil {
+		return nil, err
+	}
+	return keys, nil
+}
+
 func (ce *exporter) Finalize(ctx context.Context) (map[string]string, error) {
 	// res := make(map[string]string)
 	config, descs, err := ce.chains.Marshal(ctx)
@@ -159,13 +201,25 @@ func (ce *exporter) Finalize(ctx context.Context) (map[string]string, error) {
 			return nil, errors.Wrapf(err, "failed to parse uncompressed annotation")
 		}
 		diffID = dgst
+		ce.initActiveKeyMap(ctx)
 
 		key := ce.blobKey(dgstPair.Descriptor.Digest)
-		b, err := ce.cache.Load(ctx, key)
-		if err != nil {
-			return nil, err
+
+		exists := false
+		if ce.keyMap != nil {
+			if _, ok := ce.keyMap[key]; ok {
+				exists = true
+			}
+		} else {
+			b, err := ce.cache.Load(ctx, key)
+			if err != nil {
+				return nil, err
+			}
+			if b != nil {
+				exists = true
+			}
 		}
-		if b == nil {
+		if !exists {
 			layerDone := progress.OneOff(ctx, fmt.Sprintf("writing layer %s", l.Blob))
 			ra, err := dgstPair.Provider.ReaderAt(ctx, dgstPair.Descriptor)
 			if err != nil {
@@ -260,9 +314,11 @@ func (ci *importer) makeDescriptorProviderPair(l v1.CacheLayer) (*v1.DescriptorP
 		Size:        l.Annotations.Size,
 		Annotations: annotations,
 	}
+	p := &ciProvider{desc: desc, ci: ci}
 	return &v1.DescriptorProviderPair{
-		Descriptor: desc,
-		Provider:   &ciProvider{desc: desc, ci: ci},
+		Descriptor:   desc,
+		Provider:     p,
+		InfoProvider: p,
 	}, nil
 }
 
@@ -347,13 +403,18 @@ type ciProvider struct {
 	entries map[digest.Digest]*actionscache.Entry
 }
 
-func (p *ciProvider) CheckDescriptor(ctx context.Context, desc ocispecs.Descriptor) error {
-	if desc.Digest != p.desc.Digest {
-		return nil
+func (p *ciProvider) Info(ctx context.Context, dgst digest.Digest) (content.Info, error) {
+	if dgst != p.desc.Digest {
+		return content.Info{}, errors.Errorf("content not found %s", dgst)
 	}
 
-	_, err := p.loadEntry(ctx, desc)
-	return err
+	if _, err := p.loadEntry(ctx, p.desc); err != nil {
+		return content.Info{}, err
+	}
+	return content.Info{
+		Digest: p.desc.Digest,
+		Size:   p.desc.Size,
+	}, nil
 }
 
 func (p *ciProvider) loadEntry(ctx context.Context, desc ocispecs.Descriptor) (*actionscache.Entry, error) {