Skip to content

Commit

Permalink
Use Block CIM layers for container RootFS
Browse files Browse the repository at this point in the history 
This commit adds the ability to parse block CIM layer mounts and to mount the merged block
CIMs to be used as a rootfs for a container.

Signed-off-by: Amit Barve <[email protected]>
  • Loading branch information
@ambarve
ambarve committed Dec 17, 2024
1 parent 1f4af71 commit 97956fb
Show file tree
Hide file tree
Showing 5 changed files with 325 additions and 56 deletions.
7 changes: 5 additions & 2 deletions internal/layers/helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,8 +75,11 @@ const (
// parent layer CIMs
parentLayerCimPathsFlag = "parentCimPaths="

LegacyMountType string = "windows-layer"
CimFSMountType string = "CimFS"
legacyMountType string = "windows-layer"
forkedCIMMountType string = "CimFS"
blockCIMMountType string = "BlockCIM"
blockCIMTypeFlag string = "blockCIMType="
mergedCIMPathFlag string = "mergedCIMPath="
)

// getOptionAsArray finds if there is an option which has the given prefix and if such an
Expand Down
166 changes: 119 additions & 47 deletions internal/layers/wcow_mount.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,14 @@ import (

"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"go.opencensus.io/trace"
"golang.org/x/sys/windows"

"github.com/Microsoft/hcsshim/computestorage"
hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2"
"github.com/Microsoft/hcsshim/internal/hcserror"
"github.com/Microsoft/hcsshim/internal/log"
"github.com/Microsoft/hcsshim/internal/oc"
"github.com/Microsoft/hcsshim/internal/resources"
"github.com/Microsoft/hcsshim/internal/uvm"
"github.com/Microsoft/hcsshim/internal/uvm/scsi"
Expand All @@ -37,6 +39,11 @@ func MountWCOWLayers(ctx context.Context, containerID string, vm *uvm.UtilityVM,
return mountProcessIsolatedForkedCimLayers(ctx, containerID, l)
}
return nil, nil, fmt.Errorf("hyperv isolated containers aren't supported with forked cim layers")
case *wcowBlockCIMLayers:
if vm == nil {
return mountProcessIsolatedBlockCIMLayers(ctx, containerID, l)
}
return nil, nil, fmt.Errorf("hyperv isolated containers aren't supported with block cim layers")
default:
return nil, nil, fmt.Errorf("invalid layer type %T", wl)
}
Expand Down Expand Up @@ -171,53 +178,43 @@ func mountProcessIsolatedWCIFSLayers(ctx context.Context, l *wcowWCIFSLayers) (_
}, nil
}

// wcowHostForkedCIMLayerCloser is used to cleanup forked CIM layers mounted on the host for process isolated
// containers
type wcowHostForkedCIMLayerCloser struct {
scratchLayerData
containerID string
}

func (l *wcowHostForkedCIMLayerCloser) Release(ctx context.Context) error {
mountPath, err := wclayer.GetLayerMountPath(ctx, l.scratchLayerPath)
if err != nil {
return err
}

if err = computestorage.DetachOverlayFilter(ctx, mountPath, hcsschema.UnionFS); err != nil {
return err
}

if err = cimlayer.CleanupContainerMounts(l.containerID); err != nil {
return err
}
return wclayer.DeactivateLayer(ctx, l.scratchLayerPath)
}
// Handles the common processing for mounting all 3 types of cimfs layers. This involves
// mounting the scratch, attaching the filter and preparing the return values.
// `volume` is the path to the volume at which read only layer CIMs are mounted.
func mountProcessIsolatedCimLayersCommon(ctx context.Context, containerID string, volume string, s *scratchLayerData) (_ *MountedWCOWLayers, _ resources.ResourceCloser, err error) {
ctx, span := oc.StartSpan(ctx, "mountProcessIsolatedCimLayersCommon")
defer func() {
oc.SetSpanStatus(span, err)
span.End()
}()
span.AddAttributes(
trace.StringAttribute("scratch path", s.scratchLayerPath),
trace.StringAttribute("mounted CIM volume", volume))

func mountProcessIsolatedForkedCimLayers(ctx context.Context, containerID string, l *wcowForkedCIMLayers) (_ *MountedWCOWLayers, _ resources.ResourceCloser, err error) {
if err = wclayer.ActivateLayer(ctx, l.scratchLayerPath); err != nil {
return nil, nil, err
}
rcl := &resources.ResourceCloserList{}
defer func() {
if err != nil {
_ = wclayer.DeactivateLayer(ctx, l.scratchLayerPath)
if rErr := rcl.Release(ctx); rErr != nil {
log.G(ctx).WithError(err).Warnf("mount process isolated cim layers common, undo failed with: %s", rErr)
}
}
}()

mountPath, err := wclayer.GetLayerMountPath(ctx, l.scratchLayerPath)
if err != nil {
if err = wclayer.ActivateLayer(ctx, s.scratchLayerPath); err != nil {
return nil, nil, err
}
rcl.AddFunc(func(uCtx context.Context) error {
return wclayer.DeactivateLayer(uCtx, s.scratchLayerPath)
})

volume, err := cimlayer.MountForkedCimLayer(ctx, l.layers[0].cimPath, containerID)
mountPath, err := wclayer.GetLayerMountPath(ctx, s.scratchLayerPath)
if err != nil {
return nil, nil, fmt.Errorf("mount layer cim: %w", err)
return nil, nil, err
}
defer func() {
if err != nil {
_ = cimlayer.UnmountCimLayer(ctx, volume)
}
}()
log.G(ctx).WithFields(logrus.Fields{
"scratch": s.scratchLayerPath,
"mounted path": mountPath,
}).Debug("scratch activated")

layerID, err := cimlayer.LayerID(volume)
if err != nil {
Expand All @@ -239,22 +236,97 @@ func mountProcessIsolatedForkedCimLayers(ctx context.Context, containerID string
if err = computestorage.AttachOverlayFilter(ctx, mountPath, layerData); err != nil {
return nil, nil, err
}
rcl.AddFunc(func(uCtx context.Context) error {
return computestorage.DetachOverlayFilter(uCtx, mountPath, hcsschema.UnionFS)
})

log.G(ctx).WithField("layer data", layerData).Debug("unionFS filter attached")

return &MountedWCOWLayers{
RootFS: mountPath,
MountedLayerPaths: []MountedWCOWLayer{{
LayerID: layerID,
MountedPath: volume,
}},
}, rcl, nil
}

func mountProcessIsolatedForkedCimLayers(ctx context.Context, containerID string, l *wcowForkedCIMLayers) (_ *MountedWCOWLayers, _ resources.ResourceCloser, err error) {
ctx, span := oc.StartSpan(ctx, "mountProcessIsolatedForkedCimLayers")
defer func() {
oc.SetSpanStatus(span, err)
span.End()
}()

rcl := &resources.ResourceCloserList{}
defer func() {
if err != nil {
if rErr := rcl.Release(ctx); rErr != nil {
log.G(ctx).WithError(err).Warnf("mount process isolated forked CIM layers, undo failed with: %s", rErr)
}
}
}()

volume, err := cimlayer.MountForkedCimLayer(ctx, l.layers[0].cimPath, containerID)
if err != nil {
return nil, nil, fmt.Errorf("mount forked layer cim: %w", err)
}
rcl.AddFunc(func(uCtx context.Context) error {
return cimlayer.UnmountCimLayer(uCtx, volume)
})

mountedLayers, closer, err := mountProcessIsolatedCimLayersCommon(ctx, containerID, volume, &l.scratchLayerData)
if err != nil {
return nil, nil, err
}
return mountedLayers, rcl.Add(closer), nil
}

func mountProcessIsolatedBlockCIMLayers(ctx context.Context, containerID string, l *wcowBlockCIMLayers) (_ *MountedWCOWLayers, _ resources.ResourceCloser, err error) {
ctx, span := oc.StartSpan(ctx, "mountProcessIsolatedBlockCIMLayers")
defer func() {
oc.SetSpanStatus(span, err)
span.End()
}()

var volume string

rcl := &resources.ResourceCloserList{}
defer func() {
if err != nil {
_ = computestorage.DetachOverlayFilter(ctx, mountPath, hcsschema.UnionFS)
if rErr := rcl.Release(ctx); rErr != nil {
log.G(ctx).WithError(err).Warnf("mount process isolated forked CIM layers, undo failed with: %s", rErr)
}
}
}()

return &MountedWCOWLayers{
RootFS: mountPath,
MountedLayerPaths: []MountedWCOWLayer{{
LayerID: layerID,
MountedPath: volume,
}},
}, &wcowHostForkedCIMLayerCloser{
containerID: containerID,
scratchLayerData: l.scratchLayerData,
}, nil
log.G(ctx).WithFields(logrus.Fields{
"scratch": l.scratchLayerPath,
"merged layer": l.mergedLayer,
"parent layers": l.parentLayers,
}).Debug("mounting process isolated block CIM layers")

if len(l.parentLayers) > 1 {
volume, err = cimlayer.MergeMountBlockCIMLayer(ctx, l.mergedLayer, l.parentLayers, containerID)
} else {
volume, err = cimlayer.MountBlockCIMLayer(ctx, l.parentLayers[0], containerID)
}
if err != nil {
return nil, nil, fmt.Errorf("mount block CIM layers: %w", err)
}
rcl.AddFunc(func(uCtx context.Context) error {
return cimlayer.UnmountCimLayer(uCtx, volume)
})

log.G(ctx).WithField("volume", volume).Debug("mounted blockCIM layers for process isolated container")

mountedLayers, layerCloser, err := mountProcessIsolatedCimLayersCommon(ctx, containerID, volume, &l.scratchLayerData)
if err != nil {
return nil, nil, fmt.Errorf("failed mount CIM layers common: %w", err)
}
rcl.Add(layerCloser)

return mountedLayers, rcl, nil
}

type wcowIsolatedWCIFSLayerCloser struct {
Expand Down
95 changes: 90 additions & 5 deletions internal/layers/wcow_parse.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,18 @@ package layers

import (
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"

"github.com/containerd/containerd/api/types"

"github.com/Microsoft/hcsshim/internal/copyfile"
"github.com/Microsoft/hcsshim/internal/uvm"
"github.com/Microsoft/hcsshim/internal/uvmfolder"
"github.com/Microsoft/hcsshim/pkg/cimfs"
)

// WCOW image layers is a tagging interface that all WCOW layers MUST implement. This is
Expand Down Expand Up @@ -67,6 +70,17 @@ type wcowForkedCIMLayers struct {
layers []forkedCIMLayer
}

// Represents CIM layers where each layer is stored in a block device or in a single file
// and multiple such layer CIMs are merged before mounting them. Currently can only be
// used for process isolated containers.
type wcowBlockCIMLayers struct {
scratchLayerData
// parent layers in order [layerN (top-most), layerN-1,..layer0 (base)]
parentLayers []*cimfs.BlockCIM
// a merged layer is prepared by combining all parent layers
mergedLayer *cimfs.BlockCIM
}

func parseForkedCimMount(m *types.Mount) (*wcowForkedCIMLayers, error) {
parentLayerPaths, err := getOptionAsArray(m, parentLayerPathsFlag)
if err != nil {
Expand Down Expand Up @@ -94,8 +108,77 @@ func parseForkedCimMount(m *types.Mount) (*wcowForkedCIMLayers, error) {
}, nil
}

// ParseWCOWLayers parses the layers provided by containerd into the format understood by hcsshim and prepares
// them for mounting.
// TODO(ambarve): The code to parse a mount type should be in a separate package/module
// somewhere and then should be consumed by both hcsshim & containerd from there.
func parseBlockCIMMount(m *types.Mount) (*wcowBlockCIMLayers, error) {
var (
parentPaths []string
layerType cimfs.BlockCIMType
mergedCIMPath string
)

for _, option := range m.Options {
if val, ok := strings.CutPrefix(option, parentLayerCimPathsFlag); ok {
err := json.Unmarshal([]byte(val), &parentPaths)
if err != nil {
return nil, err
}
} else if val, ok = strings.CutPrefix(option, blockCIMTypeFlag); ok {
if val == "device" {
layerType = cimfs.BlockCIMTypeDevice
} else if val == "file" {
layerType = cimfs.BlockCIMTypeSingleFile
} else {
return nil, fmt.Errorf("invalid block CIM type `%s`", val)
}
} else if val, ok = strings.CutPrefix(option, mergedCIMPathFlag); ok {
mergedCIMPath = val
}
}

if len(parentPaths) == 0 {
return nil, fmt.Errorf("need at least 1 parent layer")
}
if layerType == cimfs.BlockCIMTypeNone {
return nil, fmt.Errorf("BlockCIM type not provided")
}
if mergedCIMPath == "" && len(parentPaths) > 1 {
return nil, fmt.Errorf("merged CIM path not provided")
}

var (
parentLayers []*cimfs.BlockCIM
mergedLayer *cimfs.BlockCIM
)

if len(parentPaths) > 1 {
// for single parent layers merge won't be done
mergedLayer = &cimfs.BlockCIM{
Type: layerType,
BlockPath: filepath.Dir(mergedCIMPath),
CimName: filepath.Base(mergedCIMPath),
}
}

for _, p := range parentPaths {
parentLayers = append(parentLayers, &cimfs.BlockCIM{
Type: layerType,
BlockPath: filepath.Dir(p),
CimName: filepath.Base(p),
})
}

return &wcowBlockCIMLayers{
scratchLayerData: scratchLayerData{
scratchLayerPath: m.Source,
},
parentLayers: parentLayers,
mergedLayer: mergedLayer,
}, nil
}

// ParseWCOWLayers parses the layers provided by containerd into the format understood by
// hcsshim and prepares them for mounting.
func ParseWCOWLayers(rootfs []*types.Mount, layerFolders []string) (WCOWLayers, error) {
if err := validateRootfsAndLayers(rootfs, layerFolders); err != nil {
return nil, err
Expand All @@ -112,7 +195,7 @@ func ParseWCOWLayers(rootfs []*types.Mount, layerFolders []string) (WCOWLayers,

m := rootfs[0]
switch m.Type {
case LegacyMountType:
case legacyMountType:
parentLayers, err := getOptionAsArray(m, parentLayerPathsFlag)
if err != nil {
return nil, err
Expand All @@ -123,8 +206,10 @@ func ParseWCOWLayers(rootfs []*types.Mount, layerFolders []string) (WCOWLayers,
},
layerPaths: parentLayers,
}, nil
case CimFSMountType:
case forkedCIMMountType:
return parseForkedCimMount(m)
case blockCIMMountType:
return parseBlockCIMMount(m)
default:
return nil, fmt.Errorf("invalid windows mount type: '%s'", m.Type)
}
Expand All @@ -146,7 +231,7 @@ func GetWCOWUVMBootFilesFromLayers(ctx context.Context, rootfs []*types.Mount, l
} else {
m := rootfs[0]
switch m.Type {
case LegacyMountType:
case legacyMountType:
parentLayers, err = getOptionAsArray(m, parentLayerPathsFlag)
if err != nil {
return nil, err
Expand Down
Loading

0 comments on commit 97956fb

Please sign in to comment.