Skip to content

Commit

Permalink
Implement forceAt and resetAt annotations
Browse files Browse the repository at this point in the history 
This makes the controller actually take the
`reconcile.fluxcd.io/forceAt` and `reconcile.fluxcd.io/resetAt` into
account.

For `reconcile.fluxcd.io/resetAt`, this means that the failure counts on
the `HelmRelease` object are reset when the token value of the
annotation equals `reconcile.fluxcd.io/requestedAt`. Allowing the
controller to start over with attempting to install or upgrade the
release until the retries count has been reached again.

For `reconcile.fluxcd.io/forceAt`, this means that a one-off Helm
install or upgrade is allowed to take place even if the object is out of
retries, in a failed state where it should be remediated, or in-sync.

Signed-off-by: Hidde Beydals <[email protected]>
  • Loading branch information
@hiddeco
hiddeco committed Nov 30, 2023
1 parent 7a15000 commit 6b7789a
Show file tree
Hide file tree
Showing 5 changed files with 146 additions and 9 deletions.
12 changes: 12 additions & 0 deletions internal/action/reset.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ const (
differentGenerationReason = "generation differs from last attempt"
differentRevisionReason = "chart version differs from last attempt"
differentValuesReason = "values differ from last attempt"
resetRequestedReason = "reset requested through annotation"
)

// MustResetFailures returns a reason and true if the HelmRelease's status
Expand All @@ -38,6 +39,12 @@ const (
// For example, a change in generation, chart version, or values.
// If no change is detected, an empty string is returned along with false.
func MustResetFailures(obj *v2.HelmRelease, chart *chart.Metadata, values chartutil.Values) (string, bool) {
// Always check if a reset is requested.
// This is done first, so that the HelmReleaseStatus.LastHandledResetAt
// field is updated even if the reset request is not handled due to other
// diverging data.
resetRequested := v2.ShouldHandleResetRequest(obj)

switch {
case obj.Status.LastAttemptedGeneration != obj.Generation:
return differentGenerationReason, true
Expand All @@ -53,5 +60,10 @@ func MustResetFailures(obj *v2.HelmRelease, chart *chart.Metadata, values chartu
return differentValuesReason, true
}
}

if resetRequested {
return resetRequestedReason, true
}

return "", false
}
27 changes: 27 additions & 0 deletions internal/action/reset_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ import (
"helm.sh/helm/v3/pkg/chartutil"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

"github.com/fluxcd/pkg/apis/meta"

v2 "github.com/fluxcd/helm-controller/api/v2beta2"
)

Expand Down Expand Up @@ -108,6 +110,31 @@ func TestMustResetFailures(t *testing.T) {
want: true,
wantReason: differentValuesReason,
},
{
name: "on reset request through annotation",
obj: &v2.HelmRelease{
ObjectMeta: metav1.ObjectMeta{
Generation: 1,
Annotations: map[string]string{
meta.ReconcileRequestAnnotation: "a",
v2.ResetRequestAnnotation: "a",
},
},
Status: v2.HelmReleaseStatus{
LastAttemptedGeneration: 1,
LastAttemptedRevision: "1.0.0",
LastAttemptedConfigDigest: "sha256:1dabc4e3cbbd6a0818bd460f3a6c9855bfe95d506c74726bc0f2edb0aecb1f4e",
},
},
chart: &chart.Metadata{
Version: "1.0.0",
},
values: chartutil.Values{
"foo": "bar",
},
want: true,
wantReason: resetRequestedReason,
},
{
name: "without change no reset",
obj: &v2.HelmRelease{
Expand Down
4 changes: 4 additions & 0 deletions internal/controller/helmrelease_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,10 @@ func (r *HelmReleaseReconciler) Reconcile(ctx context.Context, req ctrl.Request)

// Always attempt to patch the object after each reconciliation.
defer func() {
if v, ok := meta.ReconcileAnnotationValue(obj.GetAnnotations()); ok {
obj.Status.SetLastHandledReconcileRequest(v)
}

patchOpts := []patch.Option{
patch.WithFieldOwner(r.FieldManager),
patch.WithOwnedConditions{Conditions: intreconcile.OwnedConditions},
Expand Down
31 changes: 30 additions & 1 deletion internal/reconcile/atomic_release.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@ func (r *AtomicRelease) Reconcile(ctx context.Context, req *Request) error {
}
return fmt.Errorf("atomic release canceled: %w", ctx.Err())
default:
// Determine the next action to run based on the current state.
// Determine the current state of the Helm release.
log.V(logger.DebugLevel).Info("determining current state of Helm release")
state, err := DetermineReleaseState(ctx, r.configFactory, req)
if err != nil {
Expand Down Expand Up @@ -273,6 +273,13 @@ func (r *AtomicRelease) Reconcile(ctx context.Context, req *Request) error {
func (r *AtomicRelease) actionForState(ctx context.Context, req *Request, state ReleaseState) (ActionReconciler, error) {
log := ctrl.LoggerFrom(ctx)

// Determine whether we may need to force a release action.
// We do this before determining the next action to run, as otherwise we may
// end up running a Helm upgrade (due to e.g. ReleaseStatusUnmanaged) and
// then forcing an upgrade (due to the release now being in
// ReleaseStatusInSync with a yet unhandled force request).
forceRequested := v2.ShouldHandleForceRequest(req.Object)

switch state.Status {
case ReleaseStatusInSync:
log.Info("release in-sync with desired state")
Expand All @@ -291,6 +298,11 @@ func (r *AtomicRelease) actionForState(ctx context.Context, req *Request, state
// field, but should be removed in a future release.
req.Object.Status.LastAppliedRevision = req.Object.Status.History.Latest().ChartVersion

if forceRequested {
log.Info(msgWithReason("forcing upgrade for in-sync release", "force requested through annotation"))
return NewUpgrade(r.configFactory, r.eventRecorder), nil
}

return nil, nil
case ReleaseStatusLocked:
log.Info(msgWithReason("release locked", state.Reason))
Expand All @@ -299,6 +311,11 @@ func (r *AtomicRelease) actionForState(ctx context.Context, req *Request, state
log.Info(msgWithReason("release not installed", state.Reason))

if req.Object.GetInstall().GetRemediation().RetriesExhausted(req.Object) {
if forceRequested {
log.Info(msgWithReason("forcing install while out of retries", "force requested through annotation"))
return NewInstall(r.configFactory, r.eventRecorder), nil
}

return nil, fmt.Errorf("%w: cannot install release", ErrExceededMaxRetries)
}

Expand All @@ -314,6 +331,11 @@ func (r *AtomicRelease) actionForState(ctx context.Context, req *Request, state
log.Info(msgWithReason("release out-of-sync with desired state", state.Reason))

if req.Object.GetUpgrade().GetRemediation().RetriesExhausted(req.Object) {
if forceRequested {
log.Info(msgWithReason("forcing upgrade while out of retries", "force requested through annotation"))
return NewUpgrade(r.configFactory, r.eventRecorder), nil
}

return nil, fmt.Errorf("%w: cannot upgrade release", ErrExceededMaxRetries)
}

Expand Down Expand Up @@ -367,6 +389,13 @@ func (r *AtomicRelease) actionForState(ctx context.Context, req *Request, state
return NewUpgrade(r.configFactory, r.eventRecorder), nil
}

// If the force annotation is set, we can attempt to upgrade the release
// without any further checks.
if forceRequested {
log.Info(msgWithReason("forcing upgrade for failed release", "force requested through annotation"))
return NewUpgrade(r.configFactory, r.eventRecorder), nil
}

// We have exhausted the number of retries for the remediation
// strategy.
if remediation.RetriesExhausted(req.Object) && !remediation.MustRemediateLastFailure() {
Expand Down
81 changes: 73 additions & 8 deletions internal/reconcile/atomic_release_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1015,14 +1015,15 @@ func TestAtomicRelease_Reconcile_Scenarios(t *testing.T) {

func TestAtomicRelease_actionForState(t *testing.T) {
tests := []struct {
name string
releases []*helmrelease.Release
spec func(spec *v2.HelmReleaseSpec)
status func(releases []*helmrelease.Release) v2.HelmReleaseStatus
state ReleaseState
want ActionReconciler
wantEvent *corev1.Event
wantErr error
name string
releases []*helmrelease.Release
annotations map[string]string
spec func(spec *v2.HelmReleaseSpec)
status func(releases []*helmrelease.Release) v2.HelmReleaseStatus
state ReleaseState
want ActionReconciler
wantEvent *corev1.Event
wantErr error
}{
{
name: "in-sync release does not trigger any action",
Expand All @@ -1036,6 +1037,22 @@ func TestAtomicRelease_actionForState(t *testing.T) {
state: ReleaseState{Status: ReleaseStatusInSync},
want: nil,
},
{
name: "in-sync release with force annotation triggers upgrade action",
state: ReleaseState{Status: ReleaseStatusInSync},
annotations: map[string]string{
meta.ReconcileRequestAnnotation: "force",
v2.ForceRequestAnnotation: "force",
},
status: func(releases []*helmrelease.Release) v2.HelmReleaseStatus {
return v2.HelmReleaseStatus{
History: v2.Snapshots{
{Version: 1},
},
}
},
want: &Upgrade{},
},
{
name: "locked release triggers unlock action",
state: ReleaseState{Status: ReleaseStatusLocked},
Expand All @@ -1046,6 +1063,20 @@ func TestAtomicRelease_actionForState(t *testing.T) {
state: ReleaseState{Status: ReleaseStatusAbsent},
want: &Install{},
},
{
name: "absent release without remaining retries and force annotation triggers install",
annotations: map[string]string{
meta.ReconcileRequestAnnotation: "force",
v2.ForceRequestAnnotation: "force",
},
state: ReleaseState{Status: ReleaseStatusAbsent},
status: func(releases []*helmrelease.Release) v2.HelmReleaseStatus {
return v2.HelmReleaseStatus{
InstallFailures: 1,
}
},
want: &Install{},
},
{
name: "absent release without remaining retries returns error",
state: ReleaseState{Status: ReleaseStatusAbsent},
Expand Down Expand Up @@ -1181,6 +1212,22 @@ func TestAtomicRelease_actionForState(t *testing.T) {
},
want: &Upgrade{},
},
{
name: "out-of-sync release with no remaining retries and force annotation triggers upgrade",
state: ReleaseState{
Status: ReleaseStatusOutOfSync,
},
annotations: map[string]string{
meta.ReconcileRequestAnnotation: "force",
v2.ForceRequestAnnotation: "force",
},
status: func(releases []*helmrelease.Release) v2.HelmReleaseStatus {
return v2.HelmReleaseStatus{
UpgradeFailures: 1,
}
},
want: &Upgrade{},
},
{
name: "out-of-sync release with no remaining retries returns error",
state: ReleaseState{
Expand Down Expand Up @@ -1220,6 +1267,21 @@ func TestAtomicRelease_actionForState(t *testing.T) {
},
want: &Upgrade{},
},
{
name: "failed release with exhausted retries and force annotation triggers upgrade",
state: ReleaseState{Status: ReleaseStatusFailed},
annotations: map[string]string{
meta.ReconcileRequestAnnotation: "force",
v2.ForceRequestAnnotation: "force",
},
status: func(releases []*helmrelease.Release) v2.HelmReleaseStatus {
return v2.HelmReleaseStatus{
LastAttemptedReleaseAction: v2.ReleaseActionUpgrade,
UpgradeFailures: 1,
}
},
want: &Upgrade{},
},
{
name: "failed release with exhausted retries returns error",
state: ReleaseState{Status: ReleaseStatusFailed},
Expand Down Expand Up @@ -1370,6 +1432,9 @@ func TestAtomicRelease_actionForState(t *testing.T) {
g := NewWithT(t)

obj := &v2.HelmRelease{
ObjectMeta: metav1.ObjectMeta{
Annotations: tt.annotations,
},
Spec: v2.HelmReleaseSpec{
ReleaseName: mockReleaseName,
TargetNamespace: mockReleaseNamespace,
Expand Down

0 comments on commit 6b7789a

Please sign in to comment.