Try storing targetgroups encoded in a tag

aws/adapter.go

@@ -702,7 +702,7 @@ func (a *Adapter) UpdateTargetGroupsAndAutoScalingGroups(stacks []*Stack, proble
 // All the required resources (listeners and target group) are created in a
 // transactional fashion.
 // Failure to create the stack causes it to be deleted automatically.
-func (a *Adapter) CreateStack(certificateARNs []string, scheme, securityGroup, owner, sslPolicy, ipAddressType, wafWebACLID string, cwAlarms CloudWatchAlarmList, loadBalancerType string, http2 bool) (string, error) {
+func (a *Adapter) CreateStack(certificateARNs []string, scheme, securityGroup, owner, sslPolicy, ipAddressType, wafWebACLID string, cwAlarms CloudWatchAlarmList, loadBalancerType string, http2 bool, stackTags map[string]string) (string, error) {
 	certARNs := make(map[string]time.Time, len(certificateARNs))
 	for _, arn := range certificateARNs {
 		certARNs[arn] = time.Time{}
@@ -754,7 +754,7 @@ func (a *Adapter) CreateStack(certificateARNs []string, scheme, securityGroup, o
 		httpRedirectToHTTPS:               a.httpRedirectToHTTPS,
 		nlbCrossZone:                      a.nlbCrossZone,
 		http2:                             http2,
-		tags:                              a.stackTags,
+		tags:                              mergeTags(a.stackTags, stackTags),
 		internalDomains:                   a.internalDomains,
 		denyInternalDomains:               a.denyInternalDomains,
 		denyInternalDomainsResponse: denyResp{
@@ -767,7 +767,7 @@ func (a *Adapter) CreateStack(certificateARNs []string, scheme, securityGroup, o
 	return createStack(a.cloudformation, spec)
 }
 
-func (a *Adapter) UpdateStack(stackName string, certificateARNs map[string]time.Time, scheme, securityGroup, owner, sslPolicy, ipAddressType, wafWebACLID string, cwAlarms CloudWatchAlarmList, loadBalancerType string, http2 bool) (string, error) {
+func (a *Adapter) UpdateStack(stackName string, certificateARNs map[string]time.Time, scheme, securityGroup, owner, sslPolicy, ipAddressType, wafWebACLID string, cwAlarms CloudWatchAlarmList, loadBalancerType string, http2 bool, stackTags map[string]string) (string, error) {
 	if _, ok := SSLPolicies[sslPolicy]; !ok {
 		return "", fmt.Errorf("invalid SSLPolicy '%s' defined", sslPolicy)
 	}
@@ -810,7 +810,7 @@ func (a *Adapter) UpdateStack(stackName string, certificateARNs map[string]time.
 		httpRedirectToHTTPS:               a.httpRedirectToHTTPS,
 		nlbCrossZone:                      a.nlbCrossZone,
 		http2:                             http2,
-		tags:                              a.stackTags,
+		tags:                              mergeTags(a.stackTags, stackTags),
 		internalDomains:                   a.internalDomains,
 		denyInternalDomains:               a.denyInternalDomains,
 		denyInternalDomainsResponse: denyResp{

aws/cf.go

@@ -1,20 +1,24 @@
 package aws
 
 import (
+	"encoding/base64"
 	"fmt"
 	"strings"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/cloudformation"
 	"github.com/aws/aws-sdk-go/service/cloudformation/cloudformationiface"
+
+	log "github.com/sirupsen/logrus"
 )
 
 const (
 	certificateARNTagLegacy = "ingress:certificate-arn"
 	certificateARNTagPrefix = "ingress:certificate-arn/"
 	ingressOwnerTag         = "ingress:owner"
 	cwAlarmConfigHashTag    = "cloudwatch:alarm-config-hash"
+	targetGroupsArnsTag     = "ingress:targetgroups"
 )
 
 // Stack is a simple wrapper around a CloudFormation Stack.
@@ -34,7 +38,7 @@ type Stack struct {
 	TargetGroupARNs   []string
 	WAFWebACLID       string
 	CertificateARNs   map[string]time.Time
-	tags              map[string]string
+	Tags              map[string]string
 }
 
 // IsComplete returns true if the stack status is a complete state.
@@ -480,18 +484,36 @@ func mapToManagedStack(stack *cloudformation.Stack) *Stack {
 		http2 = false
 	}
 
+	tgARNs := outputs.targetGroupARNs()
+
+	// If the stack is in rollback state, the outputs are not available.
+	// We need to store target group ARNs in the tags.
+	// To restore the ARNs, and keep sending traffic to the right target groups.
+	if aws.StringValue(stack.StackStatus) == cloudformation.StackStatusRollbackInProgress && len(tgARNs) == 0 {
+		if tgARNsTag, ok := tags[targetGroupsArnsTag]; ok {
+			values, err := base64.StdEncoding.DecodeString(tgARNsTag)
+			if err != nil {
+				log.Errorf("failed to decode target group ARNs from tags: %v", err)
+			} else {
+				tgARNs = strings.Split(string(values), ",")
+			}
+		}
+	} else if len(tgARNs) > 0 {
+		tags[targetGroupsArnsTag] = base64.StdEncoding.EncodeToString([]byte(strings.Join(tgARNs, ",")))
+	}
+
 	return &Stack{
 		Name:              aws.StringValue(stack.StackName),
 		DNSName:           outputs.dnsName(),
-		TargetGroupARNs:   outputs.targetGroupARNs(),
+		TargetGroupARNs:   tgARNs,
 		Scheme:            parameters[parameterLoadBalancerSchemeParameter],
 		SecurityGroup:     parameters[parameterLoadBalancerSecurityGroupParameter],
 		SSLPolicy:         parameters[parameterListenerSslPolicyParameter],
 		IpAddressType:     parameters[parameterIpAddressTypeParameter],
 		LoadBalancerType:  parameters[parameterLoadBalancerTypeParameter],
 		HTTP2:             http2,
 		CertificateARNs:   certificateARNs,
-		tags:              tags,
+		Tags:              tags,
 		OwnerIngress:      ownerIngress,
 		status:            aws.StringValue(stack.StackStatus),
 		statusReason:      aws.StringValue(stack.StackStatusReason),

aws/cf_test.go

@@ -478,10 +478,11 @@ func TestFindManagedStacks(t *testing.T) {
 						"cert-arn": {},
 					},
 					TargetGroupARNs: []string{"tg-arn"},
-					tags: map[string]string{
+					Tags: map[string]string{
 						kubernetesCreatorTag:                 DefaultControllerID,
 						clusterIDTagPrefix + "test-cluster":  resourceLifecycleOwned,
 						certificateARNTagPrefix + "cert-arn": time.Time{}.Format(time.RFC3339),
+						targetGroupsArnsTag:                  "dGctYXJu", // "tg-arn"
 					},
 					status: cloudformation.StackStatusUpdateInProgress,
 					HTTP2:  true,
@@ -493,10 +494,11 @@ func TestFindManagedStacks(t *testing.T) {
 						"cert-arn": {},
 					},
 					TargetGroupARNs: []string{"tg-arn"},
-					tags: map[string]string{
+					Tags: map[string]string{
 						kubernetesCreatorTag:                 DefaultControllerID,
 						clusterIDTagPrefix + "test-cluster":  resourceLifecycleOwned,
 						certificateARNTagPrefix + "cert-arn": time.Time{}.Format(time.RFC3339),
+						targetGroupsArnsTag:                  "dGctYXJu", // "tg-arn"
 					},
 					status: cloudformation.StackStatusCreateComplete,
 					HTTP2:  true,
@@ -508,18 +510,19 @@ func TestFindManagedStacks(t *testing.T) {
 						"cert-arn": {},
 					},
 					TargetGroupARNs: []string{"tg-arn", "http-tg-arn"},
-					tags: map[string]string{
+					Tags: map[string]string{
 						kubernetesCreatorTag:                 DefaultControllerID,
 						clusterIDTagPrefix + "test-cluster":  resourceLifecycleOwned,
 						certificateARNTagPrefix + "cert-arn": time.Time{}.Format(time.RFC3339),
+						targetGroupsArnsTag:                  "dGctYXJuLGh0dHAtdGctYXJu", // "tg-arn,http-tg-arn"
 					},
 					status: cloudformation.StackStatusCreateComplete,
 					HTTP2:  true,
 				},
 				{
 					Name:            "managed-stack-not-ready",
 					CertificateARNs: map[string]time.Time{},
-					tags: map[string]string{
+					Tags: map[string]string{
 						kubernetesCreatorTag:                DefaultControllerID,
 						clusterIDTagPrefix + "test-cluster": resourceLifecycleOwned,
 					},
@@ -542,6 +545,7 @@ func TestFindManagedStacks(t *testing.T) {
 								cfTag(kubernetesCreatorTag, DefaultControllerID),
 								cfTag(clusterIDTagPrefix+"test-cluster", resourceLifecycleOwned),
 								cfTag(certificateARNTagPrefix+"cert-arn", time.Time{}.Format(time.RFC3339)),
+								cfTag(targetGroupsArnsTag, "YXJuOmF3czpzbnM6dXMtZWFzdC0xOnRhcmdldGdyb3VwczpsYi10YXJnZXQtZ3JvdXBzMSxhcm46YXdzOnNuczp1cy1lYXN0LTE6dGFyZ2V0Z3JvdXBzOmxiLXRhcmdldC1ncm91cHMy"), // "arn:aws:sns:us-east-1:targetgroups:lb-target-groups1", "arn:aws:sns:us-east-1:targetgroups:lb-target-groups2"
 							},
 							Outputs: []*cloudformation.Output{},
 						},
@@ -554,7 +558,44 @@ func TestFindManagedStacks(t *testing.T) {
 					CertificateARNs: map[string]time.Time{
 						"cert-arn": {},
 					},
-					tags: map[string]string{
+					Tags: map[string]string{
+						kubernetesCreatorTag:                 DefaultControllerID,
+						clusterIDTagPrefix + "test-cluster":  resourceLifecycleOwned,
+						certificateARNTagPrefix + "cert-arn": time.Time{}.Format(time.RFC3339),
+						targetGroupsArnsTag:                  "YXJuOmF3czpzbnM6dXMtZWFzdC0xOnRhcmdldGdyb3VwczpsYi10YXJnZXQtZ3JvdXBzMSxhcm46YXdzOnNuczp1cy1lYXN0LTE6dGFyZ2V0Z3JvdXBzOmxiLXRhcmdldC1ncm91cHMy", // "arn:aws:sns:us-east-1:targetgroups:lb-target-groups1", "arn:aws:sns:us-east-1:targetgroups:lb-target-groups2"
+					},
+					TargetGroupARNs: []string{"arn:aws:sns:us-east-1:targetgroups:lb-target-groups1", "arn:aws:sns:us-east-1:targetgroups:lb-target-groups2"},
+					status:          cloudformation.StackStatusRollbackInProgress,
+					HTTP2:           true,
+				},
+			},
+		},
+		{
+			name: "successfull-call-with-rollback-status-and-no-tg-tag",
+			given: fake.CFOutputs{
+				DescribeStackPages: fake.R(nil, nil),
+				DescribeStacks: fake.R(&cloudformation.DescribeStacksOutput{
+					Stacks: []*cloudformation.Stack{
+						{
+							StackName:   aws.String("managed-stack-rolling-back"),
+							StackStatus: aws.String(cloudformation.StackStatusRollbackInProgress),
+							Tags: []*cloudformation.Tag{
+								cfTag(kubernetesCreatorTag, DefaultControllerID),
+								cfTag(clusterIDTagPrefix+"test-cluster", resourceLifecycleOwned),
+								cfTag(certificateARNTagPrefix+"cert-arn", time.Time{}.Format(time.RFC3339)),
+							},
+							Outputs: []*cloudformation.Output{},
+						},
+					},
+				}, nil),
+			},
+			want: []*Stack{
+				{
+					Name: "managed-stack-rolling-back",
+					CertificateARNs: map[string]time.Time{
+						"cert-arn": {},
+					},
+					Tags: map[string]string{
 						kubernetesCreatorTag:                 DefaultControllerID,
 						clusterIDTagPrefix + "test-cluster":  resourceLifecycleOwned,
 						certificateARNTagPrefix + "cert-arn": time.Time{}.Format(time.RFC3339),
@@ -603,9 +644,10 @@ func TestFindManagedStacks(t *testing.T) {
 					DNSName:         "example-notready.com",
 					TargetGroupARNs: []string{"tg-arn"},
 					CertificateARNs: map[string]time.Time{},
-					tags: map[string]string{
+					Tags: map[string]string{
 						kubernetesCreatorTag:                DefaultControllerID,
 						clusterIDTagPrefix + "test-cluster": resourceLifecycleOwned,
+						targetGroupsArnsTag:                 "dGctYXJu", // "tg-arn"
 					},
 					status: cloudformation.StackStatusReviewInProgress,
 					HTTP2:  true,
@@ -615,9 +657,10 @@ func TestFindManagedStacks(t *testing.T) {
 					DNSName:         "example.com",
 					TargetGroupARNs: []string{"tg-arn"},
 					CertificateARNs: map[string]time.Time{},
-					tags: map[string]string{
+					Tags: map[string]string{
 						kubernetesCreatorTag:                DefaultControllerID,
 						clusterIDTagPrefix + "test-cluster": resourceLifecycleOwned,
+						targetGroupsArnsTag:                 "dGctYXJu", // "tg-arn"
 					},
 					status: cloudformation.StackStatusRollbackComplete,
 					HTTP2:  true,
@@ -695,10 +738,11 @@ func TestGetStack(t *testing.T) {
 					"cert-arn": {},
 				},
 				TargetGroupARNs: []string{"tg-arn"},
-				tags: map[string]string{
+				Tags: map[string]string{
 					kubernetesCreatorTag:                 DefaultControllerID,
 					clusterIDTagPrefix + "test-cluster":  resourceLifecycleOwned,
 					certificateARNTagPrefix + "cert-arn": time.Time{}.Format(time.RFC3339),
+					targetGroupsArnsTag:                  "dGctYXJu", // "tg-arn"
 				},
 				status: cloudformation.StackStatusCreateComplete,
 				HTTP2:  true,
@@ -735,10 +779,11 @@ func TestGetStack(t *testing.T) {
 					"cert-arn": {},
 				},
 				TargetGroupARNs: []string{"tg-arn", "tg-http-arn"},
-				tags: map[string]string{
+				Tags: map[string]string{
 					kubernetesCreatorTag:                 DefaultControllerID,
 					clusterIDTagPrefix + "test-cluster":  resourceLifecycleOwned,
 					certificateARNTagPrefix + "cert-arn": time.Time{}.Format(time.RFC3339),
+					targetGroupsArnsTag:                  "dGctYXJuLHRnLWh0dHAtYXJu", // "tg-arn,tg-http-arn"
 				},
 				status: cloudformation.StackStatusCreateComplete,
 				HTTP2:  true,

aws/fake/cf.go

@@ -69,6 +69,7 @@ func (m *CFClient) DescribeStacks(in *cloudformation.DescribeStacksInput) (*clou
 }
 
 func (m *CFClient) CreateStack(params *cloudformation.CreateStackInput) (*cloudformation.CreateStackOutput, error) {
+	print("\n ======== CreateStack ======== \n")
 	m.tagCreationHistory = append(m.tagCreationHistory, params.Tags)
 	m.paramCreationHistory = append(m.paramCreationHistory, params.Parameters)
 	m.templateCreationHistory = append(m.templateCreationHistory, *params.TemplateBody)

worker.go

@@ -534,7 +534,12 @@ func createStack(awsAdapter *aws.Adapter, lb *loadBalancer, problems *problem.Li
 
 	log.Infof("Creating stack for certificates %q / ingress %q", certificates, lb.ingresses)
 
-	stackId, err := awsAdapter.CreateStack(certificates, lb.scheme, lb.securityGroup, lb.Owner(), lb.sslPolicy, lb.ipAddressType, lb.wafWebACLID, lb.cwAlarms, lb.loadBalancerType, lb.http2)
+	tags := make(map[string]string)
+	if lb.stack != nil && lb.stack.Tags != nil {
+		tags = lb.stack.Tags
+	}
+
+	stackId, err := awsAdapter.CreateStack(certificates, lb.scheme, lb.securityGroup, lb.Owner(), lb.sslPolicy, lb.ipAddressType, lb.wafWebACLID, lb.cwAlarms, lb.loadBalancerType, lb.http2, tags)
 	if err != nil {
 		if isAlreadyExistsError(err) {
 			lb.stack, err = awsAdapter.GetStack(stackId)
@@ -554,7 +559,12 @@ func updateStack(awsAdapter *aws.Adapter, lb *loadBalancer, problems *problem.Li
 
 	log.Infof("Updating %q stack for %d certificates / %d ingresses", lb.scheme, len(certificates), len(lb.ingresses))
 
-	stackId, err := awsAdapter.UpdateStack(lb.stack.Name, certificates, lb.scheme, lb.securityGroup, lb.Owner(), lb.sslPolicy, lb.ipAddressType, lb.wafWebACLID, lb.cwAlarms, lb.loadBalancerType, lb.http2)
+	tags := make(map[string]string)
+	if lb.stack != nil && lb.stack.Tags != nil {
+		tags = lb.stack.Tags
+	}
+
+	stackId, err := awsAdapter.UpdateStack(lb.stack.Name, certificates, lb.scheme, lb.securityGroup, lb.Owner(), lb.sslPolicy, lb.ipAddressType, lb.wafWebACLID, lb.cwAlarms, lb.loadBalancerType, lb.http2, tags)
 	if isNoUpdatesToBePerformedError(err) {
 		log.Debugf("Stack(%q) is already up to date", certificates)
 	} else if err != nil {

worker_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/x509"
 	"encoding/json"
+	"fmt"
 	"io"
 	"net/http/httptest"
 	"os"
@@ -527,9 +528,9 @@ func TestResourceConversionOneToOne(tt *testing.T) {
 				t.Error(problems.Errors())
 			}
 
-			assert.Equal(t, len(clientCF.GetTagCreationHistory()), len(tags))
+			assert.Equal(t, len(clientCF.GetTagCreationHistory()), len(tags), fmt.Sprintf("got %v, expected %v", tags, clientCF.GetTagCreationHistory()))
 			assert.Equal(t, len(clientCF.GetParamCreationHistory()), len(params))
-			assert.Equal(t, len(clientCF.GetTemplateCreationHistory()), len(templates))
+			assert.Equal(t, len(clientCF.GetTemplateCreationHistory()), len(templates), fmt.Sprintf("got %v, expected %v", templates, clientCF.GetTemplateCreationHistory()))
 
 			// This loop is necessary because assert.ElementsMatch only do set-style comparison
 			// for the first level of the array. So for nested arrays it would not behave like expected.