Add --user-data-format option, nodeadm support

kubetest2/internal/deployers/eksapi/deployer.go

@@ -72,6 +72,7 @@ type deployerOptions struct {
 	Region                      string        `flag:"region" desc:"AWS region for EKS cluster"`
 	UnmanagedNodes              bool          `flag:"unmanaged-nodes" desc:"Use an AutoScalingGroup instead of an EKS-managed nodegroup."`
 	UpClusterHeaders            []string      `flag:"up-cluster-header" desc:"Additional header to add to eks:CreateCluster requests. Specified in the same format as curl's -H flag."`
+	UserDataFormat              string        `flag:"user-data-format" desc:"Format of the node instance user data"`
 }
 
 // NewDeployer implements deployer.New for EKS using the EKS (and other AWS) API(s) directly (no cloudformation)
@@ -220,6 +221,13 @@ func (d *deployer) verifyUpFlags() error {
 		}
 		klog.V(2).Infof("Using default instance types: %v", d.InstanceTypes)
 	}
+	if d.UnmanagedNodes && d.AMI == "" {
+		return fmt.Errorf("--ami must be specified for --unmanaged-nodes")
+	}
+	if d.UnmanagedNodes && d.UserDataFormat == "" {
+		d.UserDataFormat = "bootstrap.sh"
+		klog.V(2).Infof("Using default user data format: %s", d.UserDataFormat)
+	}
 	if d.NodeReadyTimeout == 0 {
 		d.NodeReadyTimeout = time.Minute * 5
 	}

kubetest2/internal/deployers/eksapi/k8s.go

@@ -12,6 +12,7 @@ import (
 	"k8s.io/apimachinery/pkg/watch"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/klog/v2"
 
 	corev1 "k8s.io/api/core/v1"
 )
@@ -25,6 +26,7 @@ func newKubernetesClient(kubeconfigPath string) (*kubernetes.Clientset, error) {
 }
 
 func waitForReadyNodes(client *kubernetes.Clientset, nodeCount int, timeout time.Duration) error {
+	klog.Infof("waiting up to %v for %d node(s) to be ready...", timeout, nodeCount)
 	readyNodes := sets.NewString()
 	watcher, err := client.CoreV1().Nodes().Watch(context.TODO(), metav1.ListOptions{})
 	if err != nil {

kubetest2/internal/deployers/eksapi/nodegroup.go

@@ -91,7 +91,7 @@ func (m *NodegroupManager) createManagedNodegroup(infra *Infrastructure, cluster
 		if ok, err := m.verifyASGAMI(*asgName, opts.ExpectedAMI); err != nil {
 			return err
 		} else if !ok {
-			return fmt.Errorf("ASG %s is not using expected AMI: %s", &asgName, opts.ExpectedAMI)
+			return fmt.Errorf("ASG %s is not using expected AMI: %s", *asgName, opts.ExpectedAMI)
 		}
 	}
 	return nil
@@ -100,8 +100,12 @@ func (m *NodegroupManager) createManagedNodegroup(infra *Infrastructure, cluster
 func (m *NodegroupManager) createUnmanagedNodegroup(infra *Infrastructure, cluster *Cluster, opts *deployerOptions) error {
 	stackName := m.getUnmanagedNodegroupStackName()
 	klog.Infof("creating unmanaged nodegroup stack...")
+	userData, err := generateUserData(opts.UserDataFormat, cluster)
+	if err != nil {
+		return err
+	}
 	templateBuf := bytes.Buffer{}
-	err := templates.UnmanagedNodegroup.Execute(&templateBuf, struct {
+	err = templates.UnmanagedNodegroup.Execute(&templateBuf, struct {
 		InstanceTypes     []string
 		KubernetesVersion string
 	}{
@@ -132,12 +136,8 @@ func (m *NodegroupManager) createUnmanagedNodegroup(infra *Infrastructure, clust
 				ParameterValue: aws.String(strings.Join(infra.subnets(), ",")),
 			},
 			{
-				ParameterKey:   aws.String("ClusterCA"),
-				ParameterValue: aws.String(cluster.certificateAuthorityData),
-			},
-			{
-				ParameterKey:   aws.String("ClusterEndpoint"),
-				ParameterValue: aws.String(cluster.endpoint),
+				ParameterKey:   aws.String("UserData"),
+				ParameterValue: aws.String(userData),
 			},
 			{
 				ParameterKey:   aws.String("ClusterName"),
@@ -163,14 +163,12 @@ func (m *NodegroupManager) createUnmanagedNodegroup(infra *Infrastructure, clust
 				ParameterKey:   aws.String("SSHKeyPair"),
 				ParameterValue: aws.String(infra.sshKeyPair),
 			},
+			{
+				ParameterKey:   aws.String("AMIId"),
+				ParameterValue: aws.String(opts.AMI),
+			},
 		},
 	}
-	if opts.AMI != "" {
-		input.Parameters = append(input.Parameters, cloudformationtypes.Parameter{
-			ParameterKey:   aws.String("AMIId"),
-			ParameterValue: aws.String(opts.AMI),
-		})
-	}
 	out, err := m.clients.CFN().CreateStack(context.TODO(), &input)
 	if err != nil {
 		return err

kubetest2/internal/deployers/eksapi/templates/templates.go

@@ -8,15 +8,29 @@ import (
 //go:embed infra.yaml
 var Infrastructure string
 
-//go:embed unmanaged-nodegroup.yaml.template
-var unmanagedNodegroupTemplate string
+var (
+	//go:embed unmanaged-nodegroup.yaml.template
+	unmanagedNodegroupTemplate string
+	UnmanagedNodegroup         = template.Must(template.New("unmanagedNodegroup").Parse(unmanagedNodegroupTemplate))
+)
+
+type UnmanagedNodegroupTemplateData struct {
+	KubernetesVersion string
+	InstanceTypes     []string
+}
 
-var UnmanagedNodegroup *template.Template
+var (
+	//go:embed userdata_bootstrap.sh.mimepart.template
+	userDataBootstrapShTemplate string
+	UserDataBootstrapSh         = template.Must(template.New("userDataBootstrapSh").Parse(userDataBootstrapShTemplate))
+
+	//go:embed userdata_nodeadm.yaml.mimepart.template
+	userDataNodeadmTemplate string
+	UserDataNodeadm         = template.Must(template.New("userDataNodeadm").Parse(userDataNodeadmTemplate))
+)
 
-func init() {
-	t, err := template.New("unmanaged-nodegroup").Parse(unmanagedNodegroupTemplate)
-	if err != nil {
-		panic(err)
-	}
-	UnmanagedNodegroup = t
+type UserDataTemplateData struct {
+	Name                 string
+	CertificateAuthority string
+	APIServerEndpoint    string
 }

kubetest2/internal/deployers/eksapi/templates/templates_test.go

@@ -7,10 +7,7 @@ import (
 
 func Test_UnmanagedNodegroup(t *testing.T) {
 	buf := bytes.Buffer{}
-	err := UnmanagedNodegroup.Execute(&buf, struct {
-		KubernetesVersion string
-		InstanceTypes     []string
-	}{
+	err := UnmanagedNodegroup.Execute(&buf, UnmanagedNodegroupTemplateData{
 		KubernetesVersion: "1.28",
 		InstanceTypes: []string{
 			"t2.medium",

kubetest2/internal/deployers/eksapi/templates/unmanaged-nodegroup.yaml.template

@@ -29,40 +29,22 @@ Parameters:
   NodeCount:
     Type: Number
 
-  ClusterCA:
-    Type: String
-
-  ClusterEndpoint:
-    Type: String
-
   ClusterName:
     Type: String
 
-  ExtraBootstrapArguments:
-    Description: Arguments to pass to the bootstrap script. See files/bootstrap.sh in https://github.com/awslabs/amazon-eks-ami
-    Type: String
-    Default: ""
-
-  BootstrapArgumentsForSpotFleet:
-    Description: Arguments to pass to the bootstrap script. See files/bootstrap.sh in https://github.com/awslabs/amazon-eks-ami
-    Default: "--kubelet-extra-args '--node-labels=lifecycle=Ec2Spot,$ExtraNodeLabels --register-with-taints=spotInstance=true:PreferNoSchedule'"
-    Type: String
-
   NodeRoleName:
     Description: The IAM role name  of worker nodes.
     Type: String
-
-  PauseTime:
-    Description: Pause Time
-    Type: String
-    Default: PT5M
 
   SSHKeyPair:
     Type: String
 
   SSHSecurityGroup:
     Type: String
 
+  UserData:
+    Type: String
+
 Resources:
   NodeInstanceProfile:
     Type: AWS::IAM::InstanceProfile
@@ -76,19 +58,19 @@ Resources:
     UpdatePolicy:
       AutoScalingRollingUpdate:
         WaitOnResourceSignals: true
-        PauseTime: !Ref PauseTime
+        PauseTime: PT15M
     Properties:
       AutoScalingGroupName: !Ref ResourceId
       DesiredCapacity: !Ref NodeCount
       MinSize: !Ref NodeCount
-      MaxSize: !Ref NodeCount      
+      MaxSize: !Ref NodeCount
       MixedInstancesPolicy:
         InstancesDistribution:
           OnDemandAllocationStrategy: prioritized
           OnDemandBaseCapacity: !Ref NodeCount
           OnDemandPercentageAboveBaseCapacity: 0
         LaunchTemplate:
-          LaunchTemplateSpecification: 
+          LaunchTemplateSpecification:
             LaunchTemplateId: !Ref NodeLaunchTemplate
             # LaunchTemplateName: String
             Version: !GetAtt NodeLaunchTemplate.LatestVersionNumber
@@ -117,23 +99,32 @@ Resources:
     Type: AWS::EC2::LaunchTemplate
     Properties:
       LaunchTemplateName: !Ref ResourceId
-      LaunchTemplateData: 
+      LaunchTemplateData:
         SecurityGroupIds:
           - !Ref SecurityGroup
           - !Ref SSHSecurityGroup
         KeyName: !Ref SSHKeyPair
         UserData:
           Fn::Base64:
-            !Sub |
-            #!/bin/bash
-            /etc/eks/bootstrap.sh ${ClusterName} ${ExtraBootstrapArguments} \
-              --b64-cluster-ca ${ClusterCA} \
-              --apiserver-endpoint ${ClusterEndpoint}
-            /opt/aws/bin/cfn-signal --exit-code $? \
-                     --stack  ${AWS::StackName} \
-                     --resource NodeGroup  \
-                     --region ${AWS::Region}
-        IamInstanceProfile: 
+            Fn::Sub: |
+              Content-Type: multipart/mixed; boundary="BOUNDARY"
+              MIME-Version: 1.0
+
+              --BOUNDARY
+              ${UserData}
+
+              --BOUNDARY
+              Content-Type: text/x-shellscript; charset="us-ascii"
+              MIME-Version: 1.0
+
+              #!/usr/bin/env bash
+              /opt/aws/bin/cfn-signal \
+                --stack  ${AWS::StackName} \
+                --resource NodeGroup \
+                --region ${AWS::Region}
+
+              --BOUNDARY--
+        IamInstanceProfile:
           Arn: !GetAtt NodeInstanceProfile.Arn
         ImageId: !Ref AMIId
         InstanceType: "{{index .InstanceTypes 0}}"

kubetest2/internal/deployers/eksapi/templates/userdata_bootstrap.sh.mimepart.template

@@ -0,0 +1,7 @@
+Content-Type: text/x-shellscript; charset="us-ascii"
+MIME-Version: 1.0
+
+#!/usr/bin/env bash
+/etc/eks/bootstrap.sh {{.Name}} \
+  --b64-cluster-ca {{.CertificateAuthority}} \
+  --apiserver-endpoint {{.APIServerEndpoint}}

kubetest2/internal/deployers/eksapi/templates/userdata_nodeadm.yaml.mimepart.template

@@ -0,0 +1,11 @@
+Content-Type: application/node.eks.aws
+MIME-Version: 1.0
+
+---
+apiVersion: node.eks.aws/v1alpha1
+kind: NodeConfig
+spec:
+  cluster:
+    name: {{.Name}}
+    apiServerEndpoint: {{.APIServerEndpoint}}
+    certificateAuthority: {{.CertificateAuthority}}

kubetest2/internal/deployers/eksapi/userdata.go

@@ -0,0 +1,30 @@
+package eksapi
+
+import (
+	"bytes"
+	"fmt"
+	"text/template"
+
+	"github.com/aws/aws-k8s-tester/kubetest2/internal/deployers/eksapi/templates"
+)
+
+func generateUserData(format string, cluster *Cluster) (string, error) {
+	var t *template.Template
+	switch format {
+	case "bootstrap.sh":
+		t = templates.UserDataBootstrapSh
+	case "nodeadm":
+		t = templates.UserDataNodeadm
+	default:
+		return "", fmt.Errorf("uknown user data format: '%s'", format)
+	}
+	buf := bytes.Buffer{}
+	if err := t.Execute(&buf, templates.UserDataTemplateData{
+		Name:                 cluster.name,
+		CertificateAuthority: cluster.certificateAuthorityData,
+		APIServerEndpoint:    cluster.endpoint,
+	}); err != nil {
+		return "", err
+	}
+	return buf.String(), nil
+}

kubetest2/internal/deployers/eksapi/userdata_test.go

@@ -0,0 +1,61 @@
+package eksapi
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+var cluster = Cluster{
+	name:                     "cluster",
+	endpoint:                 "https://example.com",
+	certificateAuthorityData: "certificateAuthority",
+}
+
+const bootstrapShUserData = `Content-Type: text/x-shellscript; charset="us-ascii"
+MIME-Version: 1.0
+
+#!/usr/bin/env bash
+/etc/eks/bootstrap.sh cluster \
+  --b64-cluster-ca certificateAuthority \
+  --apiserver-endpoint https://example.com
+`
+
+const nodeadmUserData = `Content-Type: application/node.eks.aws
+MIME-Version: 1.0
+
+---
+apiVersion: node.eks.aws/v1alpha1
+kind: NodeConfig
+spec:
+  cluster:
+    name: cluster
+    apiServerEndpoint: https://example.com
+    certificateAuthority: certificateAuthority
+`
+
+func Test_generateUserData(t *testing.T) {
+	cases := []struct {
+		format   string
+		expected string
+	}{
+		{
+			format:   "bootstrap.sh",
+			expected: bootstrapShUserData,
+		},
+		{
+			format:   "nodeadm",
+			expected: nodeadmUserData,
+		},
+	}
+	for _, c := range cases {
+		t.Run(c.format, func(t *testing.T) {
+			actual, err := generateUserData(c.format, &cluster)
+			if err != nil {
+				t.Log(err)
+				t.Error(err)
+			}
+			assert.Equal(t, c.expected, actual)
+		})
+	}
+}