diff --git a/kubetest2/internal/deployers/eksapi/deployer.go b/kubetest2/internal/deployers/eksapi/deployer.go index 15ec10fa7..3e77a378b 100644 --- a/kubetest2/internal/deployers/eksapi/deployer.go +++ b/kubetest2/internal/deployers/eksapi/deployer.go @@ -72,6 +72,7 @@ type deployerOptions struct { Region string `flag:"region" desc:"AWS region for EKS cluster"` UnmanagedNodes bool `flag:"unmanaged-nodes" desc:"Use an AutoScalingGroup instead of an EKS-managed nodegroup."` UpClusterHeaders []string `flag:"up-cluster-header" desc:"Additional header to add to eks:CreateCluster requests. Specified in the same format as curl's -H flag."` + UserDataFormat string `flag:"user-data-format" desc:"Format of the node instance user data"` } // NewDeployer implements deployer.New for EKS using the EKS (and other AWS) API(s) directly (no cloudformation) @@ -220,6 +221,13 @@ func (d *deployer) verifyUpFlags() error { } klog.V(2).Infof("Using default instance types: %v", d.InstanceTypes) } + if d.UnmanagedNodes && d.AMI == "" { + return fmt.Errorf("--ami must be specified for --unmanaged-nodes") + } + if d.UnmanagedNodes && d.UserDataFormat == "" { + d.UserDataFormat = "bootstrap.sh" + klog.V(2).Infof("Using default user data format: %s", d.UserDataFormat) + } if d.NodeReadyTimeout == 0 { d.NodeReadyTimeout = time.Minute * 5 } diff --git a/kubetest2/internal/deployers/eksapi/k8s.go b/kubetest2/internal/deployers/eksapi/k8s.go index 8f014de6e..d9e6ab6bb 100644 --- a/kubetest2/internal/deployers/eksapi/k8s.go +++ b/kubetest2/internal/deployers/eksapi/k8s.go @@ -12,6 +12,7 @@ import ( "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" + "k8s.io/klog/v2" corev1 "k8s.io/api/core/v1" ) @@ -25,6 +26,7 @@ func newKubernetesClient(kubeconfigPath string) (*kubernetes.Clientset, error) { } func waitForReadyNodes(client *kubernetes.Clientset, nodeCount int, timeout time.Duration) error { + klog.Infof("waiting up to %v for %d node(s) to be ready...", timeout, nodeCount) readyNodes := sets.NewString() watcher, err := client.CoreV1().Nodes().Watch(context.TODO(), metav1.ListOptions{}) if err != nil { diff --git a/kubetest2/internal/deployers/eksapi/nodegroup.go b/kubetest2/internal/deployers/eksapi/nodegroup.go index 177ecd20c..90ec0aabf 100644 --- a/kubetest2/internal/deployers/eksapi/nodegroup.go +++ b/kubetest2/internal/deployers/eksapi/nodegroup.go @@ -91,7 +91,7 @@ func (m *NodegroupManager) createManagedNodegroup(infra *Infrastructure, cluster if ok, err := m.verifyASGAMI(*asgName, opts.ExpectedAMI); err != nil { return err } else if !ok { - return fmt.Errorf("ASG %s is not using expected AMI: %s", &asgName, opts.ExpectedAMI) + return fmt.Errorf("ASG %s is not using expected AMI: %s", *asgName, opts.ExpectedAMI) } } return nil @@ -100,8 +100,12 @@ func (m *NodegroupManager) createManagedNodegroup(infra *Infrastructure, cluster func (m *NodegroupManager) createUnmanagedNodegroup(infra *Infrastructure, cluster *Cluster, opts *deployerOptions) error { stackName := m.getUnmanagedNodegroupStackName() klog.Infof("creating unmanaged nodegroup stack...") + userData, err := generateUserData(opts.UserDataFormat, cluster) + if err != nil { + return err + } templateBuf := bytes.Buffer{} - err := templates.UnmanagedNodegroup.Execute(&templateBuf, struct { + err = templates.UnmanagedNodegroup.Execute(&templateBuf, struct { InstanceTypes []string KubernetesVersion string }{ @@ -132,12 +136,8 @@ func (m *NodegroupManager) createUnmanagedNodegroup(infra *Infrastructure, clust ParameterValue: aws.String(strings.Join(infra.subnets(), ",")), }, { - ParameterKey: aws.String("ClusterCA"), - ParameterValue: aws.String(cluster.certificateAuthorityData), - }, - { - ParameterKey: aws.String("ClusterEndpoint"), - ParameterValue: aws.String(cluster.endpoint), + ParameterKey: aws.String("UserData"), + ParameterValue: aws.String(userData), }, { ParameterKey: aws.String("ClusterName"), @@ -163,14 +163,12 @@ func (m *NodegroupManager) createUnmanagedNodegroup(infra *Infrastructure, clust ParameterKey: aws.String("SSHKeyPair"), ParameterValue: aws.String(infra.sshKeyPair), }, + { + ParameterKey: aws.String("AMIId"), + ParameterValue: aws.String(opts.AMI), + }, }, } - if opts.AMI != "" { - input.Parameters = append(input.Parameters, cloudformationtypes.Parameter{ - ParameterKey: aws.String("AMIId"), - ParameterValue: aws.String(opts.AMI), - }) - } out, err := m.clients.CFN().CreateStack(context.TODO(), &input) if err != nil { return err diff --git a/kubetest2/internal/deployers/eksapi/templates/templates.go b/kubetest2/internal/deployers/eksapi/templates/templates.go index e87dd8e36..9cb173e6c 100644 --- a/kubetest2/internal/deployers/eksapi/templates/templates.go +++ b/kubetest2/internal/deployers/eksapi/templates/templates.go @@ -8,15 +8,29 @@ import ( //go:embed infra.yaml var Infrastructure string -//go:embed unmanaged-nodegroup.yaml.template -var unmanagedNodegroupTemplate string +var ( + //go:embed unmanaged-nodegroup.yaml.template + unmanagedNodegroupTemplate string + UnmanagedNodegroup = template.Must(template.New("unmanagedNodegroup").Parse(unmanagedNodegroupTemplate)) +) + +type UnmanagedNodegroupTemplateData struct { + KubernetesVersion string + InstanceTypes []string +} -var UnmanagedNodegroup *template.Template +var ( + //go:embed userdata_bootstrap.sh.mimepart.template + userDataBootstrapShTemplate string + UserDataBootstrapSh = template.Must(template.New("userDataBootstrapSh").Parse(userDataBootstrapShTemplate)) + + //go:embed userdata_nodeadm.yaml.mimepart.template + userDataNodeadmTemplate string + UserDataNodeadm = template.Must(template.New("userDataNodeadm").Parse(userDataNodeadmTemplate)) +) -func init() { - t, err := template.New("unmanaged-nodegroup").Parse(unmanagedNodegroupTemplate) - if err != nil { - panic(err) - } - UnmanagedNodegroup = t +type UserDataTemplateData struct { + Name string + CertificateAuthority string + APIServerEndpoint string } diff --git a/kubetest2/internal/deployers/eksapi/templates/templates_test.go b/kubetest2/internal/deployers/eksapi/templates/templates_test.go index 3cae6d0f2..f10565025 100644 --- a/kubetest2/internal/deployers/eksapi/templates/templates_test.go +++ b/kubetest2/internal/deployers/eksapi/templates/templates_test.go @@ -7,10 +7,7 @@ import ( func Test_UnmanagedNodegroup(t *testing.T) { buf := bytes.Buffer{} - err := UnmanagedNodegroup.Execute(&buf, struct { - KubernetesVersion string - InstanceTypes []string - }{ + err := UnmanagedNodegroup.Execute(&buf, UnmanagedNodegroupTemplateData{ KubernetesVersion: "1.28", InstanceTypes: []string{ "t2.medium", diff --git a/kubetest2/internal/deployers/eksapi/templates/unmanaged-nodegroup.yaml.template b/kubetest2/internal/deployers/eksapi/templates/unmanaged-nodegroup.yaml.template index 938be9e1a..cfecbfe28 100644 --- a/kubetest2/internal/deployers/eksapi/templates/unmanaged-nodegroup.yaml.template +++ b/kubetest2/internal/deployers/eksapi/templates/unmanaged-nodegroup.yaml.template @@ -29,33 +29,12 @@ Parameters: NodeCount: Type: Number - ClusterCA: - Type: String - - ClusterEndpoint: - Type: String - ClusterName: Type: String - ExtraBootstrapArguments: - Description: Arguments to pass to the bootstrap script. See files/bootstrap.sh in https://github.com/awslabs/amazon-eks-ami - Type: String - Default: "" - - BootstrapArgumentsForSpotFleet: - Description: Arguments to pass to the bootstrap script. See files/bootstrap.sh in https://github.com/awslabs/amazon-eks-ami - Default: "--kubelet-extra-args '--node-labels=lifecycle=Ec2Spot,$ExtraNodeLabels --register-with-taints=spotInstance=true:PreferNoSchedule'" - Type: String - NodeRoleName: Description: The IAM role name of worker nodes. Type: String - - PauseTime: - Description: Pause Time - Type: String - Default: PT5M SSHKeyPair: Type: String @@ -63,6 +42,9 @@ Parameters: SSHSecurityGroup: Type: String + UserData: + Type: String + Resources: NodeInstanceProfile: Type: AWS::IAM::InstanceProfile @@ -76,19 +58,19 @@ Resources: UpdatePolicy: AutoScalingRollingUpdate: WaitOnResourceSignals: true - PauseTime: !Ref PauseTime + PauseTime: PT15M Properties: AutoScalingGroupName: !Ref ResourceId DesiredCapacity: !Ref NodeCount MinSize: !Ref NodeCount - MaxSize: !Ref NodeCount + MaxSize: !Ref NodeCount MixedInstancesPolicy: InstancesDistribution: OnDemandAllocationStrategy: prioritized OnDemandBaseCapacity: !Ref NodeCount OnDemandPercentageAboveBaseCapacity: 0 LaunchTemplate: - LaunchTemplateSpecification: + LaunchTemplateSpecification: LaunchTemplateId: !Ref NodeLaunchTemplate # LaunchTemplateName: String Version: !GetAtt NodeLaunchTemplate.LatestVersionNumber @@ -117,23 +99,32 @@ Resources: Type: AWS::EC2::LaunchTemplate Properties: LaunchTemplateName: !Ref ResourceId - LaunchTemplateData: + LaunchTemplateData: SecurityGroupIds: - !Ref SecurityGroup - !Ref SSHSecurityGroup KeyName: !Ref SSHKeyPair UserData: Fn::Base64: - !Sub | - #!/bin/bash - /etc/eks/bootstrap.sh ${ClusterName} ${ExtraBootstrapArguments} \ - --b64-cluster-ca ${ClusterCA} \ - --apiserver-endpoint ${ClusterEndpoint} - /opt/aws/bin/cfn-signal --exit-code $? \ - --stack ${AWS::StackName} \ - --resource NodeGroup \ - --region ${AWS::Region} - IamInstanceProfile: + Fn::Sub: | + Content-Type: multipart/mixed; boundary="BOUNDARY" + MIME-Version: 1.0 + + --BOUNDARY + ${UserData} + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + MIME-Version: 1.0 + + #!/usr/bin/env bash + /opt/aws/bin/cfn-signal \ + --stack ${AWS::StackName} \ + --resource NodeGroup \ + --region ${AWS::Region} + + --BOUNDARY-- + IamInstanceProfile: Arn: !GetAtt NodeInstanceProfile.Arn ImageId: !Ref AMIId InstanceType: "{{index .InstanceTypes 0}}" diff --git a/kubetest2/internal/deployers/eksapi/templates/userdata_bootstrap.sh.mimepart.template b/kubetest2/internal/deployers/eksapi/templates/userdata_bootstrap.sh.mimepart.template new file mode 100644 index 000000000..d00777345 --- /dev/null +++ b/kubetest2/internal/deployers/eksapi/templates/userdata_bootstrap.sh.mimepart.template @@ -0,0 +1,7 @@ +Content-Type: text/x-shellscript; charset="us-ascii" +MIME-Version: 1.0 + +#!/usr/bin/env bash +/etc/eks/bootstrap.sh {{.Name}} \ + --b64-cluster-ca {{.CertificateAuthority}} \ + --apiserver-endpoint {{.APIServerEndpoint}} diff --git a/kubetest2/internal/deployers/eksapi/templates/userdata_nodeadm.yaml.mimepart.template b/kubetest2/internal/deployers/eksapi/templates/userdata_nodeadm.yaml.mimepart.template new file mode 100644 index 000000000..695636380 --- /dev/null +++ b/kubetest2/internal/deployers/eksapi/templates/userdata_nodeadm.yaml.mimepart.template @@ -0,0 +1,11 @@ +Content-Type: application/node.eks.aws +MIME-Version: 1.0 + +--- +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + name: {{.Name}} + apiServerEndpoint: {{.APIServerEndpoint}} + certificateAuthority: {{.CertificateAuthority}} diff --git a/kubetest2/internal/deployers/eksapi/userdata.go b/kubetest2/internal/deployers/eksapi/userdata.go new file mode 100644 index 000000000..c2fa4f2d4 --- /dev/null +++ b/kubetest2/internal/deployers/eksapi/userdata.go @@ -0,0 +1,30 @@ +package eksapi + +import ( + "bytes" + "fmt" + "text/template" + + "github.com/aws/aws-k8s-tester/kubetest2/internal/deployers/eksapi/templates" +) + +func generateUserData(format string, cluster *Cluster) (string, error) { + var t *template.Template + switch format { + case "bootstrap.sh": + t = templates.UserDataBootstrapSh + case "nodeadm": + t = templates.UserDataNodeadm + default: + return "", fmt.Errorf("uknown user data format: '%s'", format) + } + buf := bytes.Buffer{} + if err := t.Execute(&buf, templates.UserDataTemplateData{ + Name: cluster.name, + CertificateAuthority: cluster.certificateAuthorityData, + APIServerEndpoint: cluster.endpoint, + }); err != nil { + return "", err + } + return buf.String(), nil +} diff --git a/kubetest2/internal/deployers/eksapi/userdata_test.go b/kubetest2/internal/deployers/eksapi/userdata_test.go new file mode 100644 index 000000000..2f48f8f65 --- /dev/null +++ b/kubetest2/internal/deployers/eksapi/userdata_test.go @@ -0,0 +1,61 @@ +package eksapi + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +var cluster = Cluster{ + name: "cluster", + endpoint: "https://example.com", + certificateAuthorityData: "certificateAuthority", +} + +const bootstrapShUserData = `Content-Type: text/x-shellscript; charset="us-ascii" +MIME-Version: 1.0 + +#!/usr/bin/env bash +/etc/eks/bootstrap.sh cluster \ + --b64-cluster-ca certificateAuthority \ + --apiserver-endpoint https://example.com +` + +const nodeadmUserData = `Content-Type: application/node.eks.aws +MIME-Version: 1.0 + +--- +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + name: cluster + apiServerEndpoint: https://example.com + certificateAuthority: certificateAuthority +` + +func Test_generateUserData(t *testing.T) { + cases := []struct { + format string + expected string + }{ + { + format: "bootstrap.sh", + expected: bootstrapShUserData, + }, + { + format: "nodeadm", + expected: nodeadmUserData, + }, + } + for _, c := range cases { + t.Run(c.format, func(t *testing.T) { + actual, err := generateUserData(c.format, &cluster) + if err != nil { + t.Log(err) + t.Error(err) + } + assert.Equal(t, c.expected, actual) + }) + } +}