[HOPSWORKS-1403] Added availability zone and Security Group to EC2 Co…

…nfig.
karamelchef · Nov 5, 2019 · 943a316 · 943a316
1 parent d0e3f1c
commit 943a316
Show file tree

Hide file tree

Showing 5 changed files with 57 additions and 6 deletions.
diff --git a/karamel-common/src/main/java/se/kth/karamel/common/clusterdef/Ec2.java b/karamel-common/src/main/java/se/kth/karamel/common/clusterdef/Ec2.java
@@ -22,6 +22,8 @@ public class Ec2 extends Provider {
   private String vpc;
   private String subnet;
   private String iamarn;
+  private String zone;
+  private String securityGroups;
 
   public String getType() {
     return type;
@@ -84,6 +86,22 @@ public static Ec2 makeDefault() {
     return ec2.applyDefaults();
   }
 
+  public String getZone() {
+    return zone;
+  }
+
+  public void setZone(String zone) {
+    this.zone = zone;
+  }
+
+  public String getSecurityGroups() {
+    return securityGroups;
+  }
+
+  public void setSecurityGroups(String securityGroups) {
+    this.securityGroups = securityGroups;
+  }
+
   @Override
   public Ec2 applyDefaults() {
     Ec2 clone = cloneMe();
@@ -115,6 +133,8 @@ public Ec2 cloneMe() {
     ec2.setSubnet(subnet);
     ec2.setVpc(vpc);
     ec2.setIamarn(iamarn);
+    ec2.setZone(zone);
+    ec2.setSecurityGroups(securityGroups);
     return ec2;
   }
 
@@ -147,6 +167,12 @@ public Ec2 applyParentScope(Provider parentScopeProvider) {
       if (clone.getIamarn() == null) {
         clone.setIamarn(parentEc2.getIamarn());
       }
+      if (clone.getZone() == null) {
+        clone.setZone(parentEc2.getZone());
+      }
+      if (clone.getSecurityGroups() == null) {
+        clone.setSecurityGroups(parentEc2.getSecurityGroups());
+      }
     }
     return clone;
   }

diff --git a/karamel-common/src/main/java/se/kth/karamel/common/launcher/amazon/InstanceType.java b/karamel-common/src/main/java/se/kth/karamel/common/launcher/amazon/InstanceType.java
@@ -252,9 +252,9 @@ public static InstanceType valueByModel(String model) {
   }
 
   public StorageDevice[] getStorageDevices() {
-    if (diskType == DiskType.EBS) {
-      return new StorageDevice[0];
-    }
+//    if (diskType == DiskType.EBS) {
+//      return new StorageDevice[0];
+//    }
 
     StorageDevice[] devices = new StorageDevice[numDisks];
     for (int i = 0; i < numDisks; i++) {
@@ -266,7 +266,7 @@ public StorageDevice[] getStorageDevices() {
         devices[i] = new StorageDevice(Settings.AWS_SSD_MAPPINGNAME_PREFIX+ (char)('b'+i),
                 "/mnt" +
                 "/ssd"+i, Settings.AWS_STORAGE_VIRTUALNAME_PREFIX +i);
-      } else if (diskType == DiskType.NVMe_SSD){
+      } else if (diskType == DiskType.NVMe_SSD || diskType == DiskType.EBS){
         devices[i] = new StorageDevice(Settings.AWS_NVME_MAPPINGNAME_PREFIX+ i+"n1", "/mnt" +
                 "/nvme_ssd"+i, Settings.AWS_STORAGE_VIRTUALNAME_PREFIX +i);
       } else {

diff --git a/karamel-core/src/main/java/se/kth/karamel/backend/launcher/amazon/Ec2Launcher.java b/karamel-core/src/main/java/se/kth/karamel/backend/launcher/amazon/Ec2Launcher.java
@@ -15,6 +15,7 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Arrays;
 import java.util.Set;
 import org.apache.log4j.Logger;
 import org.jclouds.aws.AWSResponseException;
@@ -250,6 +251,13 @@ public List<MachineRuntime> forkMachines(String keyPairName, GroupRuntime mainGr
       options.iamInstanceProfileArn(ec2.getIamarn());
     }
 
+    //set security groups if any
+    if (ec2.getSecurityGroups() != null) {
+      String[] sgs = ec2.getSecurityGroups().split(",");
+      options.securityGroups(Arrays.asList(sgs));
+      logger.info("Security Groups: " + Arrays.asList(sgs));
+    }
+
     Confs confs = Confs.loadKaramelConfs();
     String prepStorages = confs.getProperty(Settings.PREPARE_STORAGES_KEY);
     logger.info("Prepare storage is "+prepStorages);
@@ -289,14 +297,19 @@ public List<MachineRuntime> forkMachines(String keyPairName, GroupRuntime mainGr
       template.os64Bit(true);
       template.hardwareId(ec2.getType());
       template.imageId(ec2.getRegion() + "/" + ec2.getAmi());
-      template.locationId(ec2.getRegion());
+      String locationID = ec2.getRegion();
+      if (ec2.getZone() != null) {
+        locationID += ec2.getZone();
+      }
+      template.locationId(locationID);
       tries++;
       Set<NodeMetadata> succ = new HashSet<>();
       try {
         logger.info(String.format("Forking %d "+ec2.getType()+" machine(s) "
                         +"for '%s', so far " + "(succeeded:%d, failed:%d, total:%d)",
             requestSize, uniqueGroupName, successfulNodes.size(), failedNodes.size(), numberToLaunch));
-        succ.addAll(context.getComputeService().createNodesInGroup(uniqueGroupName, requestSize, template.build()));
+        succ.addAll(context.getComputeService().createNodesInGroup(uniqueGroupName, requestSize,
+                template.build()));
         long finishTime = System.currentTimeMillis();
         numSuccess += succ.size();
       } catch (RunNodesException ex) {

diff --git a/karamel-core/src/test/resources/se/kth/karamel/client/model/test-definitions/reference.yml b/karamel-core/src/test/resources/se/kth/karamel/client/model/test-definitions/reference.yml
@@ -6,6 +6,8 @@ ec2:
     username: ubuntu 
     price: 0.1
     iamarn: arn:aws:iam::822623301872:instance-profile/hopsfs-s3-access
+    zone: a
+    securityGroups: hops-security-group
 
 cookbooks:                                                                      #Github repos for cookbooks, if one cookbook is not defined here it will be looked up in default git repo (https://www.github.com/hopstart/<cookbook_name>-chef)
   kagent:                                                                     #'name' is an alias used in the yaml

diff --git a/karamel-ui/src/main/resources/assets/karamel/board/datamodel.js b/karamel-ui/src/main/resources/assets/karamel/board/datamodel.js
@@ -350,6 +350,8 @@ function Ec2() {
   this.accessKey = null;
   this.secretKey = null;
   this.iamarn = null;
+  this.zone = null;
+  this.securityGroups = null;  
 
   this.load = function(other) {
     this.type = other.type || null;
@@ -360,6 +362,8 @@ function Ec2() {
     this.vpc = other.vpc || null;
     this.subnet = other.subnet || null;
     this.iamarn = other.iamarn || null;
+    this.zone = other.zone || null;
+    this.securityGroups = other.securityGroups || null;
   };
 
   this.copy = function(other) {
@@ -373,6 +377,8 @@ function Ec2() {
     this.accessKey = other.accessKey || null;
     this.secretKey = other.secretKey || null;
     this.iamarn = other.iamarn || null;
+    this.zone = other.zone || null;
+    this.securityGroups = other.securityGroups || null;
   };
 
   this.addAccountDetails = function(other) {
@@ -841,6 +847,8 @@ function toCoreApiFormat(uiCluster) {
     this.vpc = null;
     this.subnet = null;
     this.iamarm = null;
+    this.zone = null;
+    this.securityGroups = null;
 
     this.load = function(other) {
       this.type = other.type || null;
@@ -851,6 +859,8 @@ function toCoreApiFormat(uiCluster) {
       this.vpc = other.vpc || null;
       this.subnet = other.subnet || null;
       this.iamarn = other.iamarn || null;
+      this.zone = other.zone || null;
+      this.securityGroups = other.securityGroups || null;
     }
   }