Update to Aeolus (#26)

* Migrate to pg16

* Adding migration tests

* Reusing the connection and leaving it open to speed things up

* Fixed a table name conflict and added the machine builds

cdk/app.py

@@ -7,6 +7,7 @@
 
 import os
 from database_stack import DatabaseStack
+from machine_stack import MachineStack
 from settings import settings
 
 code_dir = (Path(__file__).parent.absolute()).parent.absolute()
@@ -19,7 +20,8 @@
     f"openaq-db-{settings.ENV}",
     codeDirectory=code_dir,
     keyName=settings.KEY_NAME,
-    sshIpRange=settings.IP_ADDRESS,
+    devSecurityGroup=settings.DEV_SECURITY_GROUP,
+    privateIpAddress=settings.PRIVATE_IP_ADDRESS,
     elasticIpAllocationId=settings.ELASTIC_IP_ALLOCTION_ID,
     linuxVersion=settings.LINUX_VERSION,
     snapshotId=settings.SNAPSHOT_ID,
@@ -32,10 +34,12 @@
     databaseWritePassword=settings.DATABASE_WRITE_PASSWORD,
     databaseMonitorUser=settings.DATABASE_MONITOR_USER,
     databaseMonitorPassword=settings.DATABASE_MONITOR_PASSWORD,
+    databasePostgresUser=settings.DATABASE_POSTGRES_USER,
     databasePostgresPassword=settings.DATABASE_POSTGRES_PASSWORD,
     databaseHost=settings.DATABASE_HOST,
     databasePort=settings.DATABASE_PORT,
     databaseDb=settings.DATABASE_DB,
+    transferUri=settings.TRANSFER_URI,
     pgSharedBuffers=settings.PG_SHARED_BUFFERS,
     pgWalBuffers=settings.PG_WAL_BUFFERS,
     pgEffectiveCacheSize=settings.PG_EFFECTIVE_CACHE_SIZE,
@@ -48,6 +52,22 @@
     }
 )
 
-Tags.of(db).add("Project", settings.ENV)
+mi = MachineStack(
+    app,
+    f"openaq-db-machine-image",
+    codeDirectory=code_dir,
+    keyName=settings.KEY_NAME,
+    sshIpRange=settings.IP_ADDRESS,
+    linuxVersion=settings.LINUX_VERSION,
+    instanceType=settings.INSTANCE_TYPE,
+    vpcId=settings.VPC_ID,
+    env={
+        'account': os.environ['CDK_DEFAULT_ACCOUNT'],
+        'region': os.environ['CDK_DEFAULT_REGION']
+    }
+)
+
+
+#Tags.of(db).add("Project", settings.ENV)
 
 app.synth()

cdk/database_stack.py

@@ -28,6 +28,7 @@ def __init__(
         rootVolumeIops: int = 2000,
         dataVolumeSize: int = 1000,
         dataVolumeIops: int = 3000,
+        privateIpAddress: str = None,
         elasticIpAllocationId: str = None,
         machineImageName: str = None,
         snapshotId: str = None,
@@ -36,16 +37,18 @@ def __init__(
         expose6432: bool = True,
         expose9187: bool = True,
         expose9100: bool = True,
-        httpIpRange: str = '0.0.0.0/0',
-        sshIpRange: str = None,
-        linuxVersion: str = 'amazon_linux_2',
+        httpIpRange: str = '10.0.0.0/16',
+        devSecurityGroup: str = None,
+        transferUri: str = None,
+        linuxVersion: str = 'amazon_linux_2023',
         instanceType: str = "r5.xlarge",
-        databaseReadUser: str = 'postgres',
+        databaseReadUser: str = 'postgres_read',
+        databaseWriteUser: str = 'postgres_write',
+        databaseMonitorUser: str = 'postgres_monitor',
         databaseReadPassword: str = 'postgres',
-        databaseWriteUser: str = 'postgres',
         databaseWritePassword: str = 'postgres',
-        databaseMonitorUser: str = 'postgres',
         databaseMonitorPassword: str = 'postgres',
+        databasePostgresUser: str = 'postgres',
         databasePostgresPassword: str = 'postgres',
         databaseHost: str = 'localhost',
         databasePort: str = '5432',
@@ -79,11 +82,11 @@ def __init__(
         )
 
         # add an ingress rule for ssh purposes
-        if sshIpRange is not None:
-            sg.add_ingress_rule(
-                peer=_ec2.Peer.ipv4(sshIpRange),
-                connection=_ec2.Port.tcp(22)
-            )
+        #if sshIpRange is not None:
+        #    sg.add_ingress_rule(
+        #        peer=_ec2.Peer.ipv4(sshIpRange),
+        #        connection=_ec2.Port.tcp(22)
+        #    )
 
         # if we want to expose 5432 on the instance
         if expose5432:
@@ -126,17 +129,19 @@ def __init__(
             "DATABASE_WRITE_PASSWORD": databaseWritePassword,
             "DATABASE_MONITOR_USER": databaseMonitorUser,
             "DATABASE_MONITOR_PASSWORD": databaseMonitorPassword,
+            "DATABASE_POSTGRES_USER": databasePostgresUser,
             "DATABASE_POSTGRES_PASSWORD": databasePostgresPassword,
             "DATABASE_HOST": databaseHost,
             "DATABASE_PORT": databasePort,
             "DATABASE_DB": databaseDb,
+            "TRANSFER_URI": transferUri,
             "PG_SHARED_BUFFERS": pgSharedBuffers,
             "PG_WAL_BUFFERS": pgWalBuffers,
             "PG_EFFECTIVE_CACHE_SIZE": pgEffectiveCacheSize,
             "PG_WORK_MEM": pgWorkMem,
             "PG_MAINTENANCE_WORK_MEM": pgMaintenanceWorkMem,
             "SNAPSHOT_ID": snapshotId,
-            "PGPATH": "/usr/bin",
+            "PGPATH": "/usr/local/pgsql/bin",
             "PGDATA": "/db/data",
             "PGCONFIG": "/db/data/postgresql.conf",
         }
@@ -202,7 +207,10 @@ def __init__(
                 'cd /app && unzip -o db.zip -d openaqdb'
             ),
             _ec2.InitCommand.shell_command(
-                'mkdir -p /var/log/openaq && /app/openaqdb/install_database.sh > /var/log/openaq/install_database.log 2>&1'
+                'mkdir -p /var/log/openaq && /app/openaqdb/install_database.sh > /var/log/openaq/install_database_server.log 2>&1'
+            ),
+            _ec2.InitCommand.shell_command(
+                'sudo -i -u ec2-user IMPORT_FILE_LIMIT=5 /app/openaqdb/initial_setup.sh > /var/log/openaq/initial_setup.log 2>&1'
             ),
         )
 
@@ -241,10 +249,15 @@ def __init__(
             vpc_subnets=_ec2.SubnetSelection(
                 subnet_type=_ec2.SubnetType.PUBLIC
             ),
+            private_ip_address=privateIpAddress,
             block_devices=blockDevices,
             user_data=UserData
         )
 
+        if devSecurityGroup is not None:
+            dev_sg = _ec2.SecurityGroup.from_security_group_id(self, "SG", devSecurityGroup, mutable=False)
+            ec2.add_security_group(dev_sg)
+
         backup_bucket = 'openaq-db-backups'
         openaq_backup_bucket = _s3.Bucket.from_bucket_name(
             self, "{env_name}-BACKUP-BUCKET", backup_bucket
@@ -270,3 +283,19 @@ def __init__(
             value=ec2.instance_public_ip,
             description="public ip",
             export_name=f"{id}-public-ip")
+
+
+        CfnOutput(
+            scope=self,
+            id=f"{id}-private-ip",
+            value=ec2.instance_private_ip,
+            description="private ip",
+            export_name=f"{id}-private-ip")
+
+
+        CfnOutput(
+            scope=self,
+            id=f"{id}-public-url",
+            value=ec2.instance_public_dns_name,
+            description="public dns name",
+            export_name=f"{id}-public-url")

cdk/machine_stack.py

@@ -0,0 +1,128 @@
+from aws_cdk import (
+    aws_ec2 as _ec2,
+    aws_s3 as _s3,
+    Stack,
+    CfnOutput,
+    Duration,
+    Tags,
+)
+
+from constructs import Construct
+import os
+import sys
+
+
+class MachineStack(Stack):
+    def __init__(
+        self,
+        scope: Construct,
+        id: str,
+        codeDirectory: str,
+        rootVolumeSize: int = 25,
+        rootVolumeIops: int = 2000,
+        keyName: str = None,
+        httpIpRange: str = '0.0.0.0/0',
+        sshIpRange: str = None,
+        linuxVersion: str = 'amazon_linux_2023',
+        instanceType: str = "r5.xlarge",
+        vpcId: str = None,
+        **kwargs,
+    ) -> None:
+        """Define stack."""
+        super().__init__(scope, id, **kwargs)
+
+        # Get the VPC or create a new one
+        if vpcId is not None:
+            vpc = _ec2.Vpc.from_lookup(
+                self,
+                f"{id}-machine-vpc",
+                vpc_id=vpcId,
+            )
+        else:
+            vpc = _ec2.Vpc.from_lookup(
+               self,
+               f"{id}-machine-vpc",
+                is_default=True,
+            )
+
+        # add some security groups
+        sg = _ec2.SecurityGroup(
+            self,
+            f"{id}-dbstack-ssh-sg",
+            vpc=vpc,
+            allow_all_outbound=True,
+        )
+
+        # add an ingress rule for ssh purposes
+        if sshIpRange is not None:
+            sg.add_ingress_rule(
+                peer=_ec2.Peer.ipv4(sshIpRange),
+                connection=_ec2.Port.tcp(22)
+            )
+
+        setup_dir = os.path.join(codeDirectory, 'openaqdb')
+
+        blockDevices = []
+        rootVolume = _ec2.BlockDevice(
+            device_name="/dev/xvda",
+            volume=_ec2.BlockDeviceVolume(
+                ebs_device=_ec2.EbsDeviceProps(
+                    iops=rootVolumeIops,
+                    volume_size=rootVolumeSize,
+                    volume_type=_ec2.EbsDeviceVolumeType.IO2,
+                )
+            ),
+        )
+        blockDevices.append(rootVolume)
+
+
+        initElements = _ec2.CloudFormationInit.from_elements(
+            _ec2.InitFile.from_asset("/app/db.zip", setup_dir),
+            _ec2.InitCommand.shell_command(
+                'cd /app && unzip -o db.zip -d openaqdb'
+            ),
+            _ec2.InitCommand.shell_command(
+                'mkdir -p /var/log/openaq && /app/openaqdb/build_pg16.sh > /var/log/openaq/build_machine.log 2>&1'
+            ),
+        )
+
+        initOptions = _ec2.ApplyCloudFormationInitOptions(
+            # helpful when diagnosing issues
+            ignore_failures=True,
+            # Optional, how long the installation is expected to take
+            # (5 minutes by default)
+            timeout=Duration.minutes(120),
+        )
+
+        image = _ec2.MachineImage.latest_amazon_linux2023(
+            cpu_type=_ec2.AmazonLinuxCpuType.X86_64,
+        )
+
+        # user data results are logged to
+        # /var/log/cloud-init.log
+        # /var/log/cloud-init-output.log
+        ec2 = _ec2.Instance(
+            self,
+            f"{id}-machine-stack-database",
+            instance_name=f"{id}-machine-stack-database",
+            instance_type=_ec2.InstanceType(instanceType),
+            machine_image=image,
+            init=initElements,
+            init_options=initOptions,
+            vpc=vpc,
+            security_group=sg,
+            key_name=keyName,
+            vpc_subnets=_ec2.SubnetSelection(
+                subnet_type=_ec2.SubnetType.PUBLIC
+            ),
+            block_devices=blockDevices,
+            #user_data=UserData
+        )
+
+
+        CfnOutput(
+            scope=self,
+            id=f"{id}-public-url",
+            value=ec2.instance_public_dns_name,
+            description="public dns name",
+            export_name=f"{id}-public-url")

cdk/requirements.txt

@@ -2,3 +2,4 @@ aws-cdk-lib==2.87.0
 boto3
 pydantic==1.10
 python-dotenv
+requests

cdk/settings.py

@@ -1,33 +1,40 @@
 from pydantic import BaseSettings
 from pathlib import Path
 from os import environ
+from requests import get
 
 
+ip = get('https://api.ipify.org').content.decode('utf8')
+
 class Settings(BaseSettings):
-    ENV: str
-    DATABASE_READ_USER: str
-    DATABASE_READ_PASSWORD: str
-    DATABASE_WRITE_USER: str
-    DATABASE_WRITE_PASSWORD: str
-    DATABASE_POSTGRES_PASSWORD: str
-    DATABASE_HOST: str
-    DATABASE_PORT: int
-    DATABASE_DB: str
+    ENV: str = 'sandbox'
+    DATABASE_READ_USER: str = 'postgres_read'
+    DATABASE_READ_PASSWORD: str = 'postgres'
+    DATABASE_WRITE_USER: str = 'postgres_write'
+    DATABASE_WRITE_PASSWORD: str = 'postgres'
+    DATABASE_POSTGRES_USER: str = 'postgres'
+    DATABASE_POSTGRES_PASSWORD: str = 'postgres'
+    DATABASE_HOST: str = 'localhost'
+    DATABASE_PORT: int = '5432'
+    DATABASE_DB: str = 'openaqdev'
     KEY_NAME: str
     DATA_VOLUME_SIZE: int = 3000
     DATABASE_MONITOR_USER: str = None
     DATABASE_MONITOR_PASSWORD: str = None
+    TRANSFER_URI: str = None
     PG_SHARED_BUFFERS: str = ''
     PG_WAL_BUFFERS: str = ''
     PG_EFFECTIVE_CACHE_SIZE: str = ''
     PG_WORK_MEM: str = ''
     PG_MAINTENANCE_WORK_MEM: str = ''
-    IP_ADDRESS: str = None
+    IP_ADDRESS: str = f"{ip}/32"
+    PRIVATE_IP_ADDRESS: str = None
+    DEV_SECURITY_GROUP: str = None
     ELASTIC_IP_ALLOCTION_ID: str = None
     SNAPSHOT_ID: str = None
     VPC_ID: str = None
-    LINUX_VERSION: str = 'AMAZON_LINUX_2'  # UBUNTU | AMAZON_LINUX_2022
-    INSTANCE_TYPE: str = None
+    LINUX_VERSION: str = 'AMAZON_LINUX_2023'  # UBUNTU | AMAZON_LINUX_2022
+    INSTANCE_TYPE: str = 't3.large'
     MACHINE_IMAGE_NAME: str = None
 
     class Config: