From c56bc3015b92b2dbfa3a071d1b8067d0a8b11060 Mon Sep 17 00:00:00 2001 From: Arunprasad Rajkumar Date: Wed, 22 Jul 2020 13:51:56 +0530 Subject: [PATCH 1/3] fix: Cleanup scripts to remove quirks --- .gitignore | 1 + openshift/deploy.sh | 2 +- openshift/helpers.sh | 14 ++++++++------ openshift/secrets-template.yaml | 15 +++++++++++++++ 4 files changed, 25 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index b55e0c4..1d143fe 100644 --- a/.gitignore +++ b/.gitignore @@ -98,3 +98,4 @@ target/ openshift/config.yaml openshift/secrets.yaml openshift/env.sh +openshift/rds.json diff --git a/openshift/deploy.sh b/openshift/deploy.sh index bcc33a8..fb4ede4 100755 --- a/openshift/deploy.sh +++ b/openshift/deploy.sh @@ -57,7 +57,7 @@ for key in "$@"; do done [ "$purge_aws_resources" == false ] && echo "Use --purge-aws-resources if you want to also clear previously allocated AWS resources (RDS database, SQS queues, S3 buckets, DynamoDB tables)." -openshift_login +# openshift_login create_or_reuse_project allocate_aws_rds generate_and_deploy_config diff --git a/openshift/helpers.sh b/openshift/helpers.sh index dfa3b5b..fbd306c 100644 --- a/openshift/helpers.sh +++ b/openshift/helpers.sh @@ -3,7 +3,7 @@ function is_set_or_fail() { local name=$1 local value=$2 - if [ ! -v value ] || [ "${value}" == "not-set" ]; then + if [ "${value}" == "not-set" ]; then echo "You have to set $name" >&2 exit 1 fi @@ -41,6 +41,7 @@ function deploy_secrets() { -p RDS_PASSWORD="$(/bin/echo -n "${RDS_PASSWORD}" | base64)" \ -p SNYK_TOKEN="$(/bin/echo -n "${SNYK_TOKEN}" | base64)" \ -p SNYK_ISS="$(/bin/echo -n "${SNYK_ISS}" | base64)" \ + -p HPF_MAVEN_INSIGHTS_BUCKET="$(/bin/echo -n "${USER_ID}-hpf-insights" | base64)" \ -f "${here}/secrets-template.yaml" > "${here}/secrets.yaml" oc apply -f secrets.yaml } @@ -91,11 +92,12 @@ function tag_rds_instance() { echo "Tagging RDS instance with ${TAGS}" aws rds add-tags-to-resource \ --resource-name "${RDS_ARN}" \ - --tags "${TAGS}" + --tags "${TAGS}" >/dev/null } function get_rds_instance_info() { - aws --output=table rds describe-db-instances --db-instance-identifier "${RDS_INSTANCE_NAME}" 2>/dev/null + aws --output=json rds describe-db-instances --db-instance-identifier "${RDS_INSTANCE_NAME}" 2>/dev/null 1>rds.json + return $? } function allocate_aws_rds() { @@ -105,7 +107,6 @@ function allocate_aws_rds() { --db-instance-identifier "${RDS_INSTANCE_NAME}" \ --db-instance-class "${RDS_INSTANCE_CLASS}" \ --db-name "${RDS_DBNAME}" \ - #--db-subnet-group-name "${RDS_SUBNET_GROUP_NAME}" \ --engine postgres \ --engine-version "9.6.1" \ --master-username "${RDS_DBADMIN}" \ @@ -132,8 +133,9 @@ function wait_for_rds_instance_info() { while true; do echo "Trying to get RDS DB endpoint for ${RDS_INSTANCE_NAME} ..." - RDS_ENDPOINT=$(get_rds_instance_info | grep -w Address | awk '{print $4}') - RDS_ARN=$(get_rds_instance_info | grep -w DBInstanceArn | awk '{print $4}') + get_rds_instance_info + RDS_ENDPOINT=$(jq -r '.DBInstances[0].Endpoint.Address' rds.json) + RDS_ARN=$(jq -r '.DBInstances[0].DBInstanceArn' rds.json) if [ -z "${RDS_ENDPOINT}" ]; then echo "DB is still initializing, waiting 30 seconds and retrying ..." diff --git a/openshift/secrets-template.yaml b/openshift/secrets-template.yaml index 1b88f13..4aff1fa 100644 --- a/openshift/secrets-template.yaml +++ b/openshift/secrets-template.yaml @@ -58,6 +58,15 @@ objects: sqs-access-key-id: ${AWS_ACCESS_KEY_ID} sqs-secret-access-key: ${AWS_SECRET_ACCESS_KEY} aws_region: ${AWS_DEFAULT_REGION} +- apiVersion: v1 + kind: Secret + metadata: + name: hpf-maven-insights-s3 + type: Opaque + data: + aws_access_key_id: ${AWS_ACCESS_KEY_ID} + aws_secret_access_key: ${AWS_SECRET_ACCESS_KEY} + bucket: ${HPF_MAVEN_INSIGHTS_BUCKET} - apiVersion: v1 kind: Secret metadata: @@ -237,3 +246,9 @@ parameters: name: SNYK_ISS value: "bm90LXNldA==" # not-set +- description: Maven insights bucket name + displayName: Maven insights bucket name + required: false + name: HPF_MAVEN_INSIGHTS_BUCKET + value: "not-set" # not-set + From 43f746cac57fb6ce8e1de8fa786425f556c290d0 Mon Sep 17 00:00:00 2001 From: Arunprasad Rajkumar Date: Wed, 22 Jul 2020 21:07:25 +0530 Subject: [PATCH 2/3] feat: Use cluster local postgres --- openshift/deploy.sh | 13 ------- openshift/helpers.sh | 58 ++----------------------------- openshift/postgres.yaml | 77 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 69 deletions(-) create mode 100644 openshift/postgres.yaml diff --git a/openshift/deploy.sh b/openshift/deploy.sh index fb4ede4..00168ff 100755 --- a/openshift/deploy.sh +++ b/openshift/deploy.sh @@ -77,27 +77,14 @@ oc_process_apply "${templates_dir}/gremlin-docker.yaml" "-p CHANNELIZER=http -p sleep 20 oc_process_apply "${templates_dir}/data-model.yaml" sleep 20 -oc_process_apply "${templates_dir}/jobs.yaml" -sleep 20 -oc_process_apply "${templates_dir}/worker.yaml" "-p WORKER_ADMINISTRATION_REGION=ingestion -p WORKER_EXCLUDE_QUEUES=GraphImporterTask" -sleep 20 -oc_process_apply "${templates_dir}/worker.yaml" "-p WORKER_ADMINISTRATION_REGION=ingestion -p WORKER_INCLUDE_QUEUES=GraphImporterTask -p WORKER_NAME_SUFFIX=-graph-import" -sleep 20 oc_process_apply "${templates_dir}/worker.yaml" "-p WORKER_ADMINISTRATION_REGION=api -p WORKER_RUN_DB_MIGRATIONS=1 -p WORKER_EXCLUDE_QUEUES=GraphImporterTask" sleep 20 -oc_process_apply "${templates_dir}/worker.yaml" "-p WORKER_ADMINISTRATION_REGION=api -p WORKER_INCLUDE_QUEUES=GraphImporterTask -p WORKER_NAME_SUFFIX=-graph-import" -sleep 20 oc_process_apply "${templates_dir}/f8a-server-backbone.yaml" sleep 20 oc_process_apply "${templates_dir}/server.yaml" sleep 20 -oc_process_apply "${templates_dir}/stack-analysis.yaml" "-p KRONOS_SCORING_REGION=maven" # kronos-pypi is not used/maintained now # sleep 20 # oc_process_apply "${templates_dir}/stack-analysis.yaml" "-p KRONOS_SCORING_REGION=pypi" sleep 20 oc_process_apply "${templates_dir}/license-analysis.yaml" -sleep 20 -oc_process_apply "${templates_dir}/stack-report-ui.yaml" "-p REPLICAS=1" -sleep 20 -oc_process_apply "${templates_dir}/api-gateway.yaml" diff --git a/openshift/helpers.sh b/openshift/helpers.sh index fbd306c..944813b 100644 --- a/openshift/helpers.sh +++ b/openshift/helpers.sh @@ -87,62 +87,8 @@ function create_or_reuse_project() { fi } -function tag_rds_instance() { - TAGS="Key=ENV,Value=${DEPLOYMENT_PREFIX}" - echo "Tagging RDS instance with ${TAGS}" - aws rds add-tags-to-resource \ - --resource-name "${RDS_ARN}" \ - --tags "${TAGS}" >/dev/null -} - -function get_rds_instance_info() { - aws --output=json rds describe-db-instances --db-instance-identifier "${RDS_INSTANCE_NAME}" 2>/dev/null 1>rds.json - return $? -} - function allocate_aws_rds() { - if ! get_rds_instance_info; then - aws rds create-db-instance \ - --allocated-storage "${RDS_STORAGE}" \ - --db-instance-identifier "${RDS_INSTANCE_NAME}" \ - --db-instance-class "${RDS_INSTANCE_CLASS}" \ - --db-name "${RDS_DBNAME}" \ - --engine postgres \ - --engine-version "9.6.1" \ - --master-username "${RDS_DBADMIN}" \ - --master-user-password "${RDS_PASSWORD}" \ - --publicly-accessible \ - --storage-type gp2 - #--storage-encrypted - echo "Waiting (60s) for ${RDS_INSTANCE_NAME} to come online" - sleep 60 - wait_for_rds_instance_info - else - echo "DB instance ${RDS_INSTANCE_NAME} already exists" - wait_for_rds_instance_info - if [ "$purge_aws_resources" == true ]; then - echo "recreating database" - PGPASSWORD="${RDS_PASSWORD}" psql -d template1 -h "${RDS_ENDPOINT}" -U "${RDS_DBADMIN}" -c "drop database ${RDS_DBNAME}" - PGPASSWORD="${RDS_PASSWORD}" psql -d template1 -h "${RDS_ENDPOINT}" -U "${RDS_DBADMIN}" -c "create database ${RDS_DBNAME}" - fi - fi - tag_rds_instance -} - -function wait_for_rds_instance_info() { - while true; do - echo "Trying to get RDS DB endpoint for ${RDS_INSTANCE_NAME} ..." - - get_rds_instance_info - RDS_ENDPOINT=$(jq -r '.DBInstances[0].Endpoint.Address' rds.json) - RDS_ARN=$(jq -r '.DBInstances[0].DBInstanceArn' rds.json) - - if [ -z "${RDS_ENDPOINT}" ]; then - echo "DB is still initializing, waiting 30 seconds and retrying ..." - sleep 30 - else - break - fi - done + RDS_ENDPOINT="f8a-postgres" + oc apply -f postgres.yaml --wait=true } diff --git a/openshift/postgres.yaml b/openshift/postgres.yaml new file mode 100644 index 0000000..813ba3e --- /dev/null +++ b/openshift/postgres.yaml @@ -0,0 +1,77 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: f8a-postgres + labels: + f8a-component: f8a-postgres +spec: + replicas: 1 + selector: + matchLabels: + f8a-component: f8a-postgres + template: + metadata: + labels: + f8a-component: f8a-postgres + spec: + volumes: + - name: postgres-data + persistentVolumeClaim: + claimName: f8a-postgres + containers: + - name: postgres + image: postgres:9.6 + imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 5432 + env: + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + key: username + name: coreapi-postgres + - name: POSTGRES_DB + valueFrom: + secretKeyRef: + key: database + name: coreapi-postgres + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + key: password + name: coreapi-postgres + - name: PGDATA + value: "/var/lib/postgres/data/f8a" + volumeMounts: + - name: postgres-data + mountPath: "/var/lib/postgres/data" +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: f8a-postgres + labels: + f8a-component: f8a-postgres +spec: + accessModes: + - "ReadWriteOnce" + resources: + requests: + storage: "10Gi" + volumeName: "f8a-postgres" +--- +apiVersion: v1 +kind: Service +metadata: + name: f8a-postgres + labels: + f8a-component: f8a-postgres +spec: + type: ClusterIP + ports: + - port: 5432 + protocol: TCP + name: postgres + targetPort: 5432 + selector: + f8a-component: f8a-postgres From 1f6eea5910fb0e01061c619fd62e0faa13648998 Mon Sep 17 00:00:00 2001 From: Arunprasad Rajkumar Date: Thu, 23 Jul 2020 19:11:28 +0530 Subject: [PATCH 3/3] Add insight services and simplify scripts --- openshift/deploy.sh | 43 +++++++++++---------------------- openshift/helpers.sh | 32 ++++++++++++------------ openshift/secrets-template.yaml | 30 +++++++++++++++++++++++ 3 files changed, 59 insertions(+), 46 deletions(-) diff --git a/openshift/deploy.sh b/openshift/deploy.sh index 00168ff..a75e496 100755 --- a/openshift/deploy.sh +++ b/openshift/deploy.sh @@ -37,12 +37,6 @@ is_set_or_fail AWS_SECRET_ACCESS_KEY "${AWS_SECRET_ACCESS_KEY}" is_set_or_fail AWS_DEFAULT_REGION "${AWS_DEFAULT_REGION}" is_set_or_fail OC_TOKEN "${OC_TOKEN}" -templates_dir="${here}/templates" -templates="fabric8-analytics-jobs fabric8-analytics-server fabric8-analytics-data-model -fabric8-analytics-worker fabric8-analytics-pgbouncer gremlin-docker -fabric8-analytics-license-analysis fabric8-analytics-stack-analysis -f8a-server-backbone fabric8-analytics-stack-report-ui fabric8-analytics-api-gateway" - purge_aws_resources=false # default for key in "$@"; do case $key in @@ -63,28 +57,19 @@ allocate_aws_rds generate_and_deploy_config deploy_secrets -#Get templates for fabric8-analytics projects -for template in ${templates} -do - curl -sS "https://raw.githubusercontent.com/fabric8-analytics/${template}/master/openshift/template.yaml" > "${templates_dir}/${template#fabric8-analytics-}.yaml" -done +github_org_base="https://raw.githubusercontent.com/fabric8-analytics" +openshift_template_path="master/openshift/template.yaml" +openshift_template_path2="master/openshift/template-prod.yaml" -oc_process_apply "${templates_dir}/pgbouncer.yaml" -sleep 20 -oc_process_apply "${templates_dir}/gremlin-docker.yaml" "-p CHANNELIZER=http -p REST_VALUE=1 -p IMAGE_TAG=latest" -sleep 20 -oc_process_apply "${templates_dir}/gremlin-docker.yaml" "-p CHANNELIZER=http -p REST_VALUE=1 -p IMAGE_TAG=latest -p QUERY_ADMINISTRATION_REGION=ingestion" -sleep 20 -oc_process_apply "${templates_dir}/data-model.yaml" -sleep 20 -oc_process_apply "${templates_dir}/worker.yaml" "-p WORKER_ADMINISTRATION_REGION=api -p WORKER_RUN_DB_MIGRATIONS=1 -p WORKER_EXCLUDE_QUEUES=GraphImporterTask" -sleep 20 -oc_process_apply "${templates_dir}/f8a-server-backbone.yaml" -sleep 20 -oc_process_apply "${templates_dir}/server.yaml" -sleep 20 -# kronos-pypi is not used/maintained now -# sleep 20 -# oc_process_apply "${templates_dir}/stack-analysis.yaml" "-p KRONOS_SCORING_REGION=pypi" +oc_process_apply "${github_org_base}/fabric8-analytics-pgbouncer/${openshift_template_path}" +oc_process_apply "${github_org_base}/gremlin-docker/${openshift_template_path}" "-p CHANNELIZER=http -p REST_VALUE=1 -p IMAGE_TAG=latest" +oc_process_apply "${github_org_base}/gremlin-docker/${openshift_template_path}" "-p CHANNELIZER=http -p REST_VALUE=1 -p IMAGE_TAG=latest -p QUERY_ADMINISTRATION_REGION=ingestion" sleep 20 -oc_process_apply "${templates_dir}/license-analysis.yaml" +oc_process_apply "${github_org_base}/fabric8-analytics-data-model/${openshift_template_path}" +oc_process_apply "${github_org_base}/fabric8-analytics-worker/${openshift_template_path}" "-p WORKER_ADMINISTRATION_REGION=api -p WORKER_RUN_DB_MIGRATIONS=1 -p WORKER_EXCLUDE_QUEUES=GraphImporterTask" +oc_process_apply "${github_org_base}/f8a-server-backbone/${openshift_template_path}" +oc_process_apply "${github_org_base}/fabric8-analytics-server/${openshift_template_path}" +oc_process_apply "${github_org_base}/fabric8-analytics-license-analysis/${openshift_template_path}" +oc_process_apply "${github_org_base}/fabric8-analytics-npm-insights/${openshift_template_path}" +oc_process_apply "${github_org_base}/f8a-pypi-insights/${openshift_template_path}" +oc_process_apply "${github_org_base}/f8a-hpf-insights/${openshift_template_path2}" "-p HPF_SCORING_REGION=maven -p RESTART_POLICY=Always" diff --git a/openshift/helpers.sh b/openshift/helpers.sh index 944813b..22b30e3 100644 --- a/openshift/helpers.sh +++ b/openshift/helpers.sh @@ -29,19 +29,21 @@ function generate_and_deploy_config() { function deploy_secrets() { #All secrets must be base64 encoded - oc process -p AWS_ACCESS_KEY_ID="$(/bin/echo -n "${AWS_ACCESS_KEY_ID}" | base64)" \ - -p AWS_SECRET_ACCESS_KEY="$(/bin/echo -n "${AWS_SECRET_ACCESS_KEY}" | base64)" \ - -p AWS_DEFAULT_REGION="$(/bin/echo -n "${AWS_DEFAULT_REGION}" | base64)" \ - -p GITHUB_API_TOKENS="$(/bin/echo -n "${GITHUB_API_TOKENS}" | base64)" \ - -p GITHUB_OAUTH_CONSUMER_KEY="$(/bin/echo -n "${GITHUB_OAUTH_CONSUMER_KEY}" | base64)" \ + oc process -p AWS_ACCESS_KEY_ID="$(echo -n "${AWS_ACCESS_KEY_ID}" | base64)" \ + -p AWS_SECRET_ACCESS_KEY="$(echo -n "${AWS_SECRET_ACCESS_KEY}" | base64)" \ + -p AWS_DEFAULT_REGION="$(echo -n "${AWS_DEFAULT_REGION}" | base64)" \ + -p GITHUB_API_TOKENS="$(echo -n "${GITHUB_API_TOKENS}" | base64)" \ + -p GITHUB_OAUTH_CONSUMER_KEY="$(echo -n "${GITHUB_OAUTH_CONSUMER_KEY}" | base64)" \ -p GITHUB_OAUTH_CONSUMER_SECRET="$(/bin/echo -n "${GITHUB_OAUTH_CONSUMER_SECRET}" | base64)" \ - -p LIBRARIES_IO_TOKEN="$(/bin/echo -n "${LIBRARIES_IO_TOKEN}" | base64)" \ - -p FLASK_APP_SECRET_KEY="$(/bin/echo -n "${FLASK_APP_SECRET_KEY}" | base64)" \ - -p RDS_ENDPOINT="$(/bin/echo -n "${RDS_ENDPOINT}" | base64)" \ - -p RDS_PASSWORD="$(/bin/echo -n "${RDS_PASSWORD}" | base64)" \ - -p SNYK_TOKEN="$(/bin/echo -n "${SNYK_TOKEN}" | base64)" \ - -p SNYK_ISS="$(/bin/echo -n "${SNYK_ISS}" | base64)" \ - -p HPF_MAVEN_INSIGHTS_BUCKET="$(/bin/echo -n "${USER_ID}-hpf-insights" | base64)" \ + -p LIBRARIES_IO_TOKEN="$(echo -n "${LIBRARIES_IO_TOKEN}" | base64)" \ + -p FLASK_APP_SECRET_KEY="$(echo -n "${FLASK_APP_SECRET_KEY}" | base64)" \ + -p RDS_ENDPOINT="$(echo -n "${RDS_ENDPOINT}" | base64)" \ + -p RDS_PASSWORD="$(echo -n "${RDS_PASSWORD}" | base64)" \ + -p SNYK_TOKEN="$(echo -n "${SNYK_TOKEN}" | base64)" \ + -p SNYK_ISS="$(echo -n "${SNYK_ISS}" | base64)" \ + -p CVAE_NPM_INSIGHTS_BUCKET="$(echo -n "${USER_ID}-cvae-npm-insights" | base64)" \ + -p HPF_PYPI_INSIGHTS_BUCKET="$(echo -n "${USER_ID}-hpf-pypi-insights" | base64)" \ + -p HPF_MAVEN_INSIGHTS_BUCKET="$(echo -n "${USER_ID}-hpf-maven-insights" | base64)" \ -f "${here}/secrets-template.yaml" > "${here}/secrets.yaml" oc apply -f secrets.yaml } @@ -49,11 +51,7 @@ function deploy_secrets() { function oc_process_apply() { echo -e "\\n Processing template - $1 ($2) \\n" # Don't quote $2 as we need it to split into individual arguments - oc process -f "$1" $2 | oc apply -f - -} - -function openshift_login() { - oc login "${OC_URI}" --token="${OC_TOKEN}" --insecure-skip-tls-verify=true + oc process -f "$1" $2 | oc apply -f - --wait=true } function purge_aws_resources() { diff --git a/openshift/secrets-template.yaml b/openshift/secrets-template.yaml index 4aff1fa..6599652 100644 --- a/openshift/secrets-template.yaml +++ b/openshift/secrets-template.yaml @@ -58,6 +58,24 @@ objects: sqs-access-key-id: ${AWS_ACCESS_KEY_ID} sqs-secret-access-key: ${AWS_SECRET_ACCESS_KEY} aws_region: ${AWS_DEFAULT_REGION} +- apiVersion: v1 + kind: Secret + metadata: + name: hpf-pypi-insights-s3 + type: Opaque + data: + aws_access_key_id: ${AWS_ACCESS_KEY_ID} + aws_secret_access_key: ${AWS_SECRET_ACCESS_KEY} + bucket: ${HPF_PYPI_INSIGHTS_BUCKET} +- apiVersion: v1 + kind: Secret + metadata: + name: cvae-npm-insights-s3 + type: Opaque + data: + aws_access_key_id: ${AWS_ACCESS_KEY_ID} + aws_secret_access_key: ${AWS_SECRET_ACCESS_KEY} + bucket: ${CVAE_NPM_INSIGHTS_BUCKET} - apiVersion: v1 kind: Secret metadata: @@ -246,6 +264,18 @@ parameters: name: SNYK_ISS value: "bm90LXNldA==" # not-set +- description: Pypi insights bucket name + displayName: Pypi insights bucket name + required: false + name: HPF_PYPI_INSIGHTS_BUCKET + value: "not-set" # not-set + +- description: npm insights bucket name + displayName: npm insights bucket name + required: false + name: CVAE_NPM_INSIGHTS_BUCKET + value: "not-set" # not-set + - description: Maven insights bucket name displayName: Maven insights bucket name required: false