Features/versioning and flags

.github/workflows/deploy-staging.yml

@@ -3,48 +3,7 @@ name: Deploy staging
 on:
   push:
     branches:
-      - release-*
-
-env:
-  ENV: "staging"
-  PROJECT: "openaq"
-  DATABASE_READ_USER: ${{ secrets.DATABASE_READ_USER }}
-  DATABASE_READ_PASSWORD: ${{ secrets.DATABASE_READ_PASSWORD }}
-  DATABASE_WRITE_USER: ${{ secrets.DATABASE_WRITE_USER }}
-  DATABASE_WRITE_PASSWORD: ${{ secrets.DATABASE_WRITE_PASSWORD }}
-  DATABASE_DB: ${{ secrets.DATABASE_DB }}
-  DATABASE_HOST: ${{ secrets.DATABASE_HOST_STAGING }}
-  DATABASE_PORT: ${{ secrets.STAGING_DATABASE_PORT }}
-  API_LAMBDA_MEMORY_SIZE: ${{ secrets.API_LAMBDA_MEMORY_SIZE_STAGING }}
-  FETCH_ASCENDING: ${{ secrets.FETCH_ASCENDING }}
-  PIPELINE_LIMIT: ${{ secrets.PIPELINE_LIMIT }}
-  METADATA_LIMIT: ${{ secrets.METADATA_LIMIT }}
-  REALTIME_LIMIT: ${{ secrets.REALTIME_LIMIT }}
-  FETCH_BUCKET: ${{ secrets.FETCH_BUCKET }}
-  ETL_BUCKET: ${{ secrets.ETL_BUCKET }}
-  HOSTED_ZONE_ID: ${{ secrets.HOSTED_ZONE_ID }}
-  HOSTED_ZONE_NAME: ${{ secrets.HOSTED_ZONE_NAME }}
-  DOMAIN_NAME: "staging.openaq.org"
-  WEB_ACL_ID: ${{ secrets.WEB_ACL_ID }}
-  CERTIFICATE_ARN: ${{ secrets.CERTIFICATE_ARN }}
-  FASTAPI_URL: ""
-
-  CDK_ACCOUNT: ${{ secrets.CDK_ACCOUNT }}
-  CDK_REGION: ${{ secrets.CDK_REGION }}
-
-  VPC_ID: ${{ secrets.VPC_ID }}
-
-  RATE_LIMITING: True
-  RATE_AMOUNT_KEY: 10
-  RATE_TIME: 1
-
-  EMAIL_SENDER: ${{ secrets.EMAIL_SENDER }}
-  SMTP_EMAIL_HOST: ${{ secrets.SMTP_EMAIL_HOST }}
-  SMTP_EMAIL_USER: ${{ secrets.SMTP_EMAIL_USER }}
-  SMTP_EMAIL_PASSWORD: ${{ secrets.SMTP_EMAIL_PASSWORD }}
-
-  EXPLORER_API_KEY: ${{ secrets.EXPLORER_API_KEY }}
-
+      - 'features/**'
 
 jobs:
   deploy:
@@ -56,10 +15,18 @@ jobs:
       - name: Configure aws credentials
         uses: aws-actions/configure-aws-credentials@master
         with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_STAGING }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY_STAGING }}
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_PROD }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY_PROD }}
           aws-region: ${{ secrets.AWS_REGION }}
 
+      - name: Get envionmental values
+        uses: aws-actions/aws-secretsmanager-get-secrets@v2
+        with:
+          secret-ids: |
+            STAGING, openaq-env/staging
+          name-transformation: uppercase
+          parse-json-secrets: true
+
       - uses: actions/setup-node@v3
         with:
           node-version: "18"
@@ -79,6 +46,40 @@ jobs:
           python-version: '3.11'
 
       - name: Deploy stack
+        env:
+          ENV: "staging"
+          PROJECT: "openaq"
+          DATABASE_READ_USER: ${{ env.STAGING_DATABASE_READ_USER }}
+          DATABASE_READ_PASSWORD: ${{ env.STAGING_DATABASE_READ_PASSWORD }}
+          DATABASE_WRITE_USER: ${{ env.STAGING_DATABASE_WRITE_USER }}
+          DATABASE_WRITE_PASSWORD: ${{ env.STAGING_DATABASE_WRITE_PASSWORD }}
+          DATABASE_DB: ${{ env.STAGING_DATABASE_DB }}
+          DATABASE_HOST: ${{ env.STAGING_DATABASE_HOST }}
+          DATABASE_PORT: ${{ env.STAGING_DATABASE_PORT }}
+          API_LAMBDA_MEMORY_SIZE: ${{ env.STAGING_API_LAMBDA_MEMORY_SIZE }}
+
+          CDK_ACCOUNT: ${{ secrets.CDK_ACCOUNT }}
+          CDK_REGION: ${{ secrets.CDK_REGION }}
+
+          VPC_ID: ${{ env.STAGING_VPC_ID }}
+
+          RATE_LIMITING: True
+          RATE_AMOUNT: 10
+          RATE_AMOUNT_KEY: 60
+          RATE_TIME: 1
+          USER_AGENT: ${{ env.STAGING_USER_AGENT }}
+          ORIGIN: ${{ env.STAGING_ORIGIN }}
+          REDIS_HOST: ${{ env.STAGING_REDIS_HOST }}
+          REDIS_PORT: ${{ env.STAGING_REDIS_PORT }}
+          REDIS_SECURITY_GROUP_ID: ${{ env.STAGING_REDIS_SECURITY_GROUP_ID }}
+
+          EMAIL_SENDER: ${{ env.STAGING_EMAIL_SENDER }}
+          SMTP_EMAIL_HOST: ${{ env.STAGING_SMTP_EMAIL_HOST }}
+          SMTP_EMAIL_USER: ${{ env.STAGING_SMTP_EMAIL_USER }}
+          SMTP_EMAIL_PASSWORD: ${{ env.STAGING_SMTP_EMAIL_PASSWORD }}
+
+          EXPLORER_API_KEY: ${{ env.STAGING_EXPLORER_API_KEY }}
+
         working-directory: ./cdk
         run: |
           pip install -r requirements.txt

openaq_api/openaq_api/main.py

@@ -8,7 +8,7 @@
 from typing import Any
 
 import orjson
-from fastapi import FastAPI, Request
+from fastapi import FastAPI, Request, Depends
 from fastapi.encoders import jsonable_encoder
 from fastapi.exceptions import RequestValidationError
 from fastapi.middleware.cors import CORSMiddleware
@@ -21,10 +21,9 @@
 
 from openaq_api.db import db_pool
 from openaq_api.middleware import (
+    check_api_key,
     CacheControlMiddleware,
     LoggingMiddleware,
-    PrivatePathsMiddleware,
-    RateLimiterMiddleWare,
 )
 from openaq_api.models.logging import (
     InfrastructureErrorLog,
@@ -33,6 +32,8 @@
     WarnLog,
 )
 
+
+
 # from openaq_api.routers.auth import router as auth_router
 from openaq_api.routers.averages import router as averages_router
 from openaq_api.routers.cities import router as cities_router
@@ -105,8 +106,6 @@ def render(self, content: Any) -> bytes:
         return orjson.dumps(content, default=default)
 
 
-redis_client = None  # initialize for generalize_schema.py
-
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):
@@ -119,7 +118,7 @@ async def lifespan(app: FastAPI):
         app.state.counter += 1
     else:
         app.state.counter = 0
-    app.state.redis_client = redis_client
+
     yield
     if hasattr(app.state, "pool") and not settings.USE_SHARED_POOL:
         logger.debug("Closing connection")
@@ -128,16 +127,21 @@ async def lifespan(app: FastAPI):
         logger.debug("Connection closed")
 
 
+
+
+
 app = FastAPI(
     title="OpenAQ",
     description="OpenAQ API",
     version="2.0.0",
     default_response_class=ORJSONResponse,
+    dependencies=[Depends(check_api_key)],
     docs_url="/docs",
     lifespan=lifespan,
 )
 
 
+app.redis = None
 if settings.RATE_LIMITING is True:
     if settings.RATE_LIMITING:
         logger.debug("Connecting to redis")
@@ -150,25 +154,15 @@ async def lifespan(app: FastAPI):
                 decode_responses=True,
                 socket_timeout=5,
             )
+            # attach to the app so it can be retrieved via the request
+            app.redis = redis_client
+            logger.debug("Redis connected")
+
         except Exception as e:
             logging.error(
                 InfrastructureErrorLog(detail=f"failed to connect to redis: {e}")
             )
-        print(redis_client)
-        logger.debug("Redis connected")
-    if redis_client:
-        app.add_middleware(
-            RateLimiterMiddleWare,
-            redis_client=redis_client,
-            rate_amount_key=settings.RATE_AMOUNT_KEY,
-            rate_time=datetime.timedelta(minutes=settings.RATE_TIME),
-        )
-    else:
-        logger.warning(
-            WarnLog(
-                detail="valid redis client not provided but RATE_LIMITING set to TRUE"
-            )
-        )
+
 
 app.add_middleware(
     CORSMiddleware,
@@ -180,7 +174,6 @@ async def lifespan(app: FastAPI):
 app.add_middleware(CacheControlMiddleware, cachecontrol="public, max-age=900")
 app.add_middleware(LoggingMiddleware)
 app.add_middleware(GZipMiddleware, minimum_size=1000)
-app.add_middleware(PrivatePathsMiddleware)
 
 
 class OpenAQValidationResponseDetail(BaseModel):
@@ -198,31 +191,11 @@ async def openaq_request_validation_exception_handler(
     request: Request, exc: RequestValidationError
 ):
     return ORJSONResponse(status_code=422, content=jsonable_encoder(str(exc)))
-    # return PlainTextResponse(str(exc))
-    # print("\n\n\n\n\n")
-    # print(str(exc))
-    # print("\n\n\n\n\n")
-    # detail = orjson.loads(str(exc))
-    # logger.debug(traceback.format_exc())
-    # logger.info(
-    #     UnprocessableEntityLog(request=request, detail=str(exc)).model_dump_json()
-    # )
-    # detail = OpenAQValidationResponse(detail=detail)
-    # return ORJSONResponse(status_code=422, content=jsonable_encoder(detail))
 
 
 @app.exception_handler(ValidationError)
 async def openaq_exception_handler(request: Request, exc: ValidationError):
     return ORJSONResponse(status_code=422, content=jsonable_encoder(str(exc)))
-    # detail = orjson.loads(exc.model_dump_json())
-    # logger.debug(traceback.format_exc())
-    # logger.error(
-    #     ModelValidationError(
-    #         request=request, detail=exc.jsmodel_dump_jsonon()
-    #     ).model_dump_json()
-    # )
-    # return ORJSONResponse(status_code=422, content=jsonable_encoder(detail))
-    # return ORJSONResponse(status_code=500, content={"message": "internal server error"})
 
 
 @app.get("/ping", include_in_schema=False)