v3.0.2 release (#581)

Co-authored-by: GitHub Actions <[email protected]>

.github/workflows/release-workflow.yml

@@ -420,12 +420,12 @@ jobs:
         with:
           branch: staging/v${{ github.event.inputs.version }}
           title: Version ${{ github.event.inputs.version }} Automated Release PR
-          reviewers: brandold
+          reviewers: ianwow
           delete-branch: true
           body: |
             PR Auto-generated from Github Actions release workflow.
             Review test results here:
-            [1]: https://github.com/brandold/aws-media-insights-engine/actions/runs/${{ github.run_id }}
+            [1]: https://github.com/awslabs/aws-media-insights-engine/actions/runs/${{ github.run_id }}
 
 
 
@@ -449,7 +449,7 @@ jobs:
           branch: release/v${{ github.event.inputs.version }}
           base: development
           title: Merge automated release v${{ github.event.inputs.version }} changes into development
-          reviewers: brandold
+          reviewers: ianwow
 
 
 

.viperlightignore

@@ -17,13 +17,15 @@ docs/assets/images/MIE-architecture-diagram.pptx
 .github/workflows/scheduled-workflow.yml:19
 .github/workflows/scheduled-workflow.yml:20
 .github/workflows/scheduled-workflow.yml:21
+.github/workflows/scheduled-solution-workflow.yml:20
+.github/workflows/scheduled-solution-workflow.yml:21
 README.md:197
 docs/sigv4_get_sample.py:52
 docs/sigv4_get_sample.py:53
 docs/sigv4_post_sample.py:61
 docs/sigv4_post_sample.py:62
-deployment/media-insights-stack.yaml:325
-deployment/media-insights-stack.yaml:336
+deployment/media-insights-stack.yaml:329
+deployment/media-insights-stack.yaml:340
 docs/assets/images/Cognito3-set_password.png
 docs/source/theme/smithy/layout.html
 docs/source/theme/smithy/static/asciinema-player.js
@@ -40,8 +42,8 @@ source/workflowapi/chalicelib/apischema/create_operation_response.json:22
 source/workflowapi/chalicelib/apischema/create_workflow_response.json:70
 test/README.md:41
 test/README.md:42
-test/e2e/conftest.py:33
-test/e2e/conftest.py:34
+test/e2e/conftest.py:36
+test/e2e/conftest.py:37
 test/e2e/run_e2e.sh:31
 test/e2e/run_e2e.sh:37
 test/integ/dataplaneapi/conftest.py:32

CHANGELOG.md

@@ -4,6 +4,39 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+
+## [3.0.2] - 2021-08-18
+
+### New:
+
+* Add new e2e tests to validate CRUD functions in the workflow API for parallel data  and terminologies in Amazon Translate. (#543)
+
+### CI/CD Automation:
+
+* Allow parent stacks to specify both the solution version for the botoconfig string. This, combined with the existing functionality to specify the solution id, means that applications which use MIE can correctly describe themselves to the internal AWS mechanisms that count solution deployments. (#574)
+
+### Fixed:
+
+* When processing long videos, workflows may fail if the video transcript exceeds the max length allowed by Amazon Polly. This behavior was changed so that worklfows will skip the Polly operation and output a warning if the text is too long (#543)
+
+* The API version reported by the /version resource under the data plane and control plane APIs was incorrectly defined as version 2.0.0. It is now correctly defined as version 3.0.0. (#576)
+
+## [3.0.1] - 2021-07-22
+
+### Documentation:
+
+* Sphinx generated API documentation copied to the Implementation Guide (#532)
+
+### Security:
+
+* Validate SSL certificates when making calls to API Gateway (#525)
+* Update Python modules known to have possible security implications (#526)
+* Grant read permission to the dataplane bucket for Translate and Transcribe operators so they can access custom vocabularies, custom terminologies, etc. (#531)
+
+### CI/CD Automation:
+
+* Add a one-click deploy option for the AWS region in eu-west-1 Ireland (#536)
+
 ## [3.0.0] - 2021-06-25
 
 ### New:

README.md

@@ -21,9 +21,9 @@ You can deploy MIE in your AWS account with the following Cloud Formation templa
 
 Region| Launch
 ------|-----
-US East (N. Virginia) | [![Launch in us-east-1](docs/assets/images/launch-stack.png)](https://console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacks/new?stackName=mie&templateURL=https://rodeolabz-us-east-1.s3.amazonaws.com/aws-media-insights-engine/v3.0.1/media-insights-stack.template)
-US West (Oregon) | [![Launch in us-west-2](docs/assets/images/launch-stack.png)](https://console.aws.amazon.com/cloudformation/home?region=us-west-2#/stacks/new?stackName=mie&templateURL=https://rodeolabz-us-west-2.s3.us-west-2.amazonaws.com/aws-media-insights-engine/v3.0.1/media-insights-stack.template)
-EU West (Ireland) | [![Launch in eu-west-1](docs/assets/images/launch-stack.png)](https://console.aws.amazon.com/cloudformation/home?region=eu-west-1#/stacks/new?stackName=mie&templateURL=https://rodeolabz-eu-west-1.s3.eu-west-1.amazonaws.com/aws-media-insights-engine/v3.0.1/media-insights-stack.template)
+US East (N. Virginia) | [![Launch in us-east-1](docs/assets/images/launch-stack.png)](https://console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacks/new?stackName=mie&templateURL=https://rodeolabz-us-east-1.s3.amazonaws.com/aws-media-insights-engine/v3.0.2/media-insights-stack.template)
+US West (Oregon) | [![Launch in us-west-2](docs/assets/images/launch-stack.png)](https://console.aws.amazon.com/cloudformation/home?region=us-west-2#/stacks/new?stackName=mie&templateURL=https://rodeolabz-us-west-2.s3.us-west-2.amazonaws.com/aws-media-insights-engine/v3.0.2/media-insights-stack.template)
+EU West (Ireland) | [![Launch in eu-west-1](docs/assets/images/launch-stack.png)](https://console.aws.amazon.com/cloudformation/home?region=eu-west-1#/stacks/new?stackName=mie&templateURL=https://rodeolabz-eu-west-1.s3.eu-west-1.amazonaws.com/aws-media-insights-engine/v3.0.2/media-insights-stack.template)
 
 The Cloud Formation options for these one-click deploys are described in the [installation parameters](#installation-parameters) section.
 

deployment/media-insights-stack.yaml

@@ -1,5 +1,5 @@
 AWSTemplateFormatVersion: "2010-09-09"
-Description: (SO0163) - aws-media-insights-engine. This is the base AWS CloudFormation template that provisions Media Insights Engine services and provides parameters for user configurable MIE settings.
+Description: (SO0163) - aws-media-insights-engine %%VERSION%%. This is the base AWS CloudFormation template that provisions Media Insights Engine services and provides parameters for user configurable MIE settings.
 
 Parameters:
   DeployTestResources:
@@ -36,6 +36,10 @@ Parameters:
     Description: (Optional) AWS Solution Id used for reporting purposes
     Type: String
     Default: SO0163
+  SolutionVersion:
+    Description: (Optional) AWS Solution version used for reporting purposes
+    Type: String
+    Default: %%VERSION%%
   SendAnonymousData:
     Description: (Optional) Send anonymous data about MIE performance to AWS to help improve the quality of this solution.
     Type: String
@@ -1110,7 +1114,7 @@ Resources:
           WORKFLOW_TABLE_NAME: !Ref WorkflowTable
           SYSTEM_TABLE_NAME: !Ref SystemTable
           DEFAULT_MAX_CONCURRENT_WORKFLOWS: !Ref MaxConcurrentWorkflows
-          botoConfig: !Join ['', ['{"user_agent_extra": "AwsSolution/', !Ref "SolutionId", '/%%VERSION%%"}']]
+          botoConfig: !Join ['', ['{"user_agent_extra": "AwsSolution/', !Ref "SolutionId", '/', !Ref "SolutionVersion", '"}']]
       Handler: app.workflow_scheduler_lambda
       TracingConfig:
           Mode: !If [EnableTraceOnEntryPoints, "Active", "PassThrough"]
@@ -1243,7 +1247,7 @@ Resources:
     Properties:
       Environment:
         Variables:
-          botoConfig: !Join ['', ['{"user_agent_extra": "AwsSolution/', !Ref "SolutionId", '/%%VERSION%%"}']]
+          botoConfig: !Join ['', ['{"user_agent_extra": "AwsSolution/', !Ref "SolutionId", '/', !Ref "SolutionVersion", '"}']]
           STAGE_EXECUTION_QUEUE_URL: !Ref StageExecutionQueue
           STAGE_TABLE_NAME: !Ref StageTable
           OPERATION_TABLE_NAME: !Ref OperationTable
@@ -1292,7 +1296,7 @@ Resources:
     Properties:
       Environment:
         Variables:
-          botoConfig: !Join ['', ['{"user_agent_extra": "AwsSolution/', !Ref "SolutionId", '/%%VERSION%%"}']]
+          botoConfig: !Join ['', ['{"user_agent_extra": "AwsSolution/', !Ref "SolutionId", '/', !Ref "SolutionVersion", '"}']]
           STAGE_EXECUTION_QUEUE_URL: !Ref StageExecutionQueue
           STAGE_TABLE_NAME: !Ref StageTable
           OPERATION_TABLE_NAME: !Ref OperationTable
@@ -1368,7 +1372,7 @@ Resources:
       FunctionName: !Sub "${AWS::StackName}-start-wait-operation"
       Environment:
         Variables:
-          botoConfig: !Join ['', ['{"user_agent_extra": "AwsSolution/', !Ref "SolutionId", '/%%VERSION%%"}']]
+          botoConfig: !Join ['', ['{"user_agent_extra": "AwsSolution/', !Ref "SolutionId", '/', !Ref "SolutionVersion", '"}']]
           STAGE_EXECUTION_QUEUE_URL: !Ref StageExecutionQueue
           STAGE_TABLE_NAME: !Ref StageTable
           OPERATION_TABLE_NAME: !Ref OperationTable
@@ -1411,7 +1415,7 @@ Resources:
       FunctionName: !Sub "${AWS::StackName}-check-wait-operation"
       Environment:
         Variables:
-          botoConfig: !Join ['', ['{"user_agent_extra": "AwsSolution/', !Ref "SolutionId", '/%%VERSION%%"}']]
+          botoConfig: !Join ['', ['{"user_agent_extra": "AwsSolution/', !Ref "SolutionId", '/', !Ref "SolutionVersion", '"}']]
           STAGE_EXECUTION_QUEUE_URL: !Ref StageExecutionQueue
           STAGE_TABLE_NAME: !Ref StageTable
           OPERATION_TABLE_NAME: !Ref OperationTable
@@ -1456,7 +1460,7 @@ Resources:
       Environment:
         Variables:
           TOPIC_ARN: !Ref WorkflowExecutionEventTopic
-          botoConfig: !Join ['', ['{"user_agent_extra": "AwsSolution/', !Ref "SolutionId", '/%%VERSION%%"}']]
+          botoConfig: !Join ['', ['{"user_agent_extra": "AwsSolution/', !Ref "SolutionId", '/', !Ref "SolutionVersion", '"}']]
       Code:
         S3Bucket: !Join ["-", [!FindInMap ["SourceCode", "General", "RegionalS3Bucket"], Ref: "AWS::Region"]]
         S3Key:
@@ -1501,7 +1505,7 @@ Resources:
           ],
         ]
       Parameters:
-        botoConfig: !Join ['', ['{"user_agent_extra": "AwsSolution/', !Ref "SolutionId", '/%%VERSION%%"}']]
+        botoConfig: !Join ['', ['{"user_agent_extra": "AwsSolution/', !Ref "SolutionId", '/', !Ref "SolutionVersion", '"}']]
         DataplaneTableName: !Ref DataplaneTable
         ExternalBucketArn: !Ref ExternalBucketArn
         DataplaneBucketName: !Ref Dataplane
@@ -1542,7 +1546,7 @@ Resources:
             - DataplaneTable
             - StreamArn
         DynamoTableArn: !GetAtt DataplaneTable.Arn
-        botoConfig: !Join ['', ['{"user_agent_extra": "AwsSolution/', !Ref "SolutionId", '/%%VERSION%%"}']]
+        botoConfig: !Join ['', ['{"user_agent_extra": "AwsSolution/', !Ref "SolutionId", '/', !Ref "SolutionVersion", '"}']]
 
   # Workflow creation and execution API
   MediaInsightsWorkflowApi:
@@ -1560,7 +1564,7 @@ Resources:
           ],
         ]
       Parameters:
-        botoConfig: !Join ['', ['{"user_agent_extra": "AwsSolution/', !Ref "SolutionId", '/%%VERSION%%"}']]
+        botoConfig: !Join ['', ['{"user_agent_extra": "AwsSolution/', !Ref "SolutionId", '/', !Ref "SolutionVersion", '"}']]
         ShortUUID: !GetAtt GetShortUUID.Data
         StageExecutionQueueUrl: !Ref StageExecutionQueue
         StageExecutionRole: !GetAtt StepFunctionRole.Arn
@@ -1637,7 +1641,7 @@ Resources:
         MediaInsightsEnginePython36Layer: !Ref MediaInsightsEnginePython36Layer
         StartWaitOperationLambda: !GetAtt StartWaitOperationLambda.Arn
         CheckWaitOperationLambda: !GetAtt CheckWaitOperationLambda.Arn
-        Boto3UserAgent: !Join ['', ['{"user_agent_extra": "AwsSolution/', !Ref "SolutionId", '/%%VERSION%%"}']]
+        Boto3UserAgent: !Join ['', ['{"user_agent_extra": "AwsSolution/', !Ref "SolutionId", '/', !Ref "SolutionVersion", '"}']]
 
   TestResources:
     Condition: DeployTestResourcesCondition

source/dataplaneapi/app.py

@@ -52,7 +52,7 @@ def is_aws():
 
 app_name = 'dataplaneapi'
 app = Chalice(app_name=app_name)
-api_version = "1.0.0"
+api_version = "3.0.0"
 framework_version = os.environ['FRAMEWORK_VERSION']
 
 # DDB resources
@@ -208,7 +208,7 @@ def version():
 
     .. code-block:: python
 
-        {"ApiVersion": "vx.x.x", "FrameworkVersion": "vx.x.x"}
+        {"ApiVersion": "x.x.x", "FrameworkVersion": "vx.x.x"}
     """
     versions = {"ApiVersion": api_version, "FrameworkVersion": framework_version}
     return versions

source/operators/captions/webcaptions.py

@@ -3,6 +3,7 @@
 
 import json
 import boto3
+from botocore.client import ClientError
 import urllib3
 import math
 import os
@@ -827,6 +828,7 @@ def start_polly_webcaptions (event, context):
 
         if language_code == "not supported":
             caption["PollyStatus"] = "not supported"
+            caption["PollyMessage"] = "WARNING: Language code not supported by the Polly service"
         else:
 
             try:
@@ -869,7 +871,14 @@ def start_polly_webcaptions (event, context):
                     TextType='text',
                     VoiceId=voice_id
                 )
-
+            except ClientError as e:
+                # Ignore and skip Polly if we get the TextLengthExceededException, bubble up
+                # other exceptions.
+                if e.response['Error']['Code'] == 'TextLengthExceededException':
+                    caption["PollyMessage"] = "WARNING: Polly.Client.exceptions.TextLengthExceededException"
+                    caption["PollyStatus"] = "not supported"
+                else:
+                    raise
             except Exception as e:
                 operator_object.update_workflow_status("Error")
                 operator_object.add_workflow_metadata(PollyCollectionError="Unable to get response from polly: {e}".format(e=str(e)))

source/workflowapi/app.py

@@ -35,7 +35,7 @@
 API_STAGE = "dev"
 app = Chalice(app_name=APP_NAME)
 app.debug = True
-API_VERSION = "2.0.0"
+API_VERSION = "3.0.0"
 FRAMEWORK_VERSION = os.environ['FRAMEWORK_VERSION']
 
 
@@ -154,7 +154,7 @@ def version():
 
     .. code-block:: python
 
-        {"ApiVersion": "vx.x.x", "FrameworkVersion": "vx.x.x"}
+        {"ApiVersion": "x.x.x", "FrameworkVersion": "vx.x.x"}
     """
     versions = {"ApiVersion": API_VERSION, "FrameworkVersion": FRAMEWORK_VERSION}
     return versions
@@ -2773,13 +2773,11 @@ def get_parallel_data():
     print('get_parallel_data request: '+app.current_request.raw_body.decode())
     translate_client = boto3.client('translate', region_name=os.environ['AWS_REGION'])
     parallel_data_name = json.loads(app.current_request.raw_body.decode())['parallel_data_name']
-    response = translate_client.get_parallel_data(Name=parallel_data_name, parallel_dataDataFormat='CSV')
-    # Remove response metadata since we don't need it
-    if 'RespnseMetadata' in response:
-        del response['ResponseMetadata']
+    response = translate_client.get_parallel_data(Name=parallel_data_name)
+
     # Convert time field to a format that is JSON serializable
-    response['TerminologyProperties']['CreatedAt'] = response['TerminologyProperties']['CreatedAt'].isoformat()
-    response['TerminologyProperties']['LastUpdatedAt'] = response['TerminologyProperties']['LastUpdatedAt'].isoformat()
+    response['ParallelDataProperties']['CreatedAt'] = response['ParallelDataProperties']['CreatedAt'].isoformat()
+    response['ParallelDataProperties']['LastUpdatedAt'] = response['ParallelDataProperties']['LastUpdatedAt'].isoformat()
     return response
 
 
@@ -2797,7 +2795,7 @@ def download_parallel_data():
 
 
     Returns:
-        A string contining the CSV formatted Amazon Transcribe parallel_data
+        A pre-signed url for the the CSV formatted Amazon Transcribe parallel_data
 
         .. code-block:: python
 
@@ -2813,7 +2811,7 @@ def download_parallel_data():
     print('download_parallel_data request: '+app.current_request.raw_body.decode())
     translate_client = boto3.client('translate', region_name=os.environ['AWS_REGION'])
     parallel_data_name = json.loads(app.current_request.raw_body.decode())['parallel_data_name']
-    url = translate_client.get_parallel_data(Name=parallel_data_name, ParallelDataFormat='CSV')['ParallelDataLocation']['Location']
+    url = translate_client.get_parallel_data(Name=parallel_data_name)['DataLocation']['Location']
     import urllib.request
     parallel_data_csv = urllib.request.urlopen(url).read().decode("utf-8")
     return {"parallel_data_csv": parallel_data_csv}
@@ -2888,7 +2886,7 @@ def create_parallel_data():
 
         {
             'parallel_data_name'='string',
-            'parallel_data_csv='string'
+            'parallel_data_s3uri='string'
         }
 
 
@@ -2904,14 +2902,12 @@ def create_parallel_data():
     print('create_parallel_data request: '+app.current_request.raw_body.decode())
     translate_client = boto3.client('translate', region_name=os.environ['AWS_REGION'])
     parallel_data_name = json.loads(app.current_request.raw_body.decode())['parallel_data_name']
-    parallel_data_csv = json.loads(app.current_request.raw_body.decode())['parallel_data_csv']
-    response = translate_client.import_parallel_data(
+    parallel_data_s3uri = json.loads(app.current_request.raw_body.decode())['parallel_data_s3uri']
+    response = translate_client.create_parallel_data(
         Name=parallel_data_name,
-        MergeStrategy='OVERWRITE',
-        ParallelData={'File': parallel_data_csv, 'Format':'CSV'}
+        ParallelDataConfig={'S3Uri': parallel_data_s3uri, 'Format':'CSV'}
     )
-    response['ParallelDataProperties']['CreatedAt'] = response['ParallelDataProperties']['CreatedAt'].isoformat()
-    response['ParallelDataProperties']['LastUpdatedAt'] = response['ParallelDataProperties']['LastUpdatedAt'].isoformat()
+
     return response
 
 # ================================================================================================

source/workflowapi/external_resources.json

@@ -207,10 +207,12 @@
                 {
                   "Effect": "Allow",
                   "Action": [
+                    "translate:CreateTerminology",
                     "translate:DeleteTerminology",
                     "translate:GetTerminology",
                     "translate:ImportTerminology",
                     "translate:ListTerminologies",
+                    "translate:CreateParallelData",
                     "translate:DeleteParallelData",
                     "translate:GetParallelData",
                     "translate:ImportParallelData",