Merge pull request #25 from AllenInstitute/feature/update-sfn-state-b…

…atch-retry-config feature/update sfn state batch retry config
AllenInstitute · Dec 6, 2024 · ab5ad1d · ab5ad1d
2 parents 1dc2598 + de08b88
commit ab5ad1d
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 238 deletions.
diff --git a/.github/actions/bump-version-tag/action.yml b/.github/actions/bump-version-tag/action.yml
diff --git a/.github/workflows/bump_version.yml b/.github/workflows/bump_version.yml
diff --git a/src/aibs_informatics_cdk_lib/constructs_/sfn/states/batch.py b/src/aibs_informatics_cdk_lib/constructs_/sfn/states/batch.py
@@ -133,8 +133,10 @@ def register_job_definition(
                 "Retry": [
                     {
                         "ErrorEquals": ["Batch.BatchException"],
-                        "IntervalSeconds": 1,
-                        "MaxAttempts": 3,
+                        # Interval at attempt n = IntervalSeconds x BackoffRate ^(n-1)
+                        # Total time from first try: 3 + 6 + 12 + 24 = 45 seconds
+                        "IntervalSeconds": 3,
+                        "MaxAttempts": 5,
                         "BackoffRate": 2.0,
                     },
                 ],
@@ -208,8 +210,10 @@ def submit_job(
                 "Retry": [
                     {
                         "ErrorEquals": ["Batch.BatchException"],
-                        "IntervalSeconds": 1,
-                        "MaxAttempts": 3,
+                        # Interval at attempt n = IntervalSeconds x BackoffRate ^(n-1)
+                        # Total time from first try: 3 + 6 + 12 + 24 = 45 seconds
+                        "IntervalSeconds": 3,
+                        "MaxAttempts": 5,
                         "BackoffRate": 2.0,
                     },
                 ],
@@ -248,8 +252,10 @@ def deregister_job_definition(
                 "Retry": [
                     {
                         "ErrorEquals": ["Batch.BatchException"],
-                        "IntervalSeconds": 1,
-                        "MaxAttempts": 3,
+                        # Interval at attempt n = IntervalSeconds x BackoffRate ^(n-1)
+                        # Total time from first try: 3 + 6 + 12 + 24 = 45 seconds
+                        "IntervalSeconds": 3,
+                        "MaxAttempts": 5,
                         "BackoffRate": 2.0,
                     },
                 ],