Define DB config earlier for django-celery-results, fix INSTALLED_APPS

.secrets.baseline

@@ -165,15 +165,15 @@
         "filename": "config/settings_ci.py",
         "hashed_secret": "efacc4001e857f7eba4ae781c2932dedf843865e",
         "is_verified": false,
-        "line_number": 44,
+        "line_number": 43,
         "is_secret": false
       },
       {
         "type": "Secret Keyword",
         "filename": "config/settings_ci.py",
         "hashed_secret": "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3",
         "is_verified": false,
-        "line_number": 77,
+        "line_number": 78,
         "is_secret": false
       }
     ],
@@ -191,7 +191,7 @@
         "filename": "config/settings_local.py",
         "hashed_secret": "7c6a61c68ef8b9b6b061b28c348bc1ed7921cb53",
         "is_verified": false,
-        "line_number": 77,
+        "line_number": 78,
         "is_secret": false
       }
     ],

config/settings.py

@@ -51,9 +51,10 @@
     "django.contrib.sessions",
     "django.contrib.messages",
     "django.contrib.staticfiles",
-    "django_celery_results",
     "django_extensions",
     "django.contrib.postgres",
+    "django_celery_results",
+    "django_celery_beat",
     "psqlextra",
     "rest_framework",
     "django_filters",
@@ -154,6 +155,29 @@
 USE_L10N = True
 USE_TZ = True
 
+# Prod / stage settings need to be defined here
+# So django-celery-results can use the right DB
+# Override in an env-specific settings file if needed
+DATABASES = {
+    "default": {
+        "NAME": get_env("OSIDB_DB_NAME", default="osidb"),
+        "USER": get_env("OSIDB_DB_USER"),
+        "PASSWORD": get_env("OSIDB_DB_PASSWORD"),
+        "HOST": get_env("OSIDB_DB_HOST"),
+        "PORT": get_env("OSIDB_DB_PORT", default="5432"),
+        "ENGINE": "psqlextra.backend",
+        "ATOMIC_REQUESTS": True,  # perform HTTP requests as atomic transactions
+        "OPTIONS": {
+            "sslmode": "require",
+            # prevent libpq from automatically trying to connect to the db via GSSAPI
+            "gssencmode": "disable",
+            # this is a hack due to our inability to set a custom parameter either at
+            # the database or role level in managed databases such as AWS RDS
+            "options": "-c osidb.acl=00000000-0000-0000-0000-000000000000",
+        },
+    }
+}
+
 # Celery application definition
 CELERY_BROKER_URL = "redis://redis:6379/"
 CELERY_RESULT_BACKEND = "django-db"

config/settings_ci.py

@@ -1,5 +1,4 @@
 import ldap
-from django.core.management.utils import get_random_secret_key
 from django_auth_ldap.config import GroupOfNamesType, LDAPSearch
 
 from .settings import *
@@ -70,6 +69,8 @@
     "is_superuser": f"cn={SERVICE_MANAGE_GROUP},ou=users,dc=redhat,dc=com",
 }
 
+# Completely redefine instead of reusing certain parts
+# Since name / user / password / etc. are all different
 DATABASES = {
     "default": {
         "NAME": "osidb",
@@ -89,7 +90,6 @@
     }
 }
 
-
 LOGOUT_REDIRECT_URL = "/"  # this is the name of the url
 LOGIN_REDIRECT_URL = "/"  # this is the name of the url
 

config/settings_local.py

@@ -69,7 +69,8 @@
     "is_superuser": f"cn={SERVICE_MANAGE_GROUP},ou=users,dc=redhat,dc=com",
 }
 
-
+# Completely redefine instead of reusing certain parts
+# Since name / user / password / etc. are all different
 DATABASES = {
     "default": {
         "NAME": "osidb",

config/settings_prod.py

@@ -63,26 +63,6 @@
     "is_superuser": f"cn={SERVICE_MANAGE_GROUP},ou=adhoc,ou=managedgroups,dc=redhat,dc=com",
 }
 
-DATABASES = {
-    "default": {
-        "NAME": get_env("OSIDB_DB_NAME", default="osidb"),
-        "USER": get_env("OSIDB_DB_USER"),
-        "PASSWORD": get_env("OSIDB_DB_PASSWORD"),
-        "HOST": get_env("OSIDB_DB_HOST"),
-        "PORT": get_env("OSIDB_DB_PORT", default="5432"),
-        "ENGINE": "psqlextra.backend",
-        "ATOMIC_REQUESTS": True,  # perform HTTP requests as atomic transactions
-        "OPTIONS": {
-            "sslmode": "require",
-            # prevent libpq from automatically trying to connect to the db via GSSAPI
-            "gssencmode": "disable",
-            # this is a hack due to our inability to set a custom parameter either at
-            # the database or role level in managed databases such as AWS RDS
-            "options": "-c osidb.acl=00000000-0000-0000-0000-000000000000",
-        },
-    }
-}
-
 LOGOUT_REDIRECT_URL = "/"
 LOGIN_REDIRECT_URL = "/"
 

config/settings_shell.py

@@ -25,22 +25,6 @@
 # Minimal group for managing the OSIDB service
 SERVICE_MANAGE_GROUP = "osidb-service-manage"
 
-DATABASES = {
-    "default": {
-        "NAME": get_env("OSIDB_DB_NAME", default="osidb"),
-        "USER": get_env("OSIDB_DB_USER", default="osidb_manage_user"),
-        "PASSWORD": get_env("OSIDB_DB_PASSWORD"),
-        "HOST": get_env("OSIDB_DB_HOST", default="localhost"),
-        "PORT": get_env("OSIDB_DB_PORT", default="5432"),
-        "ENGINE": "psqlextra.backend",
-        "ATOMIC_REQUESTS": True,  # perform HTTP requests as atomic transactions
-        "OPTIONS": {
-            "sslmode": "require",
-            # prevent libpq from automatically trying to connect to the db via GSSAPI
-            "gssencmode": "disable",
-            # this is a hack due to our inability to set a custom parameter either at
-            # the database or role level in managed databases such as AWS RDS
-            "options": "-c osidb.acl=00000000-0000-0000-0000-000000000000",
-        },
-    }
-}
+# Use the same env vars, but provide a default if unset
+DATABASES["default"]["USER"] = get_env("OSIDB_DB_USER", default="osidb_manage_user")
+DATABASES["default"]["HOST"] = get_env("OSIDB_DB_HOST", default="localhost")

config/settings_stage.py

@@ -66,26 +66,6 @@
     "is_superuser": f"cn={SERVICE_MANAGE_GROUP},ou=adhoc,ou=managedgroups,dc=redhat,dc=com",
 }
 
-DATABASES = {
-    "default": {
-        "NAME": get_env("OSIDB_DB_NAME", default="osidb"),
-        "USER": get_env("OSIDB_DB_USER"),
-        "PASSWORD": get_env("OSIDB_DB_PASSWORD"),
-        "HOST": get_env("OSIDB_DB_HOST"),
-        "PORT": get_env("OSIDB_DB_PORT", default="5432"),
-        "ENGINE": "psqlextra.backend",
-        "ATOMIC_REQUESTS": True,  # perform HTTP requests as atomic transactions
-        "OPTIONS": {
-            "sslmode": "require",
-            # prevent libpq from automatically trying to connect to the db via GSSAPI
-            "gssencmode": "disable",
-            # this is a hack due to our inability to set a custom parameter either at
-            # the database or role level in managed databases such as AWS RDS
-            "options": "-c osidb.acl=00000000-0000-0000-0000-000000000000",
-        },
-    }
-}
-
 LOGOUT_REDIRECT_URL = "/"
 LOGIN_REDIRECT_URL = "/"
 

devel-requirements.txt

@@ -25,9 +25,9 @@ cachetools==4.2.4 \
     --hash=sha256:89ea6f1b638d5a73a4f9226be57ac5e4f399d22770b92355f92dcb0f7f001693 \
     --hash=sha256:92971d3cb7d2a97efff7c7bb1657f21a8f5fb309a37530537c71b1774189f2d1
     # via google-auth
-certifi==2022.12.7 \
-    --hash=sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3 \
-    --hash=sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18
+certifi==2023.7.22 \
+    --hash=sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082 \
+    --hash=sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9
     # via
     #   -c requirements.txt
     #   kubernetes

docs/CHANGELOG.md

@@ -10,7 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Implement a new FlawAcknowledgment API (OSIDB-1002)
 - Implement requires_summary in Flaw API (OSIDB-1005)
 - Implement ps_update_stream in Tracker API (OSIDB-1064)
-- Implement daily monitoring email for failed tasks
+- Implement daily monitoring email for failed or retried tasks
 
 ### Changed
 - Change article link validation to be blocking (OSIDB-1060)

requirements.in

@@ -3,6 +3,7 @@ celery[gevent]
 cvss
 django
 django-auth-ldap
+django-celery-beat
 django-celery-results
 django.contrib.postgres
 django-cors-headers

requirements.txt

@@ -48,11 +48,12 @@ celery[gevent]==5.2.7 \
     --hash=sha256:fafbd82934d30f8a004f81e8f7a062e31413a23d444be8ee3326553915958c6d
     # via
     #   -r requirements.in
+    #   django-celery-beat
     #   django-celery-results
     #   flower
-certifi==2022.12.7 \
-    --hash=sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3 \
-    --hash=sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18
+certifi==2023.7.22 \
+    --hash=sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082 \
+    --hash=sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9
     # via requests
 cffi==1.15.0 \
     --hash=sha256:00c878c90cb53ccfaae6b8bc18ad05d2036553e6d9d1d9dbcf323bbe83854ca3 \
@@ -134,6 +135,9 @@ colorama==0.4.4 \
     --hash=sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b \
     --hash=sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2
     # via ansimarkup
+cron-descriptor==1.4.0 \
+    --hash=sha256:b6ff4e3a988d7ca04a4ab150248e9f166fb7a5c828a85090e75bcc25aa93b4dd
+    # via django-celery-beat
 cryptography==39.0.1 \
     --hash=sha256:5aa67414fcdfa22cf052e640cb5ddc461924a045cacf325cd164e65312d99502 \
     --hash=sha256:d1f6198ee6d9148405e49887803907fe8962a23e6c6f83ea7d98f1c0de375695 \
@@ -160,6 +164,7 @@ django==3.2.20 \
     # via
     #   -r requirements.in
     #   django-auth-ldap
+    #   django-celery-beat
     #   django-celery-results
     #   django-cors-headers
     #   django-db
@@ -169,13 +174,18 @@ django==3.2.20 \
     #   django-model-utils
     #   django-polymorphic
     #   django-postgres-extra
+    #   django-timezone-field
     #   djangorestframework
     #   djangorestframework-simplejwt
     #   drf-spectacular
 django-auth-ldap==3.0.0 \
     --hash=sha256:19ee19034f344d9efd07ed88d3187e256ec33ae39d6a47222083b89f7d35c5f6 \
     --hash=sha256:1f2d5c562d9ba9a5e9a64099ae9798e1a63840a11afe4d1c4a9c74121f066eaa
     # via -r requirements.in
+django-celery-beat==2.5.0 \
+    --hash=sha256:ae460faa5ea142fba0875409095d22f6bd7bcc7377889b85e8cab5c0dfb781fe \
+    --hash=sha256:cd0a47f5958402f51ac0c715bc942ae33d7b50b4e48cba91bc3f2712be505df1
+    # via -r requirements.in
 django-celery-results==2.5.1 \
     --hash=sha256:0da4cd5ecc049333e4524a23fcfc3460dfae91aa0a60f1fae4b6b2889c254e01 \
     --hash=sha256:3ecb7147f773f34d0381bac6246337ce4cf88a2ea7b82774ed48e518b67bb8fd
@@ -211,6 +221,10 @@ django-postgres-extra==2.0.3 \
     --hash=sha256:447b24b91146f9ad19f960922c19e5885663cf282100529baeae5152775aa4c0 \
     --hash=sha256:c5e65ec3750b01352112526de1572f3dc0ed4d2022ba324c0521907925d29ac6
     # via -r requirements.in
+django-timezone-field==5.1 \
+    --hash=sha256:16ca9955a4e16064e32168b1a0d1cdb2839679c6cb56856c1f49f506e2ca4281 \
+    --hash=sha256:73fc49519273cd5da1c7f16abc04a4bcad87b00cc02968d0d384c0fecf9a8a86
+    # via django-celery-beat
 django.contrib.postgres==0.0.1 \
     --hash=sha256:4c8b57efd93ed0a37ba6fb457c69993ef3f2a51c560c0dd7b1b3cfb78810b0ff
     # via -r requirements.in
@@ -504,10 +518,16 @@ pyrsistent==0.18.0 \
 python-bugzilla==3.2.0 \
     --hash=sha256:4efc8cfa297837c9e4eab220e197176715bd61ee33cec2c1b0f2790f07860387
     # via -r requirements.in
+python-crontab==3.0.0 \
+    --hash=sha256:6d5ba3c190ec76e4d252989a1644fcb233dbf53fbc8fceeb9febe1657b9fb1d4 \
+    --hash=sha256:79fb7465039ddfd4fb93d072d6ee0d45c1ac8bf1597f0686ea14fd4361dba379
+    # via django-celery-beat
 python-dateutil==2.8.2 \
     --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \
     --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9
-    # via django-postgres-extra
+    # via
+    #   django-postgres-extra
+    #   python-crontab
 python-ldap==3.4.0 \
     --hash=sha256:60464c8fc25e71e0fd40449a24eae482dcd0fb7fcf823e7de627a6525b3e0d12
     # via
@@ -519,6 +539,7 @@ pytz==2021.3 \
     # via
     #   celery
     #   django
+    #   django-timezone-field
     #   flower
 pyyaml==6.0 \
     --hash=sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293 \
@@ -686,6 +707,10 @@ tornado==6.3.2 \
     --hash=sha256:4b927c4f19b71e627b13f3db2324e4ae660527143f9e1f2e2fb404f3a187e2ba \
     --hash=sha256:5b17b1cf5f8354efa3d37c6e28fdfd9c1c1e5122f2cb56dac121ac61baa47cbe
     # via flower
+tzdata==2023.3 \
+    --hash=sha256:11ef1e08e54acb0d4f95bdb1be05da659673de4acbd21bf9c69e94cc5e907a3a \
+    --hash=sha256:7e65763eef3120314099b6939b5546db7adce1e7d6f2e179e3df563c70511eda
+    # via django-celery-beat
 unicodecsv==0.14.1 \
     --hash=sha256:018c08037d48649a0412063ff4eda26eaa81eff1546dbffa51fa5293276ff7fc
     # via djangorestframework-csv