Merge pull request #4948 from grafana/dev

v1.9.17

.github/workflows/expensive-e2e-tests.yml

@@ -14,8 +14,8 @@ jobs:
     strategy:
       matrix:
         grafana_version:
-          - 10.1.7
-          - 10.3.3
+          - 10.3.0
+          - latest
       fail-fast: false
       # Run one version at a time to avoid the issue when SMS notification are bundled together for multiple versions
       # running at the same time (the affected test is in grafana-plugin/e2e-tests/alerts/sms.test.ts)

.github/workflows/linting-and-tests.yml

@@ -35,7 +35,7 @@ jobs:
         uses: ./.github/actions/install-frontend-dependencies
       - name: Build, lint and test frontend
         working-directory: grafana-plugin
-        run: yarn lint && yarn test && yarn build
+        run: yarn lint && yarn type-check && yarn test && yarn build
 
   test-technical-documentation:
     name: "Test technical documentation"
@@ -242,11 +242,8 @@ jobs:
     strategy:
       matrix:
         grafana_version:
-          - 10.1.7
-          - 10.3.3
-          # TODO: fix issues with running e2e tests against Grafana v10.2.x and latest
-          # - 10.2.4
-          # - latest
+          - 10.3.0
+          - latest
       fail-fast: false
     with:
       grafana_version: ${{ matrix.grafana_version }}

docs/make-docs

@@ -6,6 +6,15 @@
 # [Semantic versioning](https://semver.org/) is used to help the reader identify the significance of changes.
 # Changes are relevant to this script and the support docs.mk GNU Make interface.
 #
+# ## 8.1.0 (2024-08-22)
+#
+# ### Added
+#
+# - Additional website mounts for projects that use the website repository.
+#
+#   Mounts are required for `make docs` to work in the website repository or with the website project.
+#   The Makefile is also mounted for convenient development of the procedure that repository.
+#
 # ## 8.0.1 (2024-07-01)
 #
 # ### Fixed
@@ -727,6 +736,9 @@ POSIX_HERESTRING
 
     _repo="$(repo_path website)"
     volumes="--volume=${_repo}/config:/hugo/config:z"
+    volumes="${volumes} --volume=${_repo}/content/guides:/hugo/content/guides:z"
+    volumes="${volumes} --volume=${_repo}/content/whats-new:/hugo/content/whats-new:z"
+    volumes="${volumes} --volume=${_repo}/Makefile:/hugo/Makefile:z"
     volumes="${volumes} --volume=${_repo}/layouts:/hugo/layouts:z"
     volumes="${volumes} --volume=${_repo}/scripts:/hugo/scripts:z"
   fi

engine/apps/grafana_plugin/helpers/gcom.py

@@ -28,7 +28,13 @@ def check_gcom_permission(token_string: str, context) -> GcomToken:
     stack_id = context["stack_id"]
     org_id = context["org_id"]
     grafana_token = context["grafana_token"]
-    organization = Organization.objects.filter(stack_id=stack_id, org_id=org_id).first()
+    organization = Organization.objects_with_deleted.filter(stack_id=stack_id, org_id=org_id).first()
+
+    if organization and organization.deleted_at:
+        # if an organization has been deleted, it should not be allowed to be automatically reactivated
+        # (it should go through a manual request and process)
+        raise InvalidToken
+
     if (
         organization
         and organization.gcom_token == token_string

engine/apps/grafana_plugin/tasks/sync_v2.py

@@ -7,47 +7,52 @@
 from apps.grafana_plugin.helpers.gcom import get_active_instance_ids
 from apps.user_management.models import Organization
 from common.custom_celery_tasks import shared_dedicated_queue_retry_task
-from common.utils import task_lock
 
 logger = get_task_logger(__name__)
 logger.setLevel(logging.DEBUG)
 
 
 SYNC_PERIOD = timezone.timedelta(minutes=4)
+SYNC_BATCH_SIZE = 500
 
 
 @shared_dedicated_queue_retry_task(autoretry_for=(Exception,), retry_backoff=True, max_retries=0)
-def sync_organizations_v2(org_ids=None):
-    lock_id = "sync_organizations_v2"
-    with task_lock(lock_id, "main") as acquired:
-        if acquired:
-            if org_ids:
-                logger.debug(f"Starting with provided {len(org_ids)} org_ids")
-                organization_qs = Organization.objects.filter(id__in=org_ids)
-            else:
-                logger.debug("Starting with all org ids")
-                organization_qs = Organization.objects.all()
-            active_instance_ids, is_cloud_configured = get_active_instance_ids()
-            if is_cloud_configured:
-                if not active_instance_ids:
-                    logger.warning("Did not find any active instances!")
-                    return
-                else:
-                    logger.debug(f"Found {len(active_instance_ids)} active instances")
-                    organization_qs = organization_qs.filter(stack_id__in=active_instance_ids)
-
-            logger.info(f"Syncing {len(organization_qs)} organizations")
-            for idx, org in enumerate(organization_qs):
-                if GrafanaAPIClient.validate_grafana_token_format(org.api_token):
-                    client = GrafanaAPIClient(api_url=org.grafana_url, api_token=org.api_token)
-                    _, status = client.sync()
-                    if status["status_code"] != 200:
-                        logger.error(
-                            f"Failed to request sync stack_slug={org.stack_slug} status_code={status['status_code']} url={status['url']} message={status['message']}"
-                        )
-                    if idx % 1000 == 0:
-                        logger.info(f"{idx + 1} organizations processed")
-                else:
-                    logger.info(f"Skipping stack_slug={org.stack_slug}, api_token format is invalid or not set")
+def start_sync_organizations_v2():
+    organization_qs = Organization.objects.all()
+    active_instance_ids, is_cloud_configured = get_active_instance_ids()
+    if is_cloud_configured:
+        if not active_instance_ids:
+            logger.warning("Did not find any active instances!")
+            return
+        else:
+            logger.debug(f"Found {len(active_instance_ids)} active instances")
+            organization_qs = organization_qs.filter(stack_id__in=active_instance_ids)
+
+    logger.info(f"Found {len(organization_qs)} active organizations")
+    batch = []
+    for org in organization_qs:
+        if GrafanaAPIClient.validate_grafana_token_format(org.api_token):
+            batch.append(org.pk)
+            if len(batch) == SYNC_BATCH_SIZE:
+                sync_organizations_v2.apply_async(
+                    (batch,),
+                )
+                batch = []
         else:
-            logger.info(f"Issuing sync requests already in progress lock_id={lock_id}, check slow outgoing requests")
+            logger.info(f"Skipping stack_slug={org.stack_slug}, api_token format is invalid or not set")
+    if batch:
+        sync_organizations_v2.apply_async(
+            (batch,),
+        )
+
+
+@shared_dedicated_queue_retry_task(autoretry_for=(Exception,), retry_backoff=True, max_retries=0)
+def sync_organizations_v2(org_ids=None):
+    organization_qs = Organization.objects.filter(id__in=org_ids)
+    for org in organization_qs:
+        client = GrafanaAPIClient(api_url=org.grafana_url, api_token=org.api_token)
+        _, status = client.sync()
+        if status["status_code"] != 200:
+            logger.error(
+                f"Failed to request sync org_id={org.pk} stack_slug={org.stack_slug} status_code={status['status_code']} url={status['url']} message={status['message']}"
+            )

engine/apps/grafana_plugin/tests/test_gcom.py

@@ -2,6 +2,7 @@
 
 import pytest
 
+from apps.auth_token.exceptions import InvalidToken
 from apps.grafana_plugin.helpers.gcom import check_gcom_permission
 from apps.user_management.models import Organization
 
@@ -86,7 +87,8 @@ def test_check_gcom_permission_uniqueness_update_fields(make_organization):
 
     # organization does not exist in the first check but it is created before the second check
     with patch(
-        "apps.grafana_plugin.helpers.gcom.Organization.objects.filter", return_value=Organization.objects.none()
+        "apps.grafana_plugin.helpers.gcom.Organization.objects_with_deleted.filter",
+        return_value=Organization.objects.none(),
     ):
         with patch(
             "apps.grafana_plugin.helpers.GcomAPIClient.get_instance_info",
@@ -106,3 +108,34 @@ def test_check_gcom_permission_uniqueness_update_fields(make_organization):
     assert org.cluster_slug == instance_info["clusterSlug"]
     assert org.api_token == fixed_token
     assert org.gcom_token == gcom_token
+
+
+@pytest.mark.django_db
+def test_check_gcom_permission_undelete_org(make_organization):
+    gcom_token = "gcom:test_token"
+    fixed_token = "fixed_token"
+    instance_info = {
+        "id": 324534,
+        "slug": "testinstance",
+        "url": "http://example.com",
+        "orgId": 5671,
+        "orgSlug": "testorg",
+        "orgName": "Test Org",
+        "regionSlug": "us",
+        "clusterSlug": "us-test",
+    }
+    context = {
+        "stack_id": str(instance_info["id"]),
+        "org_id": str(instance_info["orgId"]),
+        "grafana_token": fixed_token,
+    }
+
+    org = make_organization(stack_id=instance_info["id"], org_id=instance_info["orgId"], api_token="broken_token")
+    org.delete()
+
+    with pytest.raises(InvalidToken):
+        check_gcom_permission(gcom_token, context)
+
+    org.refresh_from_db()
+    # org is still deleted
+    assert org.deleted_at

engine/apps/grafana_plugin/tests/test_sync_v2.py

@@ -6,7 +6,7 @@
 from rest_framework.test import APIClient
 
 from apps.api.permissions import LegacyAccessControlRole
-from apps.grafana_plugin.tasks import sync_organizations_v2
+from apps.grafana_plugin.tasks.sync_v2 import start_sync_organizations_v2
 
 
 @pytest.mark.django_db
@@ -57,7 +57,7 @@ def test_invalid_auth(make_organization_and_user_with_plugin_token, make_user_au
 )
 @pytest.mark.django_db
 def test_skip_org_without_api_token(make_organization, api_token, sync_called):
-    organization = make_organization(api_token=api_token)
+    make_organization(api_token=api_token)
 
     with patch(
         "apps.grafana_plugin.helpers.GrafanaAPIClient.sync",
@@ -70,6 +70,9 @@ def test_skip_org_without_api_token(make_organization, api_token, sync_called):
                 "message": "",
             },
         ),
-    ) as mock_sync:
-        sync_organizations_v2(org_ids=[organization.id])
-        assert mock_sync.called == sync_called
+    ):
+        with patch(
+            "apps.grafana_plugin.tasks.sync_v2.sync_organizations_v2.apply_async", return_value=None
+        ) as mock_sync:
+            start_sync_organizations_v2()
+            assert mock_sync.called == sync_called