From 62a7067570a0ebb1e3f9dea0bb6e65f8ffef6bb7 Mon Sep 17 00:00:00 2001
From: Roger Plotz <115798183+Roger-Ethyca@users.noreply.github.com>
Date: Wed, 4 Dec 2024 15:43:43 -0500
Subject: [PATCH 01/13] empty commit to start release 2.51.0


From cd95a92e4c34bc3c5a66a7f7b5b2ef2d209437ab Mon Sep 17 00:00:00 2001
From: Andres Torres <andres.torres.marroquin@gmail.com>
Date: Thu, 5 Dec 2024 08:25:16 -0600
Subject: [PATCH 02/13] HJ-97 - Missing QA from HJ-97 (#5559)

Co-authored-by: Adam Sachs <adam@ethyca.com>
---
 src/fides/api/api/v1/endpoints/system.py | 33 ++++++++++++++----------
 1 file changed, 19 insertions(+), 14 deletions(-)

diff --git a/src/fides/api/api/v1/endpoints/system.py b/src/fides/api/api/v1/endpoints/system.py
index 0171265eda..266f23e1c5 100644
--- a/src/fides/api/api/v1/endpoints/system.py
+++ b/src/fides/api/api/v1/endpoints/system.py
@@ -19,7 +19,11 @@
 
 from fides.api.api import deps
 from fides.api.api.v1.endpoints.saas_config_endpoints import instantiate_connection
-from fides.api.db.crud import get_resource, get_resource_with_custom_fields
+from fides.api.db.crud import (
+    get_resource,
+    get_resource_with_custom_fields,
+    list_resource,
+)
 from fides.api.db.ctl_session import get_async_db
 from fides.api.db.system import (
     create_system,
@@ -396,6 +400,20 @@ async def ls(  # pylint: disable=invalid-name
     Otherwise all Systems will be returned (this may be a slow operation if there are many systems,
     so using the pagination parameters is recommended).
     """
+    if not (
+        size
+        or page
+        or search
+        or data_uses
+        or data_categories
+        or data_subjects
+        or dnd_relevant
+        or show_hidden
+    ):
+        # if no advanced parameters are passed, we return a very basic list of all System resources
+        # to maintain backward compatibility of the original API, which backs some important client usages, e.g. the fides CLI
+
+        return await list_resource(System, db)
 
     query = select(System)
 
@@ -447,19 +465,6 @@ async def ls(  # pylint: disable=invalid-name
     # Add a distinct so we only get one row per system
     duplicates_removed = filtered_query.distinct(System.id)
 
-    if not (
-        size
-        or page
-        or search
-        or data_uses
-        or data_categories
-        or data_subjects
-        or dnd_relevant
-        or show_hidden
-    ):
-        result = await db.execute(duplicates_removed)
-        return result.scalars().all()
-
     return await async_paginate(db, duplicates_removed, pagination_params)
 
 

From b46a426115d4b5795bd9a2e11614d064b41bb832 Mon Sep 17 00:00:00 2001
From: Adrian Galvan <adrian@ethyca.com>
Date: Thu, 5 Dec 2024 09:43:01 -0800
Subject: [PATCH 03/13] Disable test datasets button for non-database
 integrations (#5560)

---
 .../forms/ConnectorParametersForm.tsx                | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/clients/admin-ui/src/features/datastore-connections/system_portal_config/forms/ConnectorParametersForm.tsx b/clients/admin-ui/src/features/datastore-connections/system_portal_config/forms/ConnectorParametersForm.tsx
index dae5bce4b1..92c881bd41 100644
--- a/clients/admin-ui/src/features/datastore-connections/system_portal_config/forms/ConnectorParametersForm.tsx
+++ b/clients/admin-ui/src/features/datastore-connections/system_portal_config/forms/ConnectorParametersForm.tsx
@@ -522,11 +522,13 @@ export const ConnectorParametersForm = ({
                     {testButtonLabel}
                   </Button>
                 ) : null}
-                {isPlusEnabled && !_.isEmpty(initialDatasets) && (
-                  <Button onClick={() => onTestDatasetsClick()}>
-                    Test datasets
-                  </Button>
-                )}
+                {isPlusEnabled &&
+                  SystemType.DATABASE === connectionOption.type &&
+                  !_.isEmpty(initialDatasets) && (
+                    <Button onClick={() => onTestDatasetsClick()}>
+                      Test datasets
+                    </Button>
+                  )}
                 {connectionOption.authorization_required && !authorized ? (
                   <Button
                     loading={isAuthorizing}

From d3ddb71dba475bf41ed27e7209fb037bdb23a86c Mon Sep 17 00:00:00 2001
From: Andres Torres <andres.torres.marroquin@gmail.com>
Date: Fri, 6 Dec 2024 10:08:17 -0600
Subject: [PATCH 04/13] Hj-298 - Fix
 tests/ctl/core/test_api.py::TestSystemList::test_vendor_deleted_systems test
 (#5570)

---
 tests/ctl/core/test_api.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tests/ctl/core/test_api.py b/tests/ctl/core/test_api.py
index bfca14dd84..1a47e32b80 100644
--- a/tests/ctl/core/test_api.py
+++ b/tests/ctl/core/test_api.py
@@ -1669,9 +1669,9 @@ def test_list_with_pagination_and_multiple_filters_2(
         "vendor_deleted_date, expected_systems_count, show_deleted",
         [
             (datetime.now() - timedelta(days=1), 1, True),
-            (datetime.now() - timedelta(days=1), 0, None),
-            (datetime.now() + timedelta(days=1), 1, None),
-            (None, 1, None),
+            (datetime.now() - timedelta(days=1), 0, False),
+            (datetime.now() + timedelta(days=1), 1, False),
+            (None, 1, False),
         ],
     )
     def test_vendor_deleted_systems(
@@ -1691,13 +1691,13 @@ def test_vendor_deleted_systems(
             url=test_config.cli.server_url,
             headers=test_config.user.auth_header,
             resource_type="system",
-            query_params={"show_deleted": True} if show_deleted else {},
+            query_params={"show_deleted": show_deleted, "size": 50},
         )
 
         assert result.status_code == 200
         result_json = result.json()
 
-        assert len(result_json) == expected_systems_count
+        assert len(result_json["items"]) == expected_systems_count
 
 
 @pytest.mark.unit

From 8612a451488e5ab21630e99f9c1958e6316681b8 Mon Sep 17 00:00:00 2001
From: Adrian Galvan <adrian@ethyca.com>
Date: Fri, 6 Dec 2024 09:31:13 -0800
Subject: [PATCH 05/13] Escape datetime and ObjectId values in test privacy
 results (#5567)

---
 .../v1/endpoints/privacy_request_endpoints.py |  9 +++-
 .../test_privacy_request_endpoints.py         | 51 +++++++++++++++++--
 2 files changed, 54 insertions(+), 6 deletions(-)

diff --git a/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py b/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py
index c8dc17a273..84cb453a63 100644
--- a/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py
+++ b/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py
@@ -2,6 +2,7 @@
 
 import csv
 import io
+import json
 from collections import defaultdict
 from datetime import datetime
 from typing import (
@@ -148,6 +149,7 @@
 from fides.api.util.enums import ColumnSort
 from fides.api.util.fuzzy_search_utils import get_decrypted_identities_automaton
 from fides.api.util.logger import Pii
+from fides.api.util.storage_util import storage_json_encoder
 from fides.common.api.scope_registry import (
     PRIVACY_REQUEST_CALLBACK_RESUME,
     PRIVACY_REQUEST_CREATE,
@@ -2657,8 +2659,13 @@ def get_test_privacy_request_results(
         )
         privacy_request.save(db=db)
 
+    # Escape datetime and ObjectId values
+    raw_data = privacy_request.get_raw_access_results()
+    escaped_json = json.dumps(raw_data, indent=2, default=storage_json_encoder)
+    escaped_data = json.loads(escaped_json)
+
     return {
         "privacy_request_id": privacy_request.id,
         "status": privacy_request.status,
-        "results": privacy_request.get_raw_access_results(),
+        "results": escaped_data,
     }
diff --git a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py
index 5ddbcd8f60..aaf64d4e7b 100644
--- a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py
+++ b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py
@@ -8364,8 +8364,7 @@ def test_get_access_results_contributor_but_disabled(
         assert response.status_code == 403
 
 
-@pytest.mark.integration_external
-@pytest.mark.integration_postgres
+@pytest.mark.integration
 class TestPrivacyRequestFilteredResults:
     @pytest.fixture(scope="function")
     def default_access_policy(self, db) -> None:
@@ -8427,15 +8426,18 @@ def test_filtered_results_with_roles(
         )
         assert response.status_code == expected_status
 
+    @pytest.mark.integration_postgres
     @pytest.mark.usefixtures("default_access_policy", "postgres_integration_db")
-    def test_filtered_results(
+    def test_filtered_results_postgres(
         self,
         connection_config,
-        dataset_config,
+        postgres_example_test_dataset_config,
         api_client: TestClient,
         generate_auth_header,
     ) -> None:
-        dataset_url = get_connection_dataset_url(connection_config, dataset_config)
+        dataset_url = get_connection_dataset_url(
+            connection_config, postgres_example_test_dataset_config
+        )
         auth_header = generate_auth_header(scopes=[DATASET_TEST])
         response = api_client.post(
             dataset_url + "/test",
@@ -8459,3 +8461,42 @@ def test_filtered_results(
             "status",
             "results",
         }
+
+    @pytest.mark.integration_mongo
+    @pytest.mark.usefixtures("default_access_policy")
+    def test_filtered_results_mongo(
+        self,
+        mongo_connection_config,
+        mongo_dataset_config,
+        api_client: TestClient,
+        generate_auth_header,
+    ) -> None:
+        dataset_url = get_connection_dataset_url(
+            mongo_connection_config, mongo_dataset_config
+        )
+        auth_header = generate_auth_header(scopes=[DATASET_TEST])
+        response = api_client.post(
+            dataset_url + "/test",
+            headers=auth_header,
+            json={
+                "email": "employee-1@example.com",
+                "postgres_example_test_dataset:customer:id": 1,
+            },
+        )
+        assert response.status_code == HTTP_200_OK
+
+        privacy_request_id = response.json()["privacy_request_id"]
+        url = V1_URL_PREFIX + PRIVACY_REQUEST_FILTERED_RESULTS.format(
+            privacy_request_id=privacy_request_id
+        )
+        auth_header = generate_auth_header(scopes=[PRIVACY_REQUEST_READ_ACCESS_RESULTS])
+        response = api_client.get(
+            url,
+            headers=auth_header,
+        )
+        assert response.status_code == HTTP_200_OK
+        assert set(response.json().keys()) == {
+            "privacy_request_id",
+            "status",
+            "results",
+        }

From c697a671205792d75f79d5fe7365cbfe8aa5726f Mon Sep 17 00:00:00 2001
From: jpople <jeremy@ethyca.com>
Date: Fri, 6 Dec 2024 13:03:36 -0600
Subject: [PATCH 06/13] Fix Fides brand link position on small screens (#5572)

---
 CHANGELOG.md                                  |  1 +
 .../privacy-center/components/BrandLink.tsx   | 40 ++++++++++++------
 clients/privacy-center/components/Layout.tsx  | 15 +------
 .../consent/ConfigDrivenConsent.tsx           |  2 +
 .../notice-driven/NoticeDrivenConsent.tsx     |  6 ++-
 clients/privacy-center/pages/index.tsx        | 42 ++++++++++++-------
 6 files changed, 65 insertions(+), 41 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index a962f4b20e..56dcb16d98 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -42,6 +42,7 @@ The types of changes are:
 - Updating dataset PUT to allow deleting all datasets [#5524](https://github.com/ethyca/fides/pull/5524)
 - Adds support for fides_key generation when parent_key is provided in Taxonomy create endpoints [#5542](https://github.com/ethyca/fides/pull/5542)
 - An integration will no longer re-enable after saving the connection form [#5555](https://github.com/ethyca/fides/pull/5555)
+- Fixed positioning of Fides brand link in privacy center [#5572](https://github.com/ethyca/fides/pull/5572)
 
 ### Removed
 - Removed unnecessary debug logging from the load_file config helper [#5544](https://github.com/ethyca/fides/pull/5544)
diff --git a/clients/privacy-center/components/BrandLink.tsx b/clients/privacy-center/components/BrandLink.tsx
index c069ed57f6..15976ceb06 100644
--- a/clients/privacy-center/components/BrandLink.tsx
+++ b/clients/privacy-center/components/BrandLink.tsx
@@ -1,16 +1,32 @@
 import { EthycaLogo, Link, LinkProps } from "fidesui";
 
-const BrandLink = (props: LinkProps) => (
-  <Link
-    fontSize="8px"
-    color="gray.400"
-    isExternal
-    textDecoration="none"
-    _hover={{ textDecoration: "none" }}
-    {...props}
-  >
-    Powered by <EthycaLogo color="minos.500" h="20px" w="31px" />
-  </Link>
-);
+import { useSettings } from "~/features/common/settings.slice";
+
+const BrandLink = ({
+  position = "absolute",
+  right = 6,
+  ...props
+}: LinkProps) => {
+  const { SHOW_BRAND_LINK } = useSettings();
+
+  if (!SHOW_BRAND_LINK) {
+    return null;
+  }
+
+  return (
+    <Link
+      fontSize="8px"
+      color="gray.400"
+      isExternal
+      position={position}
+      right={right}
+      textDecoration="none"
+      _hover={{ textDecoration: "none" }}
+      {...props}
+    >
+      Powered by <EthycaLogo color="minos.500" h="20px" w="31px" />
+    </Link>
+  );
+};
 
 export default BrandLink;
diff --git a/clients/privacy-center/components/Layout.tsx b/clients/privacy-center/components/Layout.tsx
index 9565fc7f20..e1e73a498b 100644
--- a/clients/privacy-center/components/Layout.tsx
+++ b/clients/privacy-center/components/Layout.tsx
@@ -2,16 +2,13 @@ import { Flex } from "fidesui";
 import Head from "next/head";
 import React, { ReactNode } from "react";
 
-import BrandLink from "~/components/BrandLink";
 import Logo from "~/components/Logo";
 import { useConfig } from "~/features/common/config.slice";
-import { useSettings } from "~/features/common/settings.slice";
 import { useStyles } from "~/features/common/styles.slice";
 
 const Layout = ({ children }: { children: ReactNode }) => {
   const config = useConfig();
   const styles = useStyles();
-  const { SHOW_BRAND_LINK } = useSettings();
   return (
     <>
       <Head>
@@ -32,17 +29,7 @@ const Layout = ({ children }: { children: ReactNode }) => {
           <Logo src={config.logo_path ?? ""} href={config.logo_url ?? ""} />
         </Flex>
       </header>
-      <div>
-        {children}
-        {SHOW_BRAND_LINK && (
-          <BrandLink
-            position="absolute"
-            bottom={16}
-            right={6}
-            href="https://fid.es/powered"
-          />
-        )}
-      </div>
+      <div>{children}</div>
     </>
   );
 };
diff --git a/clients/privacy-center/components/consent/ConfigDrivenConsent.tsx b/clients/privacy-center/components/consent/ConfigDrivenConsent.tsx
index f59e7bb4e3..3808605783 100644
--- a/clients/privacy-center/components/consent/ConfigDrivenConsent.tsx
+++ b/clients/privacy-center/components/consent/ConfigDrivenConsent.tsx
@@ -14,6 +14,7 @@ import { useAppDispatch, useAppSelector } from "~/app/hooks";
 import { inspectForBrowserIdentities } from "~/common/browser-identities";
 import { useLocalStorage } from "~/common/hooks";
 import { ErrorToastOptions, SuccessToastOptions } from "~/common/toast-options";
+import BrandLink from "~/components/BrandLink";
 import { useConfig } from "~/features/common/config.slice";
 import {
   changeConsent,
@@ -209,6 +210,7 @@ const ConfigDrivenConsent = ({
         cancelLabel="Cancel"
         saveLabel="Save"
       />
+      <BrandLink bottom={0} />
     </Stack>
   );
 };
diff --git a/clients/privacy-center/components/consent/notice-driven/NoticeDrivenConsent.tsx b/clients/privacy-center/components/consent/notice-driven/NoticeDrivenConsent.tsx
index d8765f587c..bd24228af2 100644
--- a/clients/privacy-center/components/consent/notice-driven/NoticeDrivenConsent.tsx
+++ b/clients/privacy-center/components/consent/notice-driven/NoticeDrivenConsent.tsx
@@ -25,6 +25,7 @@ import { inspectForBrowserIdentities } from "~/common/browser-identities";
 import { useLocalStorage } from "~/common/hooks";
 import useI18n from "~/common/hooks/useI18n";
 import { ErrorToastOptions, SuccessToastOptions } from "~/common/toast-options";
+import BrandLink from "~/components/BrandLink";
 import { useProperty } from "~/features/common/property.slice";
 import {
   selectPrivacyExperience,
@@ -400,7 +401,10 @@ const NoticeDrivenConsent = ({ base64Cookie }: { base64Cookie: boolean }) => {
         onCancel={handleCancel}
         justifyContent="center"
       />
-      <PrivacyPolicyLink alignSelf="center" experience={experience} />
+      <Stack flexDirection="row" alignItems="center">
+        <PrivacyPolicyLink experience={experience} />
+        <BrandLink />
+      </Stack>
     </Box>
   );
 };
diff --git a/clients/privacy-center/pages/index.tsx b/clients/privacy-center/pages/index.tsx
index 287a0b8f91..e7c21b22fa 100644
--- a/clients/privacy-center/pages/index.tsx
+++ b/clients/privacy-center/pages/index.tsx
@@ -13,6 +13,7 @@ import React, { useEffect, useState } from "react";
 
 import { useAppDispatch, useAppSelector } from "~/app/hooks";
 import { ConfigErrorToastOptions } from "~/common/toast-options";
+import BrandLink from "~/components/BrandLink";
 import ConsentCard from "~/components/consent/ConsentCard";
 import {
   ConsentRequestModal,
@@ -25,7 +26,10 @@ import {
 } from "~/components/modals/privacy-request-modal/PrivacyRequestModal";
 import PrivacyCard from "~/components/PrivacyCard";
 import { useConfig } from "~/features/common/config.slice";
-import { selectIsNoticeDriven } from "~/features/common/settings.slice";
+import {
+  selectIsNoticeDriven,
+  useSettings,
+} from "~/features/common/settings.slice";
 import {
   clearLocation,
   selectPrivacyExperience,
@@ -68,6 +72,10 @@ const Home: NextPage = () => {
   let isConsentModalOpen = isConsentModalOpenConst;
   const getIdVerificationConfigQuery = useGetIdVerificationConfigQuery();
 
+  const { SHOW_BRAND_LINK } = useSettings();
+  const showPrivacyPolicyLink =
+    !!config.privacy_policy_url && !!config.privacy_policy_url_text;
+
   // Subscribe to experiences just to see if there are any notices.
   // The subscription automatically handles skipping if overlay is not enabled
   useSubscribeToPrivacyExperienceQuery();
@@ -214,19 +222,25 @@ const Home: NextPage = () => {
             {paragraph}
           </Text>
         ))}
-        {config.privacy_policy_url && config.privacy_policy_url_text ? (
-          <Link
-            fontSize={["small", "medium"]}
-            fontWeight="medium"
-            textAlign="center"
-            textDecoration="underline"
-            color="gray.600"
-            href={config.privacy_policy_url}
-            isExternal
-          >
-            {config.privacy_policy_url_text}
-          </Link>
-        ) : null}
+
+        {(SHOW_BRAND_LINK || showPrivacyPolicyLink) && (
+          <Stack flexDirection="row">
+            {showPrivacyPolicyLink && (
+              <Link
+                fontSize={["small", "medium"]}
+                fontWeight="medium"
+                textAlign="center"
+                textDecoration="underline"
+                color="gray.600"
+                href={config.privacy_policy_url!}
+                isExternal
+              >
+                {config.privacy_policy_url_text}
+              </Link>
+            )}
+            <BrandLink />
+          </Stack>
+        )}
       </Stack>
       <PrivacyRequestModal
         isOpen={isPrivacyModalOpen}

From df1e0d70781c0975cf46558145bc2ceb68b72a4a Mon Sep 17 00:00:00 2001
From: Catherine Smith <eastandwestwind@gmail.com>
Date: Tue, 10 Dec 2024 20:39:47 +0100
Subject: [PATCH 07/13] Update changelog release 2.51.1 (#5588)

---
 CHANGELOG.md | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 56dcb16d98..23a71b2ea4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -15,11 +15,20 @@ The types of changes are:
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
-## [Unreleased](https://github.com/ethyca/fides/compare/2.51.0...main)
+## [Unreleased](https://github.com/ethyca/fides/compare/2.51.1...main)
 
+### Added
+- New page in the Cookie House sample app to demonstrate the use of embedding the FidesJS SDK on the page [#5564](https://github.com/ethyca/fides/pull/5564)
 
+### Fixed
+- Fixing quickstart.py script [#5585](https://github.com/ethyca/fides/pull/5585)
 
+## [2.51.1](https://github.com/ethyca/fides/compare/2.51.0...2.51.1)
 
+### Fixed
+- SaaS integrations using `oauth_client_credentials` now properly update their access token when editing the secrets. [#5548](https://github.com/ethyca/fides/pull/5548)
+- Saas integrations using `oauth_client_credentials` now properly refresh their access token when the current token expires [#5569](https://github.com/ethyca/fides/pull/5569)
+- Adding `dsr_testing_tools_enabled` security setting [#5573](https://github.com/ethyca/fides/pull/5573)
 
 ## [2.51.0](https://github.com/ethyca/fides/compare/2.50.0...2.51.0)
 

From ca48bae72f0baabdd5dafaf530b1ec86e9c42328 Mon Sep 17 00:00:00 2001
From: Adrian Galvan <adrian@ethyca.com>
Date: Mon, 9 Dec 2024 14:59:05 -0800
Subject: [PATCH 08/13] Putting dataset test results behind security flag
 (#5573)

---
 CHANGELOG.md                                  |  1 +
 .../api/api/v1/endpoints/dataset_endpoints.py |  9 +++
 .../v1/endpoints/privacy_request_endpoints.py |  8 ++-
 src/fides/api/schemas/privacy_request.py      |  2 +-
 src/fides/config/security_settings.py         |  4 ++
 tests/fixtures/application_fixtures.py        | 16 ++++++
 .../endpoints/test_dataset_test_endpoints.py  | 31 +++++++++-
 .../test_privacy_request_endpoints.py         | 57 ++++++++++++++++++-
 8 files changed, 119 insertions(+), 9 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 23a71b2ea4..e40cdf302f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -19,6 +19,7 @@ The types of changes are:
 
 ### Added
 - New page in the Cookie House sample app to demonstrate the use of embedding the FidesJS SDK on the page [#5564](https://github.com/ethyca/fides/pull/5564)
+- Adding `dsr_testing_tools_enabled` security setting [#5573](https://github.com/ethyca/fides/pull/5573)
 
 ### Fixed
 - Fixing quickstart.py script [#5585](https://github.com/ethyca/fides/pull/5585)
diff --git a/src/fides/api/api/v1/endpoints/dataset_endpoints.py b/src/fides/api/api/v1/endpoints/dataset_endpoints.py
index bdc8f367c6..9240396295 100644
--- a/src/fides/api/api/v1/endpoints/dataset_endpoints.py
+++ b/src/fides/api/api/v1/endpoints/dataset_endpoints.py
@@ -21,6 +21,7 @@
     HTTP_200_OK,
     HTTP_204_NO_CONTENT,
     HTTP_400_BAD_REQUEST,
+    HTTP_403_FORBIDDEN,
     HTTP_404_NOT_FOUND,
     HTTP_415_UNSUPPORTED_MEDIA_TYPE,
     HTTP_422_UNPROCESSABLE_ENTITY,
@@ -82,6 +83,7 @@
     V1_URL_PREFIX,
     YAML_DATASETS,
 )
+from fides.config import CONFIG
 
 from fides.api.models.sql_models import (  # type: ignore[attr-defined] # isort: skip
     Dataset as CtlDataset,
@@ -856,6 +858,13 @@ def test_connection_datasets(
     fides_key: FidesKey,
     unlabeled_identities: UnlabeledIdentities,
 ) -> Dict[str, Any]:
+
+    if not CONFIG.security.dsr_testing_tools_enabled:
+        raise HTTPException(
+            status_code=HTTP_403_FORBIDDEN,
+            detail="DSR testing tools are not enabled.",
+        )
+
     dataset_config = DatasetConfig.filter(
         db=db,
         conditions=(
diff --git a/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py b/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py
index 84cb453a63..3bec34ffed 100644
--- a/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py
+++ b/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py
@@ -2662,10 +2662,14 @@ def get_test_privacy_request_results(
     # Escape datetime and ObjectId values
     raw_data = privacy_request.get_raw_access_results()
     escaped_json = json.dumps(raw_data, indent=2, default=storage_json_encoder)
-    escaped_data = json.loads(escaped_json)
+    results = json.loads(escaped_json)
 
     return {
         "privacy_request_id": privacy_request.id,
         "status": privacy_request.status,
-        "results": escaped_data,
+        "results": (
+            results
+            if CONFIG.security.dsr_testing_tools_enabled
+            else "DSR testing tools are not enabled, results will not be shown."
+        ),
     }
diff --git a/src/fides/api/schemas/privacy_request.py b/src/fides/api/schemas/privacy_request.py
index 69ebcb2df5..c4e7445a25 100644
--- a/src/fides/api/schemas/privacy_request.py
+++ b/src/fides/api/schemas/privacy_request.py
@@ -401,4 +401,4 @@ class FilteredPrivacyRequestResults(FidesSchema):
 
     privacy_request_id: str
     status: PrivacyRequestStatus
-    results: Dict[str, Any]
+    results: Union[Dict[str, Any], str]
diff --git a/src/fides/config/security_settings.py b/src/fides/config/security_settings.py
index 05ff855d60..8a5cdf426c 100644
--- a/src/fides/config/security_settings.py
+++ b/src/fides/config/security_settings.py
@@ -123,6 +123,10 @@ class SecuritySettings(FidesSettings):
         default=False,
         description="If set to True, the user interface will display a download button for subject requests.",
     )
+    dsr_testing_tools_enabled: bool = Field(
+        default=False,
+        description="If set to True, contributor and owner roles will be able to run test privacy requests.",
+    )
     subject_request_download_link_ttl_seconds: int = Field(
         default=432000,
         description="The number of seconds that a pre-signed download URL when using S3 storage will be valid. The default is equal to 5 days.",
diff --git a/tests/fixtures/application_fixtures.py b/tests/fixtures/application_fixtures.py
index 04a3472696..54fe6f3784 100644
--- a/tests/fixtures/application_fixtures.py
+++ b/tests/fixtures/application_fixtures.py
@@ -3402,6 +3402,22 @@ def subject_request_download_ui_enabled():
     CONFIG.security.subject_request_download_ui_enabled = original_value
 
 
+@pytest.fixture(scope="function")
+def dsr_testing_tools_enabled():
+    original_value = CONFIG.security.dsr_testing_tools_enabled
+    CONFIG.security.dsr_testing_tools_enabled = True
+    yield
+    CONFIG.security.dsr_testing_tools_enabled = original_value
+
+
+@pytest.fixture(scope="function")
+def dsr_testing_tools_disabled():
+    original_value = CONFIG.security.dsr_testing_tools_enabled
+    CONFIG.security.dsr_testing_tools_enabled = False
+    yield
+    CONFIG.security.dsr_testing_tools_enabled = original_value
+
+
 @pytest.fixture(scope="function")
 def system_with_no_uses(db: Session) -> Generator[System, None, None]:
     system = System.create(
diff --git a/tests/ops/api/v1/endpoints/test_dataset_test_endpoints.py b/tests/ops/api/v1/endpoints/test_dataset_test_endpoints.py
index c5056aec61..347ab9a4f1 100644
--- a/tests/ops/api/v1/endpoints/test_dataset_test_endpoints.py
+++ b/tests/ops/api/v1/endpoints/test_dataset_test_endpoints.py
@@ -208,6 +208,7 @@ def test_dataset_test_not_authenticated(
         response = api_client.post(dataset_url + "/test", headers={})
         assert response.status_code == 401
 
+    @pytest.mark.usefixtures("dsr_testing_tools_enabled")
     def test_dataset_test_wrong_scope(
         self,
         dataset_config,
@@ -220,7 +221,6 @@ def test_dataset_test_wrong_scope(
         response = api_client.post(dataset_url + "/test", headers=auth_header)
         assert response.status_code == 403
 
-    @pytest.mark.usefixtures("default_access_policy")
     @pytest.mark.parametrize(
         "auth_header,expected_status",
         [
@@ -231,6 +231,7 @@ def test_dataset_test_wrong_scope(
             ("approver_auth_header", HTTP_403_FORBIDDEN),
         ],
     )
+    @pytest.mark.usefixtures("default_access_policy", "dsr_testing_tools_enabled")
     def test_dataset_test_with_roles(
         self,
         dataset_config,
@@ -249,6 +250,7 @@ def test_dataset_test_with_roles(
         )
         assert response.status_code == expected_status
 
+    @pytest.mark.usefixtures("dsr_testing_tools_enabled")
     def test_dataset_test_connection_does_not_exist(
         self,
         api_client: TestClient,
@@ -263,6 +265,7 @@ def test_dataset_test_connection_does_not_exist(
         )
         assert response.status_code == 404
 
+    @pytest.mark.usefixtures("dsr_testing_tools_enabled")
     def test_dataset_test_dataset_does_not_exist(
         self,
         connection_config,
@@ -290,7 +293,7 @@ def test_dataset_test_dataset_does_not_exist(
             ),
         ],
     )
-    @pytest.mark.usefixtures("default_access_policy")
+    @pytest.mark.usefixtures("default_access_policy", "dsr_testing_tools_enabled")
     def test_dataset_test_invalid_payloads(
         self,
         connection_config,
@@ -310,7 +313,9 @@ def test_dataset_test_invalid_payloads(
         assert response.status_code == HTTP_400_BAD_REQUEST
         assert response.json()["detail"] == expected_response
 
-    @pytest.mark.usefixtures("default_access_policy", "postgres_integration_db")
+    @pytest.mark.usefixtures(
+        "default_access_policy", "postgres_integration_db", "dsr_testing_tools_enabled"
+    )
     def test_dataset_test(
         self,
         connection_config,
@@ -327,3 +332,23 @@ def test_dataset_test(
         )
         assert response.status_code == HTTP_200_OK
         assert "privacy_request_id" in response.json().keys()
+
+    @pytest.mark.usefixtures(
+        "default_access_policy", "postgres_integration_db", "dsr_testing_tools_disabled"
+    )
+    def test_dataset_test_disabled(
+        self,
+        connection_config,
+        dataset_config,
+        api_client: TestClient,
+        generate_auth_header,
+    ) -> None:
+        dataset_url = get_connection_dataset_url(connection_config, dataset_config)
+        auth_header = generate_auth_header(scopes=[DATASET_TEST])
+        response = api_client.post(
+            dataset_url + "/test",
+            headers=auth_header,
+            json={"email": "jane@example.com"},
+        )
+        assert response.status_code == HTTP_403_FORBIDDEN
+        assert response.json()["detail"] == "DSR testing tools are not enabled."
diff --git a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py
index aaf64d4e7b..c4f386d7b0 100644
--- a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py
+++ b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py
@@ -8392,7 +8392,7 @@ def test_filtered_results_wrong_scope(
         response = api_client.get(url, headers=auth_header)
         assert response.status_code == 403
 
-    @pytest.mark.usefixtures("default_access_policy")
+    @pytest.mark.usefixtures("default_access_policy", "dsr_testing_tools_enabled")
     @pytest.mark.parametrize(
         "auth_header,expected_status",
         [
@@ -8427,7 +8427,9 @@ def test_filtered_results_with_roles(
         assert response.status_code == expected_status
 
     @pytest.mark.integration_postgres
-    @pytest.mark.usefixtures("default_access_policy", "postgres_integration_db")
+    @pytest.mark.usefixtures(
+        "default_access_policy", "postgres_integration_db", "dsr_testing_tools_enabled"
+    )
     def test_filtered_results_postgres(
         self,
         connection_config,
@@ -8462,8 +8464,57 @@ def test_filtered_results_postgres(
             "results",
         }
 
+    @pytest.mark.integration_postgres
+    @pytest.mark.usefixtures(
+        "default_access_policy",
+        "postgres_integration_db",
+        "dsr_testing_tools_enabled",
+    )
+    def test_filtered_results_postgres_access_testing_disabled(
+        self,
+        connection_config,
+        postgres_example_test_dataset_config,
+        api_client: TestClient,
+        generate_auth_header,
+    ) -> None:
+        dataset_url = get_connection_dataset_url(
+            connection_config, postgres_example_test_dataset_config
+        )
+        auth_header = generate_auth_header(scopes=[DATASET_TEST])
+        response = api_client.post(
+            dataset_url + "/test",
+            headers=auth_header,
+            json={"email": "jane@example.com"},
+        )
+        assert response.status_code == HTTP_200_OK
+
+        original_value = CONFIG.security.dsr_testing_tools_enabled
+        CONFIG.security.dsr_testing_tools_enabled = False
+
+        privacy_request_id = response.json()["privacy_request_id"]
+        url = V1_URL_PREFIX + PRIVACY_REQUEST_FILTERED_RESULTS.format(
+            privacy_request_id=privacy_request_id
+        )
+        auth_header = generate_auth_header(scopes=[PRIVACY_REQUEST_READ_ACCESS_RESULTS])
+        response = api_client.get(
+            url,
+            headers=auth_header,
+        )
+        assert response.status_code == HTTP_200_OK
+        assert set(response.json().keys()) == {
+            "privacy_request_id",
+            "status",
+            "results",
+        }
+        assert (
+            response.json()["results"]
+            == "DSR testing tools are not enabled, results will not be shown."
+        )
+
+        CONFIG.security.dsr_testing_tools_enabled = original_value
+
     @pytest.mark.integration_mongo
-    @pytest.mark.usefixtures("default_access_policy")
+    @pytest.mark.usefixtures("default_access_policy", "dsr_testing_tools_enabled")
     def test_filtered_results_mongo(
         self,
         mongo_connection_config,

From 6631def82785cd643bf11d22a45255e36baf4c12 Mon Sep 17 00:00:00 2001
From: Bruno Gutierrez Rios <bruno.gutierrez96@gmail.com>
Date: Mon, 9 Dec 2024 17:23:50 -0300
Subject: [PATCH 09/13] LA 194 refreshing oauth credentials (#5569)

Co-authored-by: Adrian Galvan <adrian@ethyca.com>
---
 CHANGELOG.md                                  |  1 -
 src/fides/api/models/connectionconfig.py      |  7 ++++--
 ...tion_strategy_oauth2_client_credentials.py | 19 +++++++++++++++
 ...tion_strategy_oauth2_client_credentials.py | 24 +------------------
 4 files changed, 25 insertions(+), 26 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e40cdf302f..23a71b2ea4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -19,7 +19,6 @@ The types of changes are:
 
 ### Added
 - New page in the Cookie House sample app to demonstrate the use of embedding the FidesJS SDK on the page [#5564](https://github.com/ethyca/fides/pull/5564)
-- Adding `dsr_testing_tools_enabled` security setting [#5573](https://github.com/ethyca/fides/pull/5573)
 
 ### Fixed
 - Fixing quickstart.py script [#5585](https://github.com/ethyca/fides/pull/5585)
diff --git a/src/fides/api/models/connectionconfig.py b/src/fides/api/models/connectionconfig.py
index 0859c516b2..2dc518cd44 100644
--- a/src/fides/api/models/connectionconfig.py
+++ b/src/fides/api/models/connectionconfig.py
@@ -220,10 +220,13 @@ def authorized(self) -> bool:
             return False
 
         # hard-coding to avoid cyclic dependency
-        if authentication.strategy != "oauth2_authorization_code":
+        if authentication.strategy not in [
+            "oauth2_authorization_code",
+            "oauth2_client_credentials",
+        ]:
             return False
 
-        return bool(self.secrets and self.secrets.get("access_token"))
+        return bool(self.secrets and "access_token" in self.secrets.keys())
 
     @property
     def name_or_key(self) -> str:
diff --git a/src/fides/api/service/authentication/authentication_strategy_oauth2_client_credentials.py b/src/fides/api/service/authentication/authentication_strategy_oauth2_client_credentials.py
index 70034b6901..f911e28049 100644
--- a/src/fides/api/service/authentication/authentication_strategy_oauth2_client_credentials.py
+++ b/src/fides/api/service/authentication/authentication_strategy_oauth2_client_credentials.py
@@ -33,3 +33,22 @@ def add_authentication(
         # add access_token to request
         request.headers["Authorization"] = "Bearer " + access_token
         return request
+
+    def _refresh_token(self, connection_config: ConnectionConfig) -> str:
+        """
+        Persists and returns a refreshed access_token if the token is close to expiring.
+        Otherwise just returns the existing access_token.
+
+        For the Client Credentials OAuth flow we reuse the access request to get a new token.
+        """
+
+        expires_at = connection_config.secrets.get("expires_at")  # type: ignore
+        if self._close_to_expiration(expires_at, connection_config):
+            refresh_response = self._call_token_request(
+                "refresh", self.token_request, connection_config
+            )
+            return self._validate_and_store_response(
+                refresh_response, connection_config
+            )
+
+        return connection_config.secrets.get("access_token")  # type: ignore
diff --git a/tests/ops/service/authentication/test_authentication_strategy_oauth2_client_credentials.py b/tests/ops/service/authentication/test_authentication_strategy_oauth2_client_credentials.py
index 5ac18977b8..d3d19c3e0d 100644
--- a/tests/ops/service/authentication/test_authentication_strategy_oauth2_client_credentials.py
+++ b/tests/ops/service/authentication/test_authentication_strategy_oauth2_client_credentials.py
@@ -40,23 +40,7 @@ def oauth2_client_credentials_configuration() -> (
                 {"name": "client_secret", "value": "<client_secret>"},
                 {"name": "grant_type", "value": "client_credentials"},
             ],
-        },
-        "refresh_request": {
-            "method": "POST",
-            "path": "/oauth/token",
-            "headers": [
-                {
-                    "name": "Content-Type",
-                    "value": "application/x-www-form-urlencoded",
-                }
-            ],
-            "query_params": [
-                {"name": "client_id", "value": "<client_id>"},
-                {"name": "client_secret", "value": "<client_secret>"},
-                {"name": "grant_type", "value": "refresh_token"},
-                {"name": "refresh_token", "value": "<refresh_token>"},
-            ],
-        },
+        }
     }
 
 
@@ -69,7 +53,6 @@ def oauth2_client_credentials_connection_config(
         "client_id": "client",
         "client_secret": "secret",
         "access_token": "access",
-        "refresh_token": "refresh",
     }
     saas_config = {
         "fides_key": "oauth2_client_credentials_connector",
@@ -249,7 +232,6 @@ def test_oauth2_authentication_successful_refresh(
                     "client_id": "client",
                     "client_secret": "secret",
                     "access_token": "new_access",
-                    "refresh_token": "refresh",
                     "expires_at": 0,
                 }
             },
@@ -327,7 +309,6 @@ def test_get_access_token(
         expires_in = 7200
         mock_send().json.return_value = {
             "access_token": "new_access",
-            "refresh_token": "new_refresh",
             "expires_in": expires_in,
         }
 
@@ -347,7 +328,6 @@ def test_get_access_token(
                     "client_id": "client",
                     "client_secret": "secret",
                     "access_token": "new_access",
-                    "refresh_token": "new_refresh",
                     "expires_at": int(datetime.utcnow().timestamp()) + expires_in,
                 }
             },
@@ -376,7 +356,6 @@ def test_get_access_token_no_expires_in(
         # mock the json response from calling the access token request
         mock_send().json.return_value = {
             "access_token": "new_access",
-            "refresh_token": "new_refresh",
         }
 
         oauth2_client_credentials_configuration["expires_in"] = 3600
@@ -396,7 +375,6 @@ def test_get_access_token_no_expires_in(
                     "client_id": "client",
                     "client_secret": "secret",
                     "access_token": "new_access",
-                    "refresh_token": "new_refresh",
                     "expires_at": int(datetime.utcnow().timestamp())
                     + oauth2_client_credentials_configuration["expires_in"],
                 }

From a97a421ae4a9caa77b17d323dffb73471246e357 Mon Sep 17 00:00:00 2001
From: Adam Sachs <adam@ethyca.com>
Date: Wed, 11 Dec 2024 12:22:46 -0500
Subject: [PATCH 10/13] revert disabling pooling on celery worker tasks (#5592)

---
 src/fides/api/tasks/__init__.py       |  3 ++-
 tests/ops/tasks/test_database_task.py | 15 ---------------
 2 files changed, 2 insertions(+), 16 deletions(-)

diff --git a/src/fides/api/tasks/__init__.py b/src/fides/api/tasks/__init__.py
index e757052ac1..ed5d9b8556 100644
--- a/src/fides/api/tasks/__init__.py
+++ b/src/fides/api/tasks/__init__.py
@@ -64,10 +64,11 @@ def get_new_session(self) -> ContextManager[Session]:
         if self._task_engine is None:
             self._task_engine = get_db_engine(
                 config=CONFIG,
+                pool_size=CONFIG.database.task_engine_pool_size,
+                max_overflow=CONFIG.database.task_engine_max_overflow,
                 keepalives_idle=CONFIG.database.task_engine_keepalives_idle,
                 keepalives_interval=CONFIG.database.task_engine_keepalives_interval,
                 keepalives_count=CONFIG.database.task_engine_keepalives_count,
-                disable_pooling=True,
             )
 
         # same for the sessionmaker
diff --git a/tests/ops/tasks/test_database_task.py b/tests/ops/tasks/test_database_task.py
index 0076f16f84..c2907ade17 100644
--- a/tests/ops/tasks/test_database_task.py
+++ b/tests/ops/tasks/test_database_task.py
@@ -42,21 +42,6 @@ def always_failing_session_maker(self):
         mock_maker.side_effect = OperationalError("connection failed", None, None)
         return mock_maker
 
-    @pytest.mark.parametrize(
-        "config_fixture", [None, "mock_config_changed_db_engine_settings"]
-    )
-    def test_get_task_session(self, config_fixture, request):
-        if config_fixture is not None:
-            request.getfixturevalue(
-                config_fixture
-            )  # used to invoke config fixture if provided
-        pool_size = CONFIG.database.task_engine_pool_size
-        max_overflow = CONFIG.database.task_engine_max_overflow
-        t = DatabaseTask()
-        session: Session = t.get_new_session()
-        engine: Engine = session.get_bind()
-        assert isinstance(engine.pool, NullPool)
-
     def test_retry_on_operational_error(self, recovering_session_maker):
         """Test that session creation retries on OperationalError"""
 

From 57b997459bd804cbf3360460127499f0bd7f4e72 Mon Sep 17 00:00:00 2001
From: Adam Sachs <adam@ethyca.com>
Date: Wed, 11 Dec 2024 13:15:21 -0500
Subject: [PATCH 11/13] update changelog for PR #5592 (#5595)

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 23a71b2ea4..28d06f8674 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -29,6 +29,7 @@ The types of changes are:
 - SaaS integrations using `oauth_client_credentials` now properly update their access token when editing the secrets. [#5548](https://github.com/ethyca/fides/pull/5548)
 - Saas integrations using `oauth_client_credentials` now properly refresh their access token when the current token expires [#5569](https://github.com/ethyca/fides/pull/5569)
 - Adding `dsr_testing_tools_enabled` security setting [#5573](https://github.com/ethyca/fides/pull/5573)
+- Reverted elimination of connection pool in worker tasks to prevent DB performance issues [#5592](https://github.com/ethyca/fides/pull/5592)
 
 ## [2.51.0](https://github.com/ethyca/fides/compare/2.50.0...2.51.0)
 

From 7ea8e903a54690aa2b8488e3b2077b6fe165b062 Mon Sep 17 00:00:00 2001
From: Catherine Smith <eastandwestwind@gmail.com>
Date: Mon, 16 Dec 2024 20:02:31 +0100
Subject: [PATCH 12/13] Updated CHANGELOG.md for release 2.51.2 (#5608)

---
 CHANGELOG.md | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 28d06f8674..d49da9bca2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -15,14 +15,24 @@ The types of changes are:
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
-## [Unreleased](https://github.com/ethyca/fides/compare/2.51.1...main)
+## [Unreleased](https://github.com/ethyca/fides/compare/2.51.2...main)
 
 ### Added
 - New page in the Cookie House sample app to demonstrate the use of embedding the FidesJS SDK on the page [#5564](https://github.com/ethyca/fides/pull/5564)
+- Added event based communication example to the Cookie House sample app [#5597](https://github.com/ethyca/fides/pull/5597)
+- Added new erasure tests for BigQuery Enterprise [#5554](https://github.com/ethyca/fides/pull/5554)
 
 ### Fixed
 - Fixing quickstart.py script [#5585](https://github.com/ethyca/fides/pull/5585)
 
+### Changed
+- Adjusted Ant's Select component colors and icon [#5594](https://github.com/ethyca/fides/pull/5594)
+
+## [2.51.2](https://github.com/ethyca/fides/compare/2.51.1...2.51.2)
+
+### Fixed
+- Fixed miscellaneous performance issues with Systems and PrivacyDeclarations [#5601](https://github.com/ethyca/fides/pull/5601)
+
 ## [2.51.1](https://github.com/ethyca/fides/compare/2.51.0...2.51.1)
 
 ### Fixed

From e361a0bf1222d65fbada9cf0d9dc0ade7d78afd8 Mon Sep 17 00:00:00 2001
From: Adrian Galvan <adrian@ethyca.com>
Date: Mon, 16 Dec 2024 10:28:48 -0800
Subject: [PATCH 13/13] Improving datamap performance (#5601)

---
 CHANGELOG.md                                  |   1 +
 .../reporting/DatamapReportTableColumns.tsx   |  64 ++++---
 src/fides/api/models/sql_models.py            |  57 ++++---
 src/fides/api/util/data_category.py           |  32 +++-
 tests/conftest.py                             | 157 +++++++++++++++++-
 tests/ctl/core/test_privacy_declaration.py    |  52 ++++--
 tests/ctl/core/test_system.py                 |  25 ++-
 7 files changed, 317 insertions(+), 71 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index d49da9bca2..e782bafdeb 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -24,6 +24,7 @@ The types of changes are:
 
 ### Fixed
 - Fixing quickstart.py script [#5585](https://github.com/ethyca/fides/pull/5585)
+- Fixed miscellaneous performance issues with Systems and PrivacyDeclarations [#5601](https://github.com/ethyca/fides/pull/5601)
 
 ### Changed
 - Adjusted Ant's Select component colors and icon [#5594](https://github.com/ethyca/fides/pull/5594)
diff --git a/clients/admin-ui/src/features/datamap/reporting/DatamapReportTableColumns.tsx b/clients/admin-ui/src/features/datamap/reporting/DatamapReportTableColumns.tsx
index a4dd638b51..e433f89723 100644
--- a/clients/admin-ui/src/features/datamap/reporting/DatamapReportTableColumns.tsx
+++ b/clients/admin-ui/src/features/datamap/reporting/DatamapReportTableColumns.tsx
@@ -426,49 +426,65 @@ export const getDatamapReportColumns = ({
     columnHelper.accessor((row) => row.system_undeclared_data_categories, {
       id: COLUMN_IDS.SYSTEM_UNDECLARED_DATA_CATEGORIES,
       cell: (props) => {
-        const value = props.getValue();
-
+        const cellValues = props.getValue();
+        if (!cellValues || cellValues.length === 0) {
+          return null;
+        }
+        const values = isArray(cellValues)
+          ? cellValues.map((value) => {
+              return { label: getDataCategoryDisplayName(value), key: value };
+            })
+          : [
+              {
+                label: getDataCategoryDisplayName(cellValues),
+                key: cellValues,
+              },
+            ];
         return (
-          <GroupCountBadgeCell
-            ignoreZero
-            suffix="system undeclared data categories"
-            value={
-              isArray(value)
-                ? map(value, getDataCategoryDisplayName)
-                : getDataCategoryDisplayName(value || "")
-            }
-            badgeProps={{ variant: "outline" }}
-            {...props}
+          <BadgeCellExpandable
+            values={values}
+            cellProps={props as any}
+            variant="outline"
           />
         );
       },
       meta: {
         showHeaderMenu: !isRenaming,
+        showHeaderMenuWrapOption: true,
         width: "auto",
+        overflow: "hidden",
       },
     }),
     columnHelper.accessor((row) => row.data_use_undeclared_data_categories, {
       id: COLUMN_IDS.DATA_USE_UNDECLARED_DATA_CATEGORIES,
       cell: (props) => {
-        const value = props.getValue();
-
+        const cellValues = props.getValue();
+        if (!cellValues || cellValues.length === 0) {
+          return null;
+        }
+        const values = isArray(cellValues)
+          ? cellValues.map((value) => {
+              return { label: getDataCategoryDisplayName(value), key: value };
+            })
+          : [
+              {
+                label: getDataCategoryDisplayName(cellValues),
+                key: cellValues,
+              },
+            ];
         return (
-          <GroupCountBadgeCell
-            ignoreZero
-            suffix="data use undeclared data categories"
-            value={
-              isArray(value)
-                ? map(value, getDataCategoryDisplayName)
-                : getDataCategoryDisplayName(value || "")
-            }
-            badgeProps={{ variant: "outline" }}
-            {...props}
+          <BadgeCellExpandable
+            values={values}
+            cellProps={props as any}
+            variant="outline"
           />
         );
       },
       meta: {
         showHeaderMenu: !isRenaming,
+        showHeaderMenuWrapOption: true,
         width: "auto",
+        overflow: "hidden",
       },
     }),
     columnHelper.accessor((row) => row.cookies, {
diff --git a/src/fides/api/models/sql_models.py b/src/fides/api/models/sql_models.py
index f80abae5f9..2132ffe597 100644
--- a/src/fides/api/models/sql_models.py
+++ b/src/fides/api/models/sql_models.py
@@ -27,6 +27,7 @@
     UniqueConstraint,
     case,
     cast,
+    func,
     select,
     text,
     type_coerce,
@@ -36,7 +37,7 @@
 from sqlalchemy.ext.asyncio import AsyncSession, async_object_session
 from sqlalchemy.ext.hybrid import hybrid_property
 from sqlalchemy.orm import Session, relationship
-from sqlalchemy.sql import Select, func
+from sqlalchemy.sql import Select
 from sqlalchemy.sql.elements import Case
 from sqlalchemy.sql.sqltypes import DateTime
 from typing_extensions import Protocol, runtime_checkable
@@ -404,15 +405,6 @@ class System(Base, FidesBase):
         "Cookies", back_populates="system", lazy="selectin", uselist=True, viewonly=True
     )
 
-    # index scan using ix_ctl_datasets_fides_key on ctl_datasets
-    datasets = relationship(
-        "Dataset",
-        primaryjoin="foreign(Dataset.fides_key)==any_(System.dataset_references)",
-        lazy="selectin",
-        uselist=True,
-        viewonly=True,
-    )
-
     @classmethod
     def get_data_uses(
         cls: Type[System], systems: List[System], include_parents: bool = True
@@ -430,11 +422,21 @@ def get_data_uses(
                         data_uses.add(data_use)
         return data_uses
 
-    @property
-    def undeclared_data_categories(self) -> Set[str]:
+    def dataset_data_categories(self, data_categories: Dict[str, Set[str]]) -> Set[str]:
+        aggregate = set()
+        for dataset_key in self.dataset_references or []:
+            aggregate.update(data_categories.get(dataset_key, set()))
+        return aggregate
+
+    def undeclared_data_categories(
+        self, data_categories: Dict[str, Set[str]]
+    ) -> Set[str]:
         """
         Returns a set of data categories defined on the system's datasets
         that are not associated with any data use (privacy declaration).
+
+        Looks up the unique set of data categories for a given dataset from the pre-computed data_categories map.
+        This is done to improve performance.
         """
 
         privacy_declaration_data_categories = set()
@@ -443,9 +445,9 @@ def undeclared_data_categories(self) -> Set[str]:
                 privacy_declaration.data_categories
             )
 
-        system_dataset_data_categories = set()
-        for dataset in self.datasets:
-            system_dataset_data_categories.update(dataset.field_data_categories)
+        system_dataset_data_categories = set(
+            self.dataset_data_categories(data_categories)
+        )
 
         return find_undeclared_categories(
             system_dataset_data_categories, privacy_declaration_data_categories
@@ -501,13 +503,6 @@ class PrivacyDeclaration(Base):
     cookies = relationship(
         "Cookies", back_populates="privacy_declaration", lazy="joined", uselist=True
     )
-    datasets = relationship(
-        "Dataset",
-        primaryjoin="foreign(Dataset.fides_key)==any_(PrivacyDeclaration.dataset_references)",
-        lazy="selectin",
-        uselist=True,
-        viewonly=True,
-    )
 
     @classmethod
     def create(
@@ -547,11 +542,21 @@ def purpose(cls) -> Case:
             else_=None,
         )
 
-    @property
-    def undeclared_data_categories(self) -> Set[str]:
+    def dataset_data_categories(self, data_categories: Dict[str, Set[str]]) -> Set[str]:
+        aggregate = set()
+        for dataset_key in self.dataset_references or []:
+            aggregate.update(data_categories.get(dataset_key, set()))
+        return aggregate
+
+    def undeclared_data_categories(
+        self, data_categories: Dict[str, Set[str]]
+    ) -> Set[str]:
         """
         Aggregates a unique set of data categories across the collections in the associated datasets and
         returns the data categories that are not defined directly on this or any sibling privacy declarations.
+
+        Looks up the unique set of data categories for a given dataset from the pre-computed data_categories map.
+        This is done to improve performance.
         """
 
         # Note: This property evaluates the data categories attached to the datasets associated with this specific
@@ -559,9 +564,7 @@ def undeclared_data_categories(self) -> Set[str]:
         # data categories across this privacy declaration and its sibling privacy declarations.
 
         # all data categories from the datasets
-        dataset_data_categories = set()
-        for dataset in self.datasets:
-            dataset_data_categories.update(dataset.field_data_categories)
+        dataset_data_categories = set(self.dataset_data_categories(data_categories))
 
         # all data categories specified directly on this and sibling privacy declarations
         declared_data_categories = set()
diff --git a/src/fides/api/util/data_category.py b/src/fides/api/util/data_category.py
index 71c24cf366..5c533bd44f 100644
--- a/src/fides/api/util/data_category.py
+++ b/src/fides/api/util/data_category.py
@@ -1,14 +1,16 @@
 from enum import Enum as EnumType
-from typing import List, Type
+from typing import Dict, List, Set, Type
 
 from fideslang.default_taxonomy import DEFAULT_TAXONOMY
 from fideslang.validation import FidesKey
+from sqlalchemy import func, select, text
 from sqlalchemy.orm import Session
 
 from fides.api import common_exceptions
 
 from fides.api.models.sql_models import (  # type: ignore[attr-defined] # isort: skip
     DataCategory as DataCategoryDbModel,
+    Dataset,
 )
 
 
@@ -89,3 +91,31 @@ def filter_data_categories(
         }
         return sorted(list(default_categories))
     return sorted(user_categories)
+
+
+def get_data_categories_map(db: Session) -> Dict[str, Set[str]]:
+    """
+    Returns a map of all datasets, where the keys are the fides keys
+    of each dataset and the value is a set of data categories associated with each dataset
+    """
+
+    subquery = (
+        select(
+            Dataset.fides_key,
+            func.jsonb_array_elements_text(
+                text(
+                    "jsonb_path_query(collections::jsonb, '$.** ? (@.data_categories != null).data_categories')"
+                )
+            ).label("category"),
+        ).select_from(Dataset)
+    ).cte()
+
+    query = (
+        select(
+            [subquery.c.fides_key, func.array_agg(func.distinct(subquery.c.category))]
+        )
+        .select_from(subquery)
+        .group_by(subquery.c.fides_key)
+    )
+    result = db.execute(query)
+    return {key: set(value) if value else set() for key, value in result.all()}
diff --git a/tests/conftest.py b/tests/conftest.py
index d513472ee3..98e8f07f97 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1400,7 +1400,78 @@ def system_with_undeclared_data_categories(db: Session) -> System:
 
 
 @pytest.fixture(scope="function")
-def privacy_declaration_with_dataset_references(db: Session) -> System:
+def system_with_a_single_dataset_reference(db: Session) -> System:
+    first_dataset = CtlDataset.create_from_dataset_dict(
+        db,
+        {
+            "fides_key": f"dataset_key-f{uuid4()}",
+            "collections": [
+                {
+                    "name": "loyalty",
+                    "fields": [
+                        {
+                            "name": "id",
+                            "data_categories": ["user.unique_id"],
+                        },
+                    ],
+                }
+            ],
+        },
+    )
+    second_dataset = CtlDataset.create_from_dataset_dict(
+        db,
+        {
+            "fides_key": f"dataset_key-f{uuid4()}",
+            "collections": [
+                {
+                    "name": "customer",
+                    "fields": [
+                        {
+                            "name": "shipping_info",
+                            "fields": [
+                                {
+                                    "name": "street",
+                                    "data_categories": ["user.contact.address.street"],
+                                }
+                            ],
+                        },
+                        {
+                            "name": "first_name",
+                            "data_categories": ["user.name.first"],
+                        },
+                    ],
+                },
+                {
+                    "name": "activity",
+                    "fields": [
+                        {
+                            "name": "last_login",
+                            "data_categories": ["user.behavior"],
+                        },
+                    ],
+                },
+            ],
+        },
+    )
+    system = System.create(
+        db=db,
+        data={
+            "fides_key": f"system_key-f{uuid4()}",
+            "name": f"system-{uuid4()}",
+            "description": "fixture-made-system",
+            "organization_fides_key": "default_organization",
+            "system_type": "Service",
+            "dataset_references": [first_dataset.fides_key, second_dataset.fides_key],
+        },
+    )
+
+    return system
+
+
+@pytest.fixture(scope="function")
+def privacy_declaration_with_single_dataset_reference(
+    db: Session,
+) -> PrivacyDeclaration:
     ctl_dataset = CtlDataset.create_from_dataset_dict(
         db,
         {
@@ -1447,6 +1518,90 @@ def privacy_declaration_with_dataset_references(db: Session) -> System:
     return privacy_declaration
 
 
+@pytest.fixture(scope="function")
+def privacy_declaration_with_multiple_dataset_references(
+    db: Session,
+) -> PrivacyDeclaration:
+    first_dataset = CtlDataset.create_from_dataset_dict(
+        db,
+        {
+            "fides_key": f"dataset_key-f{uuid4()}",
+            "collections": [
+                {
+                    "name": "loyalty",
+                    "fields": [
+                        {
+                            "name": "id",
+                            "data_categories": ["user.unique_id"],
+                        },
+                    ],
+                }
+            ],
+        },
+    )
+    second_dataset = CtlDataset.create_from_dataset_dict(
+        db,
+        {
+            "fides_key": f"dataset_key-f{uuid4()}",
+            "collections": [
+                {
+                    "name": "customer",
+                    "fields": [
+                        {
+                            "name": "shipping_info",
+                            "fields": [
+                                {
+                                    "name": "street",
+                                    "data_categories": ["user.contact.address.street"],
+                                }
+                            ],
+                        },
+                        {
+                            "name": "first_name",
+                            "data_categories": ["user.name.first"],
+                        },
+                    ],
+                },
+                {
+                    "name": "activity",
+                    "fields": [
+                        {
+                            "name": "last_login",
+                            "data_categories": ["user.behavior"],
+                        },
+                    ],
+                },
+            ],
+        },
+    )
+    system = System.create(
+        db=db,
+        data={
+            "fides_key": f"system_key-f{uuid4()}",
+            "name": f"system-{uuid4()}",
+            "description": "fixture-made-system",
+            "organization_fides_key": "default_organization",
+            "system_type": "Service",
+        },
+    )
+
+    privacy_declaration = PrivacyDeclaration.create(
+        db=db,
+        data={
+            "name": "Collect data for third party sharing",
+            "system_id": system.id,
+            "data_categories": ["user.device.cookie_id"],
+            "data_use": "third_party_sharing",
+            "data_subjects": ["customer"],
+            "dataset_references": [first_dataset.fides_key, second_dataset.fides_key],
+            "egress": None,
+            "ingress": None,
+        },
+    )
+
+    return privacy_declaration
+
+
 @pytest.fixture(scope="function")
 def system_multiple_decs(db: Session, system: System) -> Generator[System, None, None]:
     """
diff --git a/tests/ctl/core/test_privacy_declaration.py b/tests/ctl/core/test_privacy_declaration.py
index ba2e11cdb2..0adc58949a 100644
--- a/tests/ctl/core/test_privacy_declaration.py
+++ b/tests/ctl/core/test_privacy_declaration.py
@@ -1,25 +1,51 @@
+from typing import Dict, Set
+
+import pytest
+
 from fides.api.models.sql_models import PrivacyDeclaration
+from fides.api.util.data_category import get_data_categories_map
 
 
 class TestPrivacyDeclaration:
-    def test_privacy_declaration_datasets(
+
+    @pytest.fixture(scope="function")
+    def data_categories_map(self, db) -> Dict[str, Set[str]]:
+        return get_data_categories_map(db)
+
+    def test_privacy_declaration_dataset_data_categories(
         self,
-        privacy_declaration_with_dataset_references: PrivacyDeclaration,
-    ) -> None:
-        assert len(privacy_declaration_with_dataset_references.datasets) == 1
+        privacy_declaration_with_multiple_dataset_references: PrivacyDeclaration,
+        data_categories_map,
+    ):
+        assert set(
+            privacy_declaration_with_multiple_dataset_references.dataset_data_categories(
+                data_categories_map
+            )
+        ) == {
+            "user.behavior",
+            "user.name.first",
+            "user.unique_id",
+            "user.contact.address.street",
+        }
 
     def test_privacy_declaration_undeclared_data_categories(
-        self, privacy_declaration_with_dataset_references
+        self,
+        privacy_declaration_with_single_dataset_reference: PrivacyDeclaration,
+        data_categories_map,
     ):
-        assert (
-            privacy_declaration_with_dataset_references.undeclared_data_categories
-            == {"user.contact.email"}
-        )
+        assert privacy_declaration_with_single_dataset_reference.undeclared_data_categories(
+            data_categories_map
+        ) == {
+            "user.contact.email"
+        }
 
     def test_privacy_declaration_data_category_defined_on_sibling(
-        self, db, privacy_declaration_with_dataset_references
+        self,
+        db,
+        privacy_declaration_with_single_dataset_reference: PrivacyDeclaration,
+        data_categories_map,
     ):
-        system = privacy_declaration_with_dataset_references.system
+        system = privacy_declaration_with_single_dataset_reference.system
 
         # Create a new privacy declaration with the data category we're searching for
         PrivacyDeclaration.create(
@@ -41,6 +67,8 @@ def test_privacy_declaration_data_category_defined_on_sibling(
         # Check that the original privacy declaration doesn't have any undeclared data categories
         # because we also search sibling privacy declarations for the data category
         assert (
-            privacy_declaration_with_dataset_references.undeclared_data_categories
+            privacy_declaration_with_single_dataset_reference.undeclared_data_categories(
+                data_categories_map
+            )
             == set()
         )
diff --git a/tests/ctl/core/test_system.py b/tests/ctl/core/test_system.py
index 8761c96915..e93363aef0 100644
--- a/tests/ctl/core/test_system.py
+++ b/tests/ctl/core/test_system.py
@@ -13,6 +13,7 @@
 from fides.api.db.system import create_system, upsert_cookies
 from fides.api.models.sql_models import Cookies, PrivacyDeclaration
 from fides.api.models.sql_models import System as sql_System
+from fides.api.util.data_category import get_data_categories_map
 from fides.config import FidesConfig
 from fides.connectors.models import OktaConfig
 from fides.core import api
@@ -60,16 +61,28 @@ def test_get_system_data_uses(db, system) -> None:
     assert sql_System.get_data_uses([system]) == set()
 
 
-def test_system_datasets(system_with_dataset_references: System) -> None:
-    assert len(system_with_dataset_references.datasets) == 1
+def test_system_dataset_data_categories(
+    db,
+    system_with_a_single_dataset_reference: System,
+) -> None:
+    assert set(
+        system_with_a_single_dataset_reference.dataset_data_categories(
+            get_data_categories_map(db)
+        )
+    ) == {
+        "user.behavior",
+        "user.contact.address.street",
+        "user.name.first",
+        "user.unique_id",
+    }
 
 
 def test_system_undeclared_data_categories(
-    system_with_undeclared_data_categories: System,
+    db, system_with_undeclared_data_categories: System
 ) -> None:
-    assert system_with_undeclared_data_categories.undeclared_data_categories == {
-        "user.contact.email"
-    }
+    assert system_with_undeclared_data_categories.undeclared_data_categories(
+        get_data_categories_map(db)
+    ) == {"user.contact.email"}
 
 
 @pytest.fixture(scope="function")