diff --git a/CHANGELOG.md b/CHANGELOG.md index a962f4b20e..e782bafdeb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,11 +15,32 @@ The types of changes are: - `Fixed` for any bug fixes. - `Security` in case of vulnerabilities. -## [Unreleased](https://github.com/ethyca/fides/compare/2.51.0...main) +## [Unreleased](https://github.com/ethyca/fides/compare/2.51.2...main) +### Added +- New page in the Cookie House sample app to demonstrate the use of embedding the FidesJS SDK on the page [#5564](https://github.com/ethyca/fides/pull/5564) +- Added event based communication example to the Cookie House sample app [#5597](https://github.com/ethyca/fides/pull/5597) +- Added new erasure tests for BigQuery Enterprise [#5554](https://github.com/ethyca/fides/pull/5554) + +### Fixed +- Fixing quickstart.py script [#5585](https://github.com/ethyca/fides/pull/5585) +- Fixed miscellaneous performance issues with Systems and PrivacyDeclarations [#5601](https://github.com/ethyca/fides/pull/5601) + +### Changed +- Adjusted Ant's Select component colors and icon [#5594](https://github.com/ethyca/fides/pull/5594) + +## [2.51.2](https://github.com/ethyca/fides/compare/2.51.1...2.51.2) +### Fixed +- Fixed miscellaneous performance issues with Systems and PrivacyDeclarations [#5601](https://github.com/ethyca/fides/pull/5601) +## [2.51.1](https://github.com/ethyca/fides/compare/2.51.0...2.51.1) +### Fixed +- SaaS integrations using `oauth_client_credentials` now properly update their access token when editing the secrets. [#5548](https://github.com/ethyca/fides/pull/5548) +- Saas integrations using `oauth_client_credentials` now properly refresh their access token when the current token expires [#5569](https://github.com/ethyca/fides/pull/5569) +- Adding `dsr_testing_tools_enabled` security setting [#5573](https://github.com/ethyca/fides/pull/5573) +- Reverted elimination of connection pool in worker tasks to prevent DB performance issues [#5592](https://github.com/ethyca/fides/pull/5592) ## [2.51.0](https://github.com/ethyca/fides/compare/2.50.0...2.51.0) @@ -42,6 +63,7 @@ The types of changes are: - Updating dataset PUT to allow deleting all datasets [#5524](https://github.com/ethyca/fides/pull/5524) - Adds support for fides_key generation when parent_key is provided in Taxonomy create endpoints [#5542](https://github.com/ethyca/fides/pull/5542) - An integration will no longer re-enable after saving the connection form [#5555](https://github.com/ethyca/fides/pull/5555) +- Fixed positioning of Fides brand link in privacy center [#5572](https://github.com/ethyca/fides/pull/5572) ### Removed - Removed unnecessary debug logging from the load_file config helper [#5544](https://github.com/ethyca/fides/pull/5544) diff --git a/clients/admin-ui/src/features/datamap/reporting/DatamapReportTableColumns.tsx b/clients/admin-ui/src/features/datamap/reporting/DatamapReportTableColumns.tsx index a4dd638b51..e433f89723 100644 --- a/clients/admin-ui/src/features/datamap/reporting/DatamapReportTableColumns.tsx +++ b/clients/admin-ui/src/features/datamap/reporting/DatamapReportTableColumns.tsx @@ -426,49 +426,65 @@ export const getDatamapReportColumns = ({ columnHelper.accessor((row) => row.system_undeclared_data_categories, { id: COLUMN_IDS.SYSTEM_UNDECLARED_DATA_CATEGORIES, cell: (props) => { - const value = props.getValue(); - + const cellValues = props.getValue(); + if (!cellValues || cellValues.length === 0) { + return null; + } + const values = isArray(cellValues) + ? cellValues.map((value) => { + return { label: getDataCategoryDisplayName(value), key: value }; + }) + : [ + { + label: getDataCategoryDisplayName(cellValues), + key: cellValues, + }, + ]; return ( - ); }, meta: { showHeaderMenu: !isRenaming, + showHeaderMenuWrapOption: true, width: "auto", + overflow: "hidden", }, }), columnHelper.accessor((row) => row.data_use_undeclared_data_categories, { id: COLUMN_IDS.DATA_USE_UNDECLARED_DATA_CATEGORIES, cell: (props) => { - const value = props.getValue(); - + const cellValues = props.getValue(); + if (!cellValues || cellValues.length === 0) { + return null; + } + const values = isArray(cellValues) + ? cellValues.map((value) => { + return { label: getDataCategoryDisplayName(value), key: value }; + }) + : [ + { + label: getDataCategoryDisplayName(cellValues), + key: cellValues, + }, + ]; return ( - ); }, meta: { showHeaderMenu: !isRenaming, + showHeaderMenuWrapOption: true, width: "auto", + overflow: "hidden", }, }), columnHelper.accessor((row) => row.cookies, { diff --git a/clients/admin-ui/src/features/datastore-connections/system_portal_config/forms/ConnectorParametersForm.tsx b/clients/admin-ui/src/features/datastore-connections/system_portal_config/forms/ConnectorParametersForm.tsx index dae5bce4b1..92c881bd41 100644 --- a/clients/admin-ui/src/features/datastore-connections/system_portal_config/forms/ConnectorParametersForm.tsx +++ b/clients/admin-ui/src/features/datastore-connections/system_portal_config/forms/ConnectorParametersForm.tsx @@ -522,11 +522,13 @@ export const ConnectorParametersForm = ({ {testButtonLabel} ) : null} - {isPlusEnabled && !_.isEmpty(initialDatasets) && ( - onTestDatasetsClick()}> - Test datasets - - )} + {isPlusEnabled && + SystemType.DATABASE === connectionOption.type && + !_.isEmpty(initialDatasets) && ( + onTestDatasetsClick()}> + Test datasets + + )} {connectionOption.authorization_required && !authorized ? ( ( - - Powered by - -); +import { useSettings } from "~/features/common/settings.slice"; + +const BrandLink = ({ + position = "absolute", + right = 6, + ...props +}: LinkProps) => { + const { SHOW_BRAND_LINK } = useSettings(); + + if (!SHOW_BRAND_LINK) { + return null; + } + + return ( + + Powered by + + ); +}; export default BrandLink; diff --git a/clients/privacy-center/components/Layout.tsx b/clients/privacy-center/components/Layout.tsx index 9565fc7f20..e1e73a498b 100644 --- a/clients/privacy-center/components/Layout.tsx +++ b/clients/privacy-center/components/Layout.tsx @@ -2,16 +2,13 @@ import { Flex } from "fidesui"; import Head from "next/head"; import React, { ReactNode } from "react"; -import BrandLink from "~/components/BrandLink"; import Logo from "~/components/Logo"; import { useConfig } from "~/features/common/config.slice"; -import { useSettings } from "~/features/common/settings.slice"; import { useStyles } from "~/features/common/styles.slice"; const Layout = ({ children }: { children: ReactNode }) => { const config = useConfig(); const styles = useStyles(); - const { SHOW_BRAND_LINK } = useSettings(); return ( <> @@ -32,17 +29,7 @@ const Layout = ({ children }: { children: ReactNode }) => { - - {children} - {SHOW_BRAND_LINK && ( - - )} - + {children} > ); }; diff --git a/clients/privacy-center/components/consent/ConfigDrivenConsent.tsx b/clients/privacy-center/components/consent/ConfigDrivenConsent.tsx index f59e7bb4e3..3808605783 100644 --- a/clients/privacy-center/components/consent/ConfigDrivenConsent.tsx +++ b/clients/privacy-center/components/consent/ConfigDrivenConsent.tsx @@ -14,6 +14,7 @@ import { useAppDispatch, useAppSelector } from "~/app/hooks"; import { inspectForBrowserIdentities } from "~/common/browser-identities"; import { useLocalStorage } from "~/common/hooks"; import { ErrorToastOptions, SuccessToastOptions } from "~/common/toast-options"; +import BrandLink from "~/components/BrandLink"; import { useConfig } from "~/features/common/config.slice"; import { changeConsent, @@ -209,6 +210,7 @@ const ConfigDrivenConsent = ({ cancelLabel="Cancel" saveLabel="Save" /> + ); }; diff --git a/clients/privacy-center/components/consent/notice-driven/NoticeDrivenConsent.tsx b/clients/privacy-center/components/consent/notice-driven/NoticeDrivenConsent.tsx index d8765f587c..bd24228af2 100644 --- a/clients/privacy-center/components/consent/notice-driven/NoticeDrivenConsent.tsx +++ b/clients/privacy-center/components/consent/notice-driven/NoticeDrivenConsent.tsx @@ -25,6 +25,7 @@ import { inspectForBrowserIdentities } from "~/common/browser-identities"; import { useLocalStorage } from "~/common/hooks"; import useI18n from "~/common/hooks/useI18n"; import { ErrorToastOptions, SuccessToastOptions } from "~/common/toast-options"; +import BrandLink from "~/components/BrandLink"; import { useProperty } from "~/features/common/property.slice"; import { selectPrivacyExperience, @@ -400,7 +401,10 @@ const NoticeDrivenConsent = ({ base64Cookie }: { base64Cookie: boolean }) => { onCancel={handleCancel} justifyContent="center" /> - + + + + ); }; diff --git a/clients/privacy-center/pages/index.tsx b/clients/privacy-center/pages/index.tsx index 287a0b8f91..e7c21b22fa 100644 --- a/clients/privacy-center/pages/index.tsx +++ b/clients/privacy-center/pages/index.tsx @@ -13,6 +13,7 @@ import React, { useEffect, useState } from "react"; import { useAppDispatch, useAppSelector } from "~/app/hooks"; import { ConfigErrorToastOptions } from "~/common/toast-options"; +import BrandLink from "~/components/BrandLink"; import ConsentCard from "~/components/consent/ConsentCard"; import { ConsentRequestModal, @@ -25,7 +26,10 @@ import { } from "~/components/modals/privacy-request-modal/PrivacyRequestModal"; import PrivacyCard from "~/components/PrivacyCard"; import { useConfig } from "~/features/common/config.slice"; -import { selectIsNoticeDriven } from "~/features/common/settings.slice"; +import { + selectIsNoticeDriven, + useSettings, +} from "~/features/common/settings.slice"; import { clearLocation, selectPrivacyExperience, @@ -68,6 +72,10 @@ const Home: NextPage = () => { let isConsentModalOpen = isConsentModalOpenConst; const getIdVerificationConfigQuery = useGetIdVerificationConfigQuery(); + const { SHOW_BRAND_LINK } = useSettings(); + const showPrivacyPolicyLink = + !!config.privacy_policy_url && !!config.privacy_policy_url_text; + // Subscribe to experiences just to see if there are any notices. // The subscription automatically handles skipping if overlay is not enabled useSubscribeToPrivacyExperienceQuery(); @@ -214,19 +222,25 @@ const Home: NextPage = () => { {paragraph} ))} - {config.privacy_policy_url && config.privacy_policy_url_text ? ( - - {config.privacy_policy_url_text} - - ) : null} + + {(SHOW_BRAND_LINK || showPrivacyPolicyLink) && ( + + {showPrivacyPolicyLink && ( + + {config.privacy_policy_url_text} + + )} + + + )} Dict[str, Any]: + + if not CONFIG.security.dsr_testing_tools_enabled: + raise HTTPException( + status_code=HTTP_403_FORBIDDEN, + detail="DSR testing tools are not enabled.", + ) + dataset_config = DatasetConfig.filter( db=db, conditions=( diff --git a/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py b/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py index c8dc17a273..3bec34ffed 100644 --- a/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py +++ b/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py @@ -2,6 +2,7 @@ import csv import io +import json from collections import defaultdict from datetime import datetime from typing import ( @@ -148,6 +149,7 @@ from fides.api.util.enums import ColumnSort from fides.api.util.fuzzy_search_utils import get_decrypted_identities_automaton from fides.api.util.logger import Pii +from fides.api.util.storage_util import storage_json_encoder from fides.common.api.scope_registry import ( PRIVACY_REQUEST_CALLBACK_RESUME, PRIVACY_REQUEST_CREATE, @@ -2657,8 +2659,17 @@ def get_test_privacy_request_results( ) privacy_request.save(db=db) + # Escape datetime and ObjectId values + raw_data = privacy_request.get_raw_access_results() + escaped_json = json.dumps(raw_data, indent=2, default=storage_json_encoder) + results = json.loads(escaped_json) + return { "privacy_request_id": privacy_request.id, "status": privacy_request.status, - "results": privacy_request.get_raw_access_results(), + "results": ( + results + if CONFIG.security.dsr_testing_tools_enabled + else "DSR testing tools are not enabled, results will not be shown." + ), } diff --git a/src/fides/api/api/v1/endpoints/system.py b/src/fides/api/api/v1/endpoints/system.py index 0171265eda..266f23e1c5 100644 --- a/src/fides/api/api/v1/endpoints/system.py +++ b/src/fides/api/api/v1/endpoints/system.py @@ -19,7 +19,11 @@ from fides.api.api import deps from fides.api.api.v1.endpoints.saas_config_endpoints import instantiate_connection -from fides.api.db.crud import get_resource, get_resource_with_custom_fields +from fides.api.db.crud import ( + get_resource, + get_resource_with_custom_fields, + list_resource, +) from fides.api.db.ctl_session import get_async_db from fides.api.db.system import ( create_system, @@ -396,6 +400,20 @@ async def ls( # pylint: disable=invalid-name Otherwise all Systems will be returned (this may be a slow operation if there are many systems, so using the pagination parameters is recommended). """ + if not ( + size + or page + or search + or data_uses + or data_categories + or data_subjects + or dnd_relevant + or show_hidden + ): + # if no advanced parameters are passed, we return a very basic list of all System resources + # to maintain backward compatibility of the original API, which backs some important client usages, e.g. the fides CLI + + return await list_resource(System, db) query = select(System) @@ -447,19 +465,6 @@ async def ls( # pylint: disable=invalid-name # Add a distinct so we only get one row per system duplicates_removed = filtered_query.distinct(System.id) - if not ( - size - or page - or search - or data_uses - or data_categories - or data_subjects - or dnd_relevant - or show_hidden - ): - result = await db.execute(duplicates_removed) - return result.scalars().all() - return await async_paginate(db, duplicates_removed, pagination_params) diff --git a/src/fides/api/models/connectionconfig.py b/src/fides/api/models/connectionconfig.py index 0859c516b2..2dc518cd44 100644 --- a/src/fides/api/models/connectionconfig.py +++ b/src/fides/api/models/connectionconfig.py @@ -220,10 +220,13 @@ def authorized(self) -> bool: return False # hard-coding to avoid cyclic dependency - if authentication.strategy != "oauth2_authorization_code": + if authentication.strategy not in [ + "oauth2_authorization_code", + "oauth2_client_credentials", + ]: return False - return bool(self.secrets and self.secrets.get("access_token")) + return bool(self.secrets and "access_token" in self.secrets.keys()) @property def name_or_key(self) -> str: diff --git a/src/fides/api/models/sql_models.py b/src/fides/api/models/sql_models.py index f80abae5f9..2132ffe597 100644 --- a/src/fides/api/models/sql_models.py +++ b/src/fides/api/models/sql_models.py @@ -27,6 +27,7 @@ UniqueConstraint, case, cast, + func, select, text, type_coerce, @@ -36,7 +37,7 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_object_session from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm import Session, relationship -from sqlalchemy.sql import Select, func +from sqlalchemy.sql import Select from sqlalchemy.sql.elements import Case from sqlalchemy.sql.sqltypes import DateTime from typing_extensions import Protocol, runtime_checkable @@ -404,15 +405,6 @@ class System(Base, FidesBase): "Cookies", back_populates="system", lazy="selectin", uselist=True, viewonly=True ) - # index scan using ix_ctl_datasets_fides_key on ctl_datasets - datasets = relationship( - "Dataset", - primaryjoin="foreign(Dataset.fides_key)==any_(System.dataset_references)", - lazy="selectin", - uselist=True, - viewonly=True, - ) - @classmethod def get_data_uses( cls: Type[System], systems: List[System], include_parents: bool = True @@ -430,11 +422,21 @@ def get_data_uses( data_uses.add(data_use) return data_uses - @property - def undeclared_data_categories(self) -> Set[str]: + def dataset_data_categories(self, data_categories: Dict[str, Set[str]]) -> Set[str]: + aggregate = set() + for dataset_key in self.dataset_references or []: + aggregate.update(data_categories.get(dataset_key, set())) + return aggregate + + def undeclared_data_categories( + self, data_categories: Dict[str, Set[str]] + ) -> Set[str]: """ Returns a set of data categories defined on the system's datasets that are not associated with any data use (privacy declaration). + + Looks up the unique set of data categories for a given dataset from the pre-computed data_categories map. + This is done to improve performance. """ privacy_declaration_data_categories = set() @@ -443,9 +445,9 @@ def undeclared_data_categories(self) -> Set[str]: privacy_declaration.data_categories ) - system_dataset_data_categories = set() - for dataset in self.datasets: - system_dataset_data_categories.update(dataset.field_data_categories) + system_dataset_data_categories = set( + self.dataset_data_categories(data_categories) + ) return find_undeclared_categories( system_dataset_data_categories, privacy_declaration_data_categories @@ -501,13 +503,6 @@ class PrivacyDeclaration(Base): cookies = relationship( "Cookies", back_populates="privacy_declaration", lazy="joined", uselist=True ) - datasets = relationship( - "Dataset", - primaryjoin="foreign(Dataset.fides_key)==any_(PrivacyDeclaration.dataset_references)", - lazy="selectin", - uselist=True, - viewonly=True, - ) @classmethod def create( @@ -547,11 +542,21 @@ def purpose(cls) -> Case: else_=None, ) - @property - def undeclared_data_categories(self) -> Set[str]: + def dataset_data_categories(self, data_categories: Dict[str, Set[str]]) -> Set[str]: + aggregate = set() + for dataset_key in self.dataset_references or []: + aggregate.update(data_categories.get(dataset_key, set())) + return aggregate + + def undeclared_data_categories( + self, data_categories: Dict[str, Set[str]] + ) -> Set[str]: """ Aggregates a unique set of data categories across the collections in the associated datasets and returns the data categories that are not defined directly on this or any sibling privacy declarations. + + Looks up the unique set of data categories for a given dataset from the pre-computed data_categories map. + This is done to improve performance. """ # Note: This property evaluates the data categories attached to the datasets associated with this specific @@ -559,9 +564,7 @@ def undeclared_data_categories(self) -> Set[str]: # data categories across this privacy declaration and its sibling privacy declarations. # all data categories from the datasets - dataset_data_categories = set() - for dataset in self.datasets: - dataset_data_categories.update(dataset.field_data_categories) + dataset_data_categories = set(self.dataset_data_categories(data_categories)) # all data categories specified directly on this and sibling privacy declarations declared_data_categories = set() diff --git a/src/fides/api/schemas/privacy_request.py b/src/fides/api/schemas/privacy_request.py index 69ebcb2df5..c4e7445a25 100644 --- a/src/fides/api/schemas/privacy_request.py +++ b/src/fides/api/schemas/privacy_request.py @@ -401,4 +401,4 @@ class FilteredPrivacyRequestResults(FidesSchema): privacy_request_id: str status: PrivacyRequestStatus - results: Dict[str, Any] + results: Union[Dict[str, Any], str] diff --git a/src/fides/api/service/authentication/authentication_strategy_oauth2_client_credentials.py b/src/fides/api/service/authentication/authentication_strategy_oauth2_client_credentials.py index 70034b6901..f911e28049 100644 --- a/src/fides/api/service/authentication/authentication_strategy_oauth2_client_credentials.py +++ b/src/fides/api/service/authentication/authentication_strategy_oauth2_client_credentials.py @@ -33,3 +33,22 @@ def add_authentication( # add access_token to request request.headers["Authorization"] = "Bearer " + access_token return request + + def _refresh_token(self, connection_config: ConnectionConfig) -> str: + """ + Persists and returns a refreshed access_token if the token is close to expiring. + Otherwise just returns the existing access_token. + + For the Client Credentials OAuth flow we reuse the access request to get a new token. + """ + + expires_at = connection_config.secrets.get("expires_at") # type: ignore + if self._close_to_expiration(expires_at, connection_config): + refresh_response = self._call_token_request( + "refresh", self.token_request, connection_config + ) + return self._validate_and_store_response( + refresh_response, connection_config + ) + + return connection_config.secrets.get("access_token") # type: ignore diff --git a/src/fides/api/tasks/__init__.py b/src/fides/api/tasks/__init__.py index e757052ac1..ed5d9b8556 100644 --- a/src/fides/api/tasks/__init__.py +++ b/src/fides/api/tasks/__init__.py @@ -64,10 +64,11 @@ def get_new_session(self) -> ContextManager[Session]: if self._task_engine is None: self._task_engine = get_db_engine( config=CONFIG, + pool_size=CONFIG.database.task_engine_pool_size, + max_overflow=CONFIG.database.task_engine_max_overflow, keepalives_idle=CONFIG.database.task_engine_keepalives_idle, keepalives_interval=CONFIG.database.task_engine_keepalives_interval, keepalives_count=CONFIG.database.task_engine_keepalives_count, - disable_pooling=True, ) # same for the sessionmaker diff --git a/src/fides/api/util/data_category.py b/src/fides/api/util/data_category.py index 71c24cf366..5c533bd44f 100644 --- a/src/fides/api/util/data_category.py +++ b/src/fides/api/util/data_category.py @@ -1,14 +1,16 @@ from enum import Enum as EnumType -from typing import List, Type +from typing import Dict, List, Set, Type from fideslang.default_taxonomy import DEFAULT_TAXONOMY from fideslang.validation import FidesKey +from sqlalchemy import func, select, text from sqlalchemy.orm import Session from fides.api import common_exceptions from fides.api.models.sql_models import ( # type: ignore[attr-defined] # isort: skip DataCategory as DataCategoryDbModel, + Dataset, ) @@ -89,3 +91,31 @@ def filter_data_categories( } return sorted(list(default_categories)) return sorted(user_categories) + + +def get_data_categories_map(db: Session) -> Dict[str, Set[str]]: + """ + Returns a map of all datasets, where the keys are the fides keys + of each dataset and the value is a set of data categories associated with each dataset + """ + + subquery = ( + select( + Dataset.fides_key, + func.jsonb_array_elements_text( + text( + "jsonb_path_query(collections::jsonb, '$.** ? (@.data_categories != null).data_categories')" + ) + ).label("category"), + ).select_from(Dataset) + ).cte() + + query = ( + select( + [subquery.c.fides_key, func.array_agg(func.distinct(subquery.c.category))] + ) + .select_from(subquery) + .group_by(subquery.c.fides_key) + ) + result = db.execute(query) + return {key: set(value) if value else set() for key, value in result.all()} diff --git a/src/fides/config/security_settings.py b/src/fides/config/security_settings.py index 05ff855d60..8a5cdf426c 100644 --- a/src/fides/config/security_settings.py +++ b/src/fides/config/security_settings.py @@ -123,6 +123,10 @@ class SecuritySettings(FidesSettings): default=False, description="If set to True, the user interface will display a download button for subject requests.", ) + dsr_testing_tools_enabled: bool = Field( + default=False, + description="If set to True, contributor and owner roles will be able to run test privacy requests.", + ) subject_request_download_link_ttl_seconds: int = Field( default=432000, description="The number of seconds that a pre-signed download URL when using S3 storage will be valid. The default is equal to 5 days.", diff --git a/tests/conftest.py b/tests/conftest.py index d513472ee3..98e8f07f97 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1400,7 +1400,78 @@ def system_with_undeclared_data_categories(db: Session) -> System: @pytest.fixture(scope="function") -def privacy_declaration_with_dataset_references(db: Session) -> System: +def system_with_a_single_dataset_reference(db: Session) -> System: + first_dataset = CtlDataset.create_from_dataset_dict( + db, + { + "fides_key": f"dataset_key-f{uuid4()}", + "collections": [ + { + "name": "loyalty", + "fields": [ + { + "name": "id", + "data_categories": ["user.unique_id"], + }, + ], + } + ], + }, + ) + second_dataset = CtlDataset.create_from_dataset_dict( + db, + { + "fides_key": f"dataset_key-f{uuid4()}", + "collections": [ + { + "name": "customer", + "fields": [ + { + "name": "shipping_info", + "fields": [ + { + "name": "street", + "data_categories": ["user.contact.address.street"], + } + ], + }, + { + "name": "first_name", + "data_categories": ["user.name.first"], + }, + ], + }, + { + "name": "activity", + "fields": [ + { + "name": "last_login", + "data_categories": ["user.behavior"], + }, + ], + }, + ], + }, + ) + system = System.create( + db=db, + data={ + "fides_key": f"system_key-f{uuid4()}", + "name": f"system-{uuid4()}", + "description": "fixture-made-system", + "organization_fides_key": "default_organization", + "system_type": "Service", + "dataset_references": [first_dataset.fides_key, second_dataset.fides_key], + }, + ) + + return system + + +@pytest.fixture(scope="function") +def privacy_declaration_with_single_dataset_reference( + db: Session, +) -> PrivacyDeclaration: ctl_dataset = CtlDataset.create_from_dataset_dict( db, { @@ -1447,6 +1518,90 @@ def privacy_declaration_with_dataset_references(db: Session) -> System: return privacy_declaration +@pytest.fixture(scope="function") +def privacy_declaration_with_multiple_dataset_references( + db: Session, +) -> PrivacyDeclaration: + first_dataset = CtlDataset.create_from_dataset_dict( + db, + { + "fides_key": f"dataset_key-f{uuid4()}", + "collections": [ + { + "name": "loyalty", + "fields": [ + { + "name": "id", + "data_categories": ["user.unique_id"], + }, + ], + } + ], + }, + ) + second_dataset = CtlDataset.create_from_dataset_dict( + db, + { + "fides_key": f"dataset_key-f{uuid4()}", + "collections": [ + { + "name": "customer", + "fields": [ + { + "name": "shipping_info", + "fields": [ + { + "name": "street", + "data_categories": ["user.contact.address.street"], + } + ], + }, + { + "name": "first_name", + "data_categories": ["user.name.first"], + }, + ], + }, + { + "name": "activity", + "fields": [ + { + "name": "last_login", + "data_categories": ["user.behavior"], + }, + ], + }, + ], + }, + ) + system = System.create( + db=db, + data={ + "fides_key": f"system_key-f{uuid4()}", + "name": f"system-{uuid4()}", + "description": "fixture-made-system", + "organization_fides_key": "default_organization", + "system_type": "Service", + }, + ) + + privacy_declaration = PrivacyDeclaration.create( + db=db, + data={ + "name": "Collect data for third party sharing", + "system_id": system.id, + "data_categories": ["user.device.cookie_id"], + "data_use": "third_party_sharing", + "data_subjects": ["customer"], + "dataset_references": [first_dataset.fides_key, second_dataset.fides_key], + "egress": None, + "ingress": None, + }, + ) + + return privacy_declaration + + @pytest.fixture(scope="function") def system_multiple_decs(db: Session, system: System) -> Generator[System, None, None]: """ diff --git a/tests/ctl/core/test_api.py b/tests/ctl/core/test_api.py index bfca14dd84..1a47e32b80 100644 --- a/tests/ctl/core/test_api.py +++ b/tests/ctl/core/test_api.py @@ -1669,9 +1669,9 @@ def test_list_with_pagination_and_multiple_filters_2( "vendor_deleted_date, expected_systems_count, show_deleted", [ (datetime.now() - timedelta(days=1), 1, True), - (datetime.now() - timedelta(days=1), 0, None), - (datetime.now() + timedelta(days=1), 1, None), - (None, 1, None), + (datetime.now() - timedelta(days=1), 0, False), + (datetime.now() + timedelta(days=1), 1, False), + (None, 1, False), ], ) def test_vendor_deleted_systems( @@ -1691,13 +1691,13 @@ def test_vendor_deleted_systems( url=test_config.cli.server_url, headers=test_config.user.auth_header, resource_type="system", - query_params={"show_deleted": True} if show_deleted else {}, + query_params={"show_deleted": show_deleted, "size": 50}, ) assert result.status_code == 200 result_json = result.json() - assert len(result_json) == expected_systems_count + assert len(result_json["items"]) == expected_systems_count @pytest.mark.unit diff --git a/tests/ctl/core/test_privacy_declaration.py b/tests/ctl/core/test_privacy_declaration.py index ba2e11cdb2..0adc58949a 100644 --- a/tests/ctl/core/test_privacy_declaration.py +++ b/tests/ctl/core/test_privacy_declaration.py @@ -1,25 +1,51 @@ +from typing import Dict, Set + +import pytest + from fides.api.models.sql_models import PrivacyDeclaration +from fides.api.util.data_category import get_data_categories_map class TestPrivacyDeclaration: - def test_privacy_declaration_datasets( + + @pytest.fixture(scope="function") + def data_categories_map(self, db) -> Dict[str, Set[str]]: + return get_data_categories_map(db) + + def test_privacy_declaration_dataset_data_categories( self, - privacy_declaration_with_dataset_references: PrivacyDeclaration, - ) -> None: - assert len(privacy_declaration_with_dataset_references.datasets) == 1 + privacy_declaration_with_multiple_dataset_references: PrivacyDeclaration, + data_categories_map, + ): + assert set( + privacy_declaration_with_multiple_dataset_references.dataset_data_categories( + data_categories_map + ) + ) == { + "user.behavior", + "user.name.first", + "user.unique_id", + "user.contact.address.street", + } def test_privacy_declaration_undeclared_data_categories( - self, privacy_declaration_with_dataset_references + self, + privacy_declaration_with_single_dataset_reference: PrivacyDeclaration, + data_categories_map, ): - assert ( - privacy_declaration_with_dataset_references.undeclared_data_categories - == {"user.contact.email"} - ) + assert privacy_declaration_with_single_dataset_reference.undeclared_data_categories( + data_categories_map + ) == { + "user.contact.email" + } def test_privacy_declaration_data_category_defined_on_sibling( - self, db, privacy_declaration_with_dataset_references + self, + db, + privacy_declaration_with_single_dataset_reference: PrivacyDeclaration, + data_categories_map, ): - system = privacy_declaration_with_dataset_references.system + system = privacy_declaration_with_single_dataset_reference.system # Create a new privacy declaration with the data category we're searching for PrivacyDeclaration.create( @@ -41,6 +67,8 @@ def test_privacy_declaration_data_category_defined_on_sibling( # Check that the original privacy declaration doesn't have any undeclared data categories # because we also search sibling privacy declarations for the data category assert ( - privacy_declaration_with_dataset_references.undeclared_data_categories + privacy_declaration_with_single_dataset_reference.undeclared_data_categories( + data_categories_map + ) == set() ) diff --git a/tests/ctl/core/test_system.py b/tests/ctl/core/test_system.py index 8761c96915..e93363aef0 100644 --- a/tests/ctl/core/test_system.py +++ b/tests/ctl/core/test_system.py @@ -13,6 +13,7 @@ from fides.api.db.system import create_system, upsert_cookies from fides.api.models.sql_models import Cookies, PrivacyDeclaration from fides.api.models.sql_models import System as sql_System +from fides.api.util.data_category import get_data_categories_map from fides.config import FidesConfig from fides.connectors.models import OktaConfig from fides.core import api @@ -60,16 +61,28 @@ def test_get_system_data_uses(db, system) -> None: assert sql_System.get_data_uses([system]) == set() -def test_system_datasets(system_with_dataset_references: System) -> None: - assert len(system_with_dataset_references.datasets) == 1 +def test_system_dataset_data_categories( + db, + system_with_a_single_dataset_reference: System, +) -> None: + assert set( + system_with_a_single_dataset_reference.dataset_data_categories( + get_data_categories_map(db) + ) + ) == { + "user.behavior", + "user.contact.address.street", + "user.name.first", + "user.unique_id", + } def test_system_undeclared_data_categories( - system_with_undeclared_data_categories: System, + db, system_with_undeclared_data_categories: System ) -> None: - assert system_with_undeclared_data_categories.undeclared_data_categories == { - "user.contact.email" - } + assert system_with_undeclared_data_categories.undeclared_data_categories( + get_data_categories_map(db) + ) == {"user.contact.email"} @pytest.fixture(scope="function") diff --git a/tests/fixtures/application_fixtures.py b/tests/fixtures/application_fixtures.py index 04a3472696..54fe6f3784 100644 --- a/tests/fixtures/application_fixtures.py +++ b/tests/fixtures/application_fixtures.py @@ -3402,6 +3402,22 @@ def subject_request_download_ui_enabled(): CONFIG.security.subject_request_download_ui_enabled = original_value +@pytest.fixture(scope="function") +def dsr_testing_tools_enabled(): + original_value = CONFIG.security.dsr_testing_tools_enabled + CONFIG.security.dsr_testing_tools_enabled = True + yield + CONFIG.security.dsr_testing_tools_enabled = original_value + + +@pytest.fixture(scope="function") +def dsr_testing_tools_disabled(): + original_value = CONFIG.security.dsr_testing_tools_enabled + CONFIG.security.dsr_testing_tools_enabled = False + yield + CONFIG.security.dsr_testing_tools_enabled = original_value + + @pytest.fixture(scope="function") def system_with_no_uses(db: Session) -> Generator[System, None, None]: system = System.create( diff --git a/tests/ops/api/v1/endpoints/test_dataset_test_endpoints.py b/tests/ops/api/v1/endpoints/test_dataset_test_endpoints.py index c5056aec61..347ab9a4f1 100644 --- a/tests/ops/api/v1/endpoints/test_dataset_test_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_dataset_test_endpoints.py @@ -208,6 +208,7 @@ def test_dataset_test_not_authenticated( response = api_client.post(dataset_url + "/test", headers={}) assert response.status_code == 401 + @pytest.mark.usefixtures("dsr_testing_tools_enabled") def test_dataset_test_wrong_scope( self, dataset_config, @@ -220,7 +221,6 @@ def test_dataset_test_wrong_scope( response = api_client.post(dataset_url + "/test", headers=auth_header) assert response.status_code == 403 - @pytest.mark.usefixtures("default_access_policy") @pytest.mark.parametrize( "auth_header,expected_status", [ @@ -231,6 +231,7 @@ def test_dataset_test_wrong_scope( ("approver_auth_header", HTTP_403_FORBIDDEN), ], ) + @pytest.mark.usefixtures("default_access_policy", "dsr_testing_tools_enabled") def test_dataset_test_with_roles( self, dataset_config, @@ -249,6 +250,7 @@ def test_dataset_test_with_roles( ) assert response.status_code == expected_status + @pytest.mark.usefixtures("dsr_testing_tools_enabled") def test_dataset_test_connection_does_not_exist( self, api_client: TestClient, @@ -263,6 +265,7 @@ def test_dataset_test_connection_does_not_exist( ) assert response.status_code == 404 + @pytest.mark.usefixtures("dsr_testing_tools_enabled") def test_dataset_test_dataset_does_not_exist( self, connection_config, @@ -290,7 +293,7 @@ def test_dataset_test_dataset_does_not_exist( ), ], ) - @pytest.mark.usefixtures("default_access_policy") + @pytest.mark.usefixtures("default_access_policy", "dsr_testing_tools_enabled") def test_dataset_test_invalid_payloads( self, connection_config, @@ -310,7 +313,9 @@ def test_dataset_test_invalid_payloads( assert response.status_code == HTTP_400_BAD_REQUEST assert response.json()["detail"] == expected_response - @pytest.mark.usefixtures("default_access_policy", "postgres_integration_db") + @pytest.mark.usefixtures( + "default_access_policy", "postgres_integration_db", "dsr_testing_tools_enabled" + ) def test_dataset_test( self, connection_config, @@ -327,3 +332,23 @@ def test_dataset_test( ) assert response.status_code == HTTP_200_OK assert "privacy_request_id" in response.json().keys() + + @pytest.mark.usefixtures( + "default_access_policy", "postgres_integration_db", "dsr_testing_tools_disabled" + ) + def test_dataset_test_disabled( + self, + connection_config, + dataset_config, + api_client: TestClient, + generate_auth_header, + ) -> None: + dataset_url = get_connection_dataset_url(connection_config, dataset_config) + auth_header = generate_auth_header(scopes=[DATASET_TEST]) + response = api_client.post( + dataset_url + "/test", + headers=auth_header, + json={"email": "jane@example.com"}, + ) + assert response.status_code == HTTP_403_FORBIDDEN + assert response.json()["detail"] == "DSR testing tools are not enabled." diff --git a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py index 5ddbcd8f60..c4f386d7b0 100644 --- a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py @@ -8364,8 +8364,7 @@ def test_get_access_results_contributor_but_disabled( assert response.status_code == 403 -@pytest.mark.integration_external -@pytest.mark.integration_postgres +@pytest.mark.integration class TestPrivacyRequestFilteredResults: @pytest.fixture(scope="function") def default_access_policy(self, db) -> None: @@ -8393,7 +8392,7 @@ def test_filtered_results_wrong_scope( response = api_client.get(url, headers=auth_header) assert response.status_code == 403 - @pytest.mark.usefixtures("default_access_policy") + @pytest.mark.usefixtures("default_access_policy", "dsr_testing_tools_enabled") @pytest.mark.parametrize( "auth_header,expected_status", [ @@ -8427,15 +8426,60 @@ def test_filtered_results_with_roles( ) assert response.status_code == expected_status - @pytest.mark.usefixtures("default_access_policy", "postgres_integration_db") - def test_filtered_results( + @pytest.mark.integration_postgres + @pytest.mark.usefixtures( + "default_access_policy", "postgres_integration_db", "dsr_testing_tools_enabled" + ) + def test_filtered_results_postgres( + self, + connection_config, + postgres_example_test_dataset_config, + api_client: TestClient, + generate_auth_header, + ) -> None: + dataset_url = get_connection_dataset_url( + connection_config, postgres_example_test_dataset_config + ) + auth_header = generate_auth_header(scopes=[DATASET_TEST]) + response = api_client.post( + dataset_url + "/test", + headers=auth_header, + json={"email": "jane@example.com"}, + ) + assert response.status_code == HTTP_200_OK + + privacy_request_id = response.json()["privacy_request_id"] + url = V1_URL_PREFIX + PRIVACY_REQUEST_FILTERED_RESULTS.format( + privacy_request_id=privacy_request_id + ) + auth_header = generate_auth_header(scopes=[PRIVACY_REQUEST_READ_ACCESS_RESULTS]) + response = api_client.get( + url, + headers=auth_header, + ) + assert response.status_code == HTTP_200_OK + assert set(response.json().keys()) == { + "privacy_request_id", + "status", + "results", + } + + @pytest.mark.integration_postgres + @pytest.mark.usefixtures( + "default_access_policy", + "postgres_integration_db", + "dsr_testing_tools_enabled", + ) + def test_filtered_results_postgres_access_testing_disabled( self, connection_config, - dataset_config, + postgres_example_test_dataset_config, api_client: TestClient, generate_auth_header, ) -> None: - dataset_url = get_connection_dataset_url(connection_config, dataset_config) + dataset_url = get_connection_dataset_url( + connection_config, postgres_example_test_dataset_config + ) auth_header = generate_auth_header(scopes=[DATASET_TEST]) response = api_client.post( dataset_url + "/test", @@ -8444,6 +8488,54 @@ def test_filtered_results( ) assert response.status_code == HTTP_200_OK + original_value = CONFIG.security.dsr_testing_tools_enabled + CONFIG.security.dsr_testing_tools_enabled = False + + privacy_request_id = response.json()["privacy_request_id"] + url = V1_URL_PREFIX + PRIVACY_REQUEST_FILTERED_RESULTS.format( + privacy_request_id=privacy_request_id + ) + auth_header = generate_auth_header(scopes=[PRIVACY_REQUEST_READ_ACCESS_RESULTS]) + response = api_client.get( + url, + headers=auth_header, + ) + assert response.status_code == HTTP_200_OK + assert set(response.json().keys()) == { + "privacy_request_id", + "status", + "results", + } + assert ( + response.json()["results"] + == "DSR testing tools are not enabled, results will not be shown." + ) + + CONFIG.security.dsr_testing_tools_enabled = original_value + + @pytest.mark.integration_mongo + @pytest.mark.usefixtures("default_access_policy", "dsr_testing_tools_enabled") + def test_filtered_results_mongo( + self, + mongo_connection_config, + mongo_dataset_config, + api_client: TestClient, + generate_auth_header, + ) -> None: + dataset_url = get_connection_dataset_url( + mongo_connection_config, mongo_dataset_config + ) + auth_header = generate_auth_header(scopes=[DATASET_TEST]) + response = api_client.post( + dataset_url + "/test", + headers=auth_header, + json={ + "email": "employee-1@example.com", + "postgres_example_test_dataset:customer:id": 1, + }, + ) + assert response.status_code == HTTP_200_OK + privacy_request_id = response.json()["privacy_request_id"] url = V1_URL_PREFIX + PRIVACY_REQUEST_FILTERED_RESULTS.format( privacy_request_id=privacy_request_id diff --git a/tests/ops/service/authentication/test_authentication_strategy_oauth2_client_credentials.py b/tests/ops/service/authentication/test_authentication_strategy_oauth2_client_credentials.py index 5ac18977b8..d3d19c3e0d 100644 --- a/tests/ops/service/authentication/test_authentication_strategy_oauth2_client_credentials.py +++ b/tests/ops/service/authentication/test_authentication_strategy_oauth2_client_credentials.py @@ -40,23 +40,7 @@ def oauth2_client_credentials_configuration() -> ( {"name": "client_secret", "value": ""}, {"name": "grant_type", "value": "client_credentials"}, ], - }, - "refresh_request": { - "method": "POST", - "path": "/oauth/token", - "headers": [ - { - "name": "Content-Type", - "value": "application/x-www-form-urlencoded", - } - ], - "query_params": [ - {"name": "client_id", "value": ""}, - {"name": "client_secret", "value": ""}, - {"name": "grant_type", "value": "refresh_token"}, - {"name": "refresh_token", "value": ""}, - ], - }, + } } @@ -69,7 +53,6 @@ def oauth2_client_credentials_connection_config( "client_id": "client", "client_secret": "secret", "access_token": "access", - "refresh_token": "refresh", } saas_config = { "fides_key": "oauth2_client_credentials_connector", @@ -249,7 +232,6 @@ def test_oauth2_authentication_successful_refresh( "client_id": "client", "client_secret": "secret", "access_token": "new_access", - "refresh_token": "refresh", "expires_at": 0, } }, @@ -327,7 +309,6 @@ def test_get_access_token( expires_in = 7200 mock_send().json.return_value = { "access_token": "new_access", - "refresh_token": "new_refresh", "expires_in": expires_in, } @@ -347,7 +328,6 @@ def test_get_access_token( "client_id": "client", "client_secret": "secret", "access_token": "new_access", - "refresh_token": "new_refresh", "expires_at": int(datetime.utcnow().timestamp()) + expires_in, } }, @@ -376,7 +356,6 @@ def test_get_access_token_no_expires_in( # mock the json response from calling the access token request mock_send().json.return_value = { "access_token": "new_access", - "refresh_token": "new_refresh", } oauth2_client_credentials_configuration["expires_in"] = 3600 @@ -396,7 +375,6 @@ def test_get_access_token_no_expires_in( "client_id": "client", "client_secret": "secret", "access_token": "new_access", - "refresh_token": "new_refresh", "expires_at": int(datetime.utcnow().timestamp()) + oauth2_client_credentials_configuration["expires_in"], } diff --git a/tests/ops/tasks/test_database_task.py b/tests/ops/tasks/test_database_task.py index 0076f16f84..c2907ade17 100644 --- a/tests/ops/tasks/test_database_task.py +++ b/tests/ops/tasks/test_database_task.py @@ -42,21 +42,6 @@ def always_failing_session_maker(self): mock_maker.side_effect = OperationalError("connection failed", None, None) return mock_maker - @pytest.mark.parametrize( - "config_fixture", [None, "mock_config_changed_db_engine_settings"] - ) - def test_get_task_session(self, config_fixture, request): - if config_fixture is not None: - request.getfixturevalue( - config_fixture - ) # used to invoke config fixture if provided - pool_size = CONFIG.database.task_engine_pool_size - max_overflow = CONFIG.database.task_engine_max_overflow - t = DatabaseTask() - session: Session = t.get_new_session() - engine: Engine = session.get_bind() - assert isinstance(engine.pool, NullPool) - def test_retry_on_operational_error(self, recovering_session_maker): """Test that session creation retries on OperationalError"""