HJ-87 and HJ-88: model changes

.fides/db_dataset.yml

@@ -2227,6 +2227,8 @@ dataset:
       data_categories: [system]
     - name: user_assigned_data_categories
       data_categories: [system]
+    - name: data_uses
+      data_categories: [system]
   - name: fides_user_invite
     fields:
       - name: created_at

CHANGELOG.md

@@ -21,6 +21,7 @@ The types of changes are:
 - New page in the Cookie House sample app to demonstrate the use of embedding the FidesJS SDK on the page [#5564](https://github.com/ethyca/fides/pull/5564)
 - Added event based communication example to the Cookie House sample app [#5597](https://github.com/ethyca/fides/pull/5597)
 - Added new erasure tests for BigQuery Enterprise [#5554](https://github.com/ethyca/fides/pull/5554)
+- Migration to add the `hidden` and `data_use` columns to `stagedresource` table, prereqs for Data Catalog work in Fidesplus [#5600](https://github.com/ethyca/fides/pull/5600/)
 
 ### Changed
 - Adjusted Ant's Select component colors and icon [#5594](https://github.com/ethyca/fides/pull/5594)

src/fides/api/alembic/migrations/versions/d9237a0c0d5a_add_hidden_column_to_stagedresource.py

@@ -0,0 +1,28 @@
+"""add `data_uses` columns to stagedresource
+
+Revision ID: d9237a0c0d5a
+Revises: c90d46f6d3f2
+Create Date: 2024-11-21 13:18:24.085858
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "d9237a0c0d5a"
+down_revision = "c90d46f6d3f2"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column(
+        "stagedresource", sa.Column("data_uses", sa.ARRAY(sa.String), nullable=True)
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    op.drop_column("stagedresource", "data_uses")
+    # ### end Alembic commands ###

src/fides/api/api/v1/endpoints/system.py

@@ -391,54 +391,22 @@ async def ls(  # pylint: disable=invalid-name
     data_uses: Optional[List[FidesKey]] = Query(None),
     data_categories: Optional[List[FidesKey]] = Query(None),
     data_subjects: Optional[List[FidesKey]] = Query(None),
-    dnd_relevant: Optional[bool] = Query(None),
-    show_hidden: Optional[bool] = Query(False),
     show_deleted: Optional[bool] = Query(False),
 ) -> List:
     """Get a list of all of the Systems.
     If any parameters or filters are provided the response will be paginated and/or filtered.
     Otherwise all Systems will be returned (this may be a slow operation if there are many systems,
     so using the pagination parameters is recommended).
     """
-    if not (
-        size
-        or page
-        or search
-        or data_uses
-        or data_categories
-        or data_subjects
-        or dnd_relevant
-        or show_hidden
-    ):
-        # if no advanced parameters are passed, we return a very basic list of all System resources
-        # to maintain backward compatibility of the original API, which backs some important client usages, e.g. the fides CLI
-
+    if not (size or page or search or data_uses or data_categories or data_subjects):
         return await list_resource(System, db)
 
-    query = select(System)
-
     pagination_params = Params(page=page or 1, size=size or 50)
     # Need to join with PrivacyDeclaration in order to be able to filter
     # by data use, data category, and data subject
-    if any([data_uses, data_categories, data_subjects]):
-        query = query.outerjoin(
-            PrivacyDeclaration, System.id == PrivacyDeclaration.system_id
-        )
-
-    # Fetch any system that is relevant for Detection and Discovery, ie any of the following:
-    # - has connection configurations (has some integration for DnD or SaaS)
-    # - has dataset references
-    if dnd_relevant:
-        query = query.filter(
-            (System.connection_configs != None)  # pylint: disable=singleton-comparison
-            | (System.dataset_references.any())
-        )
-
-    # Filter out any hidden systems, unless explicilty asked for
-    if not show_hidden:
-        query = query.filter(
-            System.hidden == False  # pylint: disable=singleton-comparison
-        )
+    query = select(System).outerjoin(
+        PrivacyDeclaration, System.id == PrivacyDeclaration.system_id
+    )
 
     # Filter out any vendor deleted systems, unless explicitly asked for
     if not show_deleted:
@@ -464,10 +432,38 @@ async def ls(  # pylint: disable=invalid-name
 
     # Add a distinct so we only get one row per system
     duplicates_removed = filtered_query.distinct(System.id)
-
     return await async_paginate(db, duplicates_removed, pagination_params)
 
 
+@SYSTEM_ROUTER.patch(
+    "/hidden",
+    response_model=Dict,
+    dependencies=[
+        Security(
+            verify_oauth_client_prod,
+            scopes=[SYSTEM_UPDATE],
+        )
+    ],
+)
+async def patch_hidden(
+    fides_keys: List[str],
+    hidden: bool,
+    db: AsyncSession = Depends(get_async_db),
+) -> Dict:
+    """
+    Patch the hidden status of a list of systems. Request body must be a list of system Fides keys.
+    """
+    systems = await db.execute(select(System).filter(System.fides_key.in_(fides_keys)))
+    systems = systems.scalars().all()
+    for system in systems:
+        system.hidden = hidden
+    await db.commit()
+    return {
+        "message": "Updated hidden status for systems",
+        "updated": len(systems),
+    }
+
+
 @SYSTEM_ROUTER.get(
     "/{fides_key}",
     dependencies=[

src/fides/api/models/detection_discovery.py

@@ -4,12 +4,14 @@
 from enum import Enum
 from typing import Any, Dict, Iterable, List, Optional, Type
 
+from loguru import logger
 from sqlalchemy import ARRAY, Boolean, Column, DateTime, ForeignKey, String
 from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.ext.mutable import MutableDict
 from sqlalchemy.future import select
 from sqlalchemy.orm import Session, relationship
+from sqlalchemy.orm.query import Query
 
 from fides.api.db.base_class import Base, FidesBase
 from fides.api.models.connectionconfig import ConnectionConfig
@@ -278,6 +280,13 @@ class StagedResource(Base):
         default=dict,
     )
 
+    data_uses = Column(
+        ARRAY(String),
+        nullable=False,
+        server_default="{}",
+        default=dict,
+    )
+
     @classmethod
     def get_urn(cls, db: Session, urn: str) -> Optional[StagedResource]:
         """Utility to retrieve the staged resource with the given URN"""
@@ -337,3 +346,24 @@ def mark_as_addition(
             )
             if parent_resource:
                 parent_resource.add_child_diff_status(DiffStatus.ADDITION)
+
+
+def fetch_staged_resources_by_type_query(
+    resource_type: str,
+    monitor_config_ids: Optional[List[str]] = None,
+    show_hidden: bool = False,
+) -> Query[StagedResource]:
+    """
+    Fetches staged resources by type and monitor config ID. Optionally filters out muted staged resources ("hidden").
+    """
+    logger.info(
+        f"Fetching staged resources of type {resource_type}, show_hidden={show_hidden}, monitor_config_ids={monitor_config_ids}"
+    )
+    query = select(StagedResource).where(StagedResource.resource_type == resource_type)
+
+    if monitor_config_ids:
+        query = query.filter(StagedResource.monitor_config_id.in_(monitor_config_ids))
+    if not show_hidden:
+        query = query.where(StagedResource.diff_status != "muted")
+
+    return query

tests/ctl/core/test_api.py

@@ -2,7 +2,7 @@
 """Integration tests for the API module."""
 import json
 import typing
-from datetime import datetime, timedelta, timezone
+from datetime import datetime, timezone
 from json import loads
 from typing import Dict, List, Tuple
 from uuid import uuid4
@@ -1536,102 +1536,6 @@ def test_list_with_pagination_and_multiple_filters(
 
         assert result_json["items"][0]["fides_key"] == tcf_system.fides_key
 
-    def test_list_with_dnd_filter(
-        self,
-        test_config,
-        system_with_cleanup,  # one that has a connection config
-        system_third_party_sharing,  # one that doesn't have a connection config
-    ):
-        result = _api.ls(
-            url=test_config.cli.server_url,
-            headers=test_config.user.auth_header,
-            resource_type="system",
-            query_params={
-                "page": 1,
-                "size": 5,
-                "dnd_relevant": "true",
-            },
-        )
-
-        assert result.status_code == 200
-        result_json = result.json()
-        assert result_json["total"] == 1
-        assert len(result_json["items"]) == 1
-
-        # only "system_with_cleanup" has a connection config attached to it in fixtures
-        assert result_json["items"][0]["fides_key"] == system_with_cleanup.fides_key
-
-    def test_list_with_show_hidden(
-        self,
-        test_config,
-        system_hidden,
-        system_with_cleanup,
-    ):
-
-        result = _api.ls(
-            url=test_config.cli.server_url,
-            headers=test_config.user.auth_header,
-            resource_type="system",
-            query_params={
-                "page": 1,
-                "size": 5,
-                "show_hidden": "true",
-            },
-        )
-
-        assert result.status_code == 200
-        result_json = result.json()
-        assert result_json["total"] == 2
-        assert len(result_json["items"]) == 2
-
-        actual_keys = [item["fides_key"] for item in result_json["items"]]
-        assert system_hidden.fides_key in actual_keys
-        assert system_with_cleanup.fides_key in actual_keys
-
-        result = _api.ls(
-            url=test_config.cli.server_url,
-            headers=test_config.user.auth_header,
-            resource_type="system",
-            query_params={
-                "page": 1,
-                "size": 5,
-                "show_hidden": "false",
-            },
-        )
-
-        assert result.status_code == 200
-        result_json = result.json()
-        assert result_json["total"] == 1
-        assert len(result_json["items"]) == 1
-
-        assert result_json["items"][0]["fides_key"] == system_with_cleanup.fides_key
-
-    def test_list_with_show_hidden_and_dnd_relevant(
-        self,
-        test_config,
-        system_hidden,
-        system_with_cleanup,
-    ):
-
-        result = _api.ls(
-            url=test_config.cli.server_url,
-            headers=test_config.user.auth_header,
-            resource_type="system",
-            query_params={
-                "page": 1,
-                "size": 5,
-                "show_hidden": "true",
-                "dnd_relevant": "true",
-            },
-        )
-
-        assert result.status_code == 200
-        result_json = result.json()
-        assert result_json["total"] == 1
-        assert len(result_json["items"]) == 1
-
-        assert result_json["items"][0]["fides_key"] == system_with_cleanup.fides_key
-
     @pytest.mark.skip("Until we re-visit filter implementation")
     def test_list_with_pagination_and_multiple_filters_2(
         self,
@@ -1665,40 +1569,6 @@ def test_list_with_pagination_and_multiple_filters_2(
 
         assert result_json["items"][0]["fides_key"] == tcf_system.fides_key
 
-    @pytest.mark.parametrize(
-        "vendor_deleted_date, expected_systems_count, show_deleted",
-        [
-            (datetime.now() - timedelta(days=1), 1, True),
-            (datetime.now() - timedelta(days=1), 0, False),
-            (datetime.now() + timedelta(days=1), 1, False),
-            (None, 1, False),
-        ],
-    )
-    def test_vendor_deleted_systems(
-        self,
-        db,
-        test_config,
-        system_with_cleanup,
-        vendor_deleted_date,
-        expected_systems_count,
-        show_deleted,
-    ):
-
-        system_with_cleanup.vendor_deleted_date = vendor_deleted_date
-        db.commit()
-
-        result = _api.ls(
-            url=test_config.cli.server_url,
-            headers=test_config.user.auth_header,
-            resource_type="system",
-            query_params={"show_deleted": show_deleted, "size": 50},
-        )
-
-        assert result.status_code == 200
-        result_json = result.json()
-
-        assert len(result_json["items"]) == expected_systems_count
-
 
 @pytest.mark.unit
 class TestSystemUpdate:

tests/ops/api/v1/endpoints/test_system.py

@@ -31,6 +31,7 @@
     SAAS_CONNECTION_INSTANTIATE,
     STORAGE_DELETE,
     SYSTEM_MANAGER_UPDATE,
+    SYSTEM_UPDATE,
 )
 from fides.common.api.v1.urn_registry import V1_URL_PREFIX
 from tests.conftest import generate_role_header_for_user
@@ -273,6 +274,26 @@ def test_patch_connection_secrets_removes_access_token_for_client_config(
         assert resp.status_code == HTTP_200_OK
         assert resp.json()["items"][0]["authorized"] is False
 
+    def test_system_patch_hidden(
+        self,
+        system,
+        api_client: TestClient,
+        generate_auth_header,
+    ):
+        url = V1_URL_PREFIX + f"/system/hidden"
+        auth_header = generate_auth_header(scopes=[SYSTEM_UPDATE])
+
+        result = api_client.patch(
+            url=f"{url}?hidden=true",
+            headers=auth_header,
+            json=[system.fides_key],
+        )
+        assert result.status_code == HTTP_200_OK
+        assert result.json() == {
+            "message": "Updated hidden status for systems",
+            "updated": 1,
+        }
+
 
 class TestGetConnections:
     def test_get_connections_not_authenticated(