Skip to content

Commit

Permalink
[Integration][Snyk] - Migrate API from v1 to REST (#828)
Browse files Browse the repository at this point in the history 
# Description

What - Snyk has announced that they will burn down their v1 API by end
of Q3, and all subsequent calls to their v1 APIs will return 410 (gone).
This PR migrates all instances of our Snyk exporter that relies on the
v1 API, and moves them to the REST API. The PR contains the following:

1. Migration of v1 API to REST API
2. Added additional kind called `vulnerabilities`. and this will be used
to ingest Snyk vulnerabilities for new customers. To support backwards
compatibility, the existing `issue` kind is maintained, and will be
switched for the vulnerabilities after Q3 when we have communicated the
deprecation notice to client
3. Moved calculation of low|medium|high|critical open vulnerabilities on
the Target blueprint to Aggregation property to reduce making calls to
Snyk.
4. Other blueprint and mapping changes
 
Why -
How - Get issues, get organizations, get users etc are now using REST
API

## Type of change

Please leave one option from the following and delete the rest:

- [ ] Bug fix (non-breaking change which fixes an issue)
- [ ] New feature (non-breaking change which adds functionality)
- [ ] New Integration (non-breaking change which adds a new integration)
- [x] Breaking change (fix or feature that would cause existing
functionality to not work as expected)
- [ ] Non-breaking change (fix of existing functionality that will not
change current behavior)
- [ ] Documentation (added/updated documentation)

## Screenshots

Include screenshots from your environment showing how the resources of
the integration will look.

## API Documentation

[Issues
API](https://apidocs.snyk.io/?version=2024-06-21#get-/orgs/-org_id-/issues)
[Users
API](https://apidocs.snyk.io/?version=2024-06-21%7Ebeta#get-/orgs/-org_id-/users/-id-)

## API SAMPLE RESPONSE
Issues from REST API
```json
{
  "id": "0ecc9091-c1cc-4107-a6cb-d66bb7935ede",
  "type": "issue",
  "attributes": {
    "classes": [
      {
        "id": "CWE-94",
        "source": "CWE",
        "type": "weakness"
      }
    ],
    "coordinates": [
      {
        "representations": [
          {
            "dependency": {
              "package_name": "vm2",
              "package_version": "3.9.19"
            }
          }
        ]
      }
    ],
    "created_at": "2023-09-03T15:02:41.211Z",
    "effective_severity_level": "critical",
    "ignored": false,
    "key": "SNYK-JS-VM2-5772825",
    "problems": [
      {
        "id": "CVE-2023-37466",
        "source": "SNYK",
        "type": "vulnerability",
        "updated_at": "2024-01-12T07:35:26.623212Z"
      },
      {
        "id": "SNYK-JS-VM2-5772825",
        "source": "SNYK",
        "type": "vulnerability",
        "updated_at": "2024-01-12T07:35:26.623212Z"
      }
    ],
    "risk": {
      "factors": [],
      "score": {
        "model": "v1",
        "value": 597
      }
    },
    "status": "open",
    "title": "Remote Code Execution (RCE)",
    "type": "package_vulnerability",
    "updated_at": "2023-09-03T15:02:41.211Z"
  },
  "relationships": {
    "organization": {
      "data": {
        "id": "7c5908e4d4d2",
        "type": "organization"
      },
      "links": {
        "related": "/orgs/7c5908e4d4d2"
      }
    },
    "scan_item": {
      "data": {
        "id": "36f9fbb0-caf2-48a7-bbac-e7c1dc093f37",
        "type": "project"
      },
      "links": {
        "related": "/orgs/7c5908e4d4d2/projects/36f9fbb0-caf2-48a7-bbac-e7c1dc093f37"
      }
    }
  }
}
```

Organizations from REST API

```json
    {
      "id": "7c5908e4d4d2",
      "type": "org",
      "attributes": {
        "group_id": "49270cfc54b5",
        "is_personal": false,
        "name": "Port NFR - Shared",
        "slug": "port-nfr-shared"
      },
      "relationships": {
        "member_role": {
          "data": {
            "id": "9409d7b4f883",
            "type": "org_role"
          }
        }
      }
    }
```
  • Loading branch information
@PeyGis
PeyGis authored Jul 25, 2024
1 parent 0735687 commit 021d893
Show file tree
Hide file tree
Showing 4 changed files with 200 additions and 89 deletions.
118 changes: 81 additions & 37 deletions integrations/snyk/.port/resources/blueprints.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,12 @@
"type": "number",
"title": "Score"
},
"packageName": {
"type": "string",
"title": "Package Name",
"packageNames": {
"items": {
"type": "string"
},
"type": "array",
"title": "Package Names",
"icon": "DefaultProperty"
},
"packageVersions": {
Expand All @@ -45,20 +48,30 @@
"type": "array"
},
"type": {
"type": "string",
"icon": "DefaultProperty",
"title": "Type",
"type": "string",
"enum": [
"vuln",
"license",
"configuration"
],
"icon": "DefaultProperty"
"configuration",
"config",
"custom",
"code",
"cloud",
"package_vulnerability"
]
},
"severity": {
"icon": "Alert",
"title": "Issue Severity",
"type": "string",
"enum": ["low", "medium", "high", "critical"],
"enum": [
"low",
"medium",
"high",
"critical"
],
"enumColors": {
"low": "green",
"medium": "yellow",
Expand All @@ -72,21 +85,24 @@
"title": "Issue URL",
"format": "url"
},
"language": {
"type": "string",
"title": "Language",
"icon": "DefaultProperty"
},
"publicationTime": {
"type": "string",
"format": "date-time",
"title": "Publication Time",
"icon": "DefaultProperty"
},
"isPatched": {
"type": "boolean",
"title": "Is Patched",
"icon": "DefaultProperty"
"status": {
"title": "Status",
"icon": "",
"type": "string",
"enum": [
"open",
"resolved"
],
"enumColors": {
"open": "red",
"resolved": "green"
}
}
},
"required": []
Expand Down Expand Up @@ -160,6 +176,11 @@
},
"icon": "DefaultProperty"
},
"criticalOpenVulnerabilities": {
"icon": "Vulnerability",
"type": "number",
"title": "Open Critical Vulnerabilities"
},
"highOpenVulnerabilities": {
"icon": "Vulnerability",
"type": "number",
Expand Down Expand Up @@ -205,26 +226,6 @@
"icon": "Snyk",
"schema": {
"properties": {
"criticalOpenVulnerabilities": {
"icon": "Vulnerability",
"type": "number",
"title": "Open Critical Vulnerabilities"
},
"highOpenVulnerabilities": {
"icon": "Vulnerability",
"type": "number",
"title": "Open High Vulnerabilities"
},
"mediumOpenVulnerabilities": {
"icon": "Vulnerability",
"type": "number",
"title": "Open Medium Vulnerabilities"
},
"lowOpenVulnerabilities": {
"icon": "Vulnerability",
"type": "number",
"title": "Open Low Vulnerabilities"
},
"origin": {
"title": "Target Origin",
"type": "string",
Expand Down Expand Up @@ -266,6 +267,49 @@
},
"mirrorProperties": {},
"calculationProperties": {},
"aggregationProperties": {
"open_critical_vulnerabilities": {
"title": "Open Critical Vulnerabilities",
"type": "number",
"target": "snykProject",
"calculationSpec": {
"func": "sum",
"property": "criticalOpenVulnerabilities",
"calculationBy": "property"
}
},
"open_high_vulnerabilities": {
"title": "Open High Vulnerabilities",
"type": "number",
"target": "snykProject",
"calculationSpec": {
"func": "sum",
"property": "highOpenVulnerabilities",
"calculationBy": "property"
}
},
"open_medium_vulnerabilities": {
"title": "Open Medium Vulnerabilities",
"type": "number",
"target": "snykProject",
"calculationSpec": {
"func": "sum",
"property": "mediumOpenVulnerabilities",
"calculationBy": "property"
}
},
"open_low_vulnerabilities": {
"title": "Open Low Vulnerabilities",
"icon": "DefaultProperty",
"type": "number",
"target": "snykProject",
"calculationSpec": {
"func": "sum",
"property": "lowOpenVulnerabilities",
"calculationBy": "property"
}
}
},
"relations": {
"synk_organization": {
"title": "Snyk Organization",
Expand Down
51 changes: 23 additions & 28 deletions integrations/snyk/.port/resources/port-app-config.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,32 +6,11 @@ resources:
entity:
mappings:
identifier: .id
title: .name
title: .attributes.name
blueprint: '"snykOrganization"'
properties:
slug: .slug
url: ("https://app.snyk.io/org/" + .slug | tostring)
- kind: issue
selector:
query: '.issueType == "vuln"'
port:
entity:
mappings:
identifier: .issueData.id + "-" + (.links.paths | split("/") | .[8])
title: .issueData.title
blueprint: '"snykVulnerability"'
properties:
score: .priorityScore
packageName: .pkgName
packageVersions: .pkgVersions
type: .issueType
severity: .issueData.severity
url: .issueData.url
language: .issueData.language // .issueType
publicationTime: .issueData.publicationTime
isPatched: .isPatched
relations:
project: '.links.paths | split("/") | .[8]'
slug: .attributes.slug
url: ("https://app.snyk.io/org/" + .attributes.slug | tostring)
- kind: project
selector:
query: 'true'
Expand Down Expand Up @@ -66,9 +45,25 @@ resources:
blueprint: '"snykTarget"'
properties:
origin: .relationships.integration.data.attributes.integration_type
highOpenVulnerabilities: '[.__projects[].meta.latest_issue_counts.high] | add'
mediumOpenVulnerabilities: '[.__projects[].meta.latest_issue_counts.medium] | add'
lowOpenVulnerabilities: '[.__projects[].meta.latest_issue_counts.low] | add'
criticalOpenVulnerabilities: '[.__projects[].meta.latest_issue_counts.critical] | add'
relations:
synk_organization: '.relationships.organization.data.id'
- kind: vulnerability
selector:
query: 'true'
port:
entity:
mappings:
identifier: .id
title: .attributes.title
blueprint: '"snykVulnerability"'
properties:
score: .attributes.risk.score.value
packageNames: '[.attributes.coordinates[].representations[].dependency?.package_name | select(. != null)]'
packageVersions: '[.attributes.coordinates[].representations[].dependency?.package_version | select(. != null)]'
severity: .attributes.effective_severity_level
url: ("https://app.snyk.io/org/" + .attributes.key | tostring)
publicationTime: .attributes.created_at
status: .attributes.status
type: .attributes.type
relations:
project: .relationships.scan_item.data.id
14 changes: 14 additions & 0 deletions integrations/snyk/main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ class ObjectKind(StrEnum):
PROJECT = "project"
ISSUE = "issue"
TARGET = "target"
VULNERABILITY = "vulnerability"


async def verify_signature(request: Request, secret: str) -> bool:
Expand Down Expand Up @@ -97,6 +98,10 @@ async def on_issues_resync(kind: str) -> list[dict[str, Any]]:
snyk_client = init_client()
all_issues: list[dict[str, Any]] = []

logger.warning(
"This kind will be deprecated at the end of Q3, in favour of our new data model for Snyk resources. This change is necessary because Snyk has announced a migration and end of life of their v1 API to focus on their REST API. Refer to our documentation for more information: https://docs.getport.io/build-your-software-catalog/sync-data-to-catalog/code-quality-security/snyk/#issue"
)

semaphore = asyncio.Semaphore(CONCURRENT_REQUESTS)

async for projects in snyk_client.get_paginated_projects():
Expand All @@ -111,6 +116,15 @@ async def on_issues_resync(kind: str) -> list[dict[str, Any]]:
return list({issue["id"]: issue for issue in all_issues}.values())


@ocean.on_resync(ObjectKind.VULNERABILITY)
async def on_vulnerability_resync(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE:
snyk_client = init_client()

async for issues_batch in snyk_client.get_paginated_issues():
logger.debug(f"Received batch with {len(issues_batch)} issues")
yield issues_batch


@ocean.router.post("/webhook")
async def on_vulnerability_webhook_handler(request: Request) -> None:
verify_signature_result = await verify_signature(
Expand Down
Loading

0 comments on commit 021d893

Please sign in to comment.