Skip to content

Commit

Permalink
qualys_vmdr: retain event.original as json (#11248)
Browse files Browse the repository at this point in the history 
In real use of the API, the XML document that is sent may be hundreds of
megabytes resulting in inability to ingest the resulting event documents. Even
with compression, the XML is on the order of tens of megabytes. So do not retain
the full XML, but only the original generated JSON when preserve_original_event
is enabled.
  • Loading branch information
@efd6
efd6 authored Sep 26, 2024
1 parent 9d46e70 commit 72195af
Show file tree
Hide file tree
Showing 10 changed files with 72 additions and 67 deletions.
5 changes: 5 additions & 0 deletions packages/qualys_vmdr/changelog.yml
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
# newer versions go on top
- version: "5.2.0"
changes:
- description: Retain event.original for asset_host_detection and knowledge_base as JSON.
type: enhancement
link: https://github.com/elastic/integrations/pull/11248
- version: "5.1.0"
changes:
- description: Set `vulnerability.score.base` field based on the item *CVSS* item under field `qualys_vmdr.asset_host_detection.vulnerability.qds_factors`
Expand Down
1 change: 0 additions & 1 deletion packages/qualys_vmdr/data_stream/asset_host_detection/agent/stream/input.yml.hbs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@ program: |
resp.Body.as(xml, bytes(xml).decode_xml('qualys_api_2_0').as(body, {
"events": body.doc.HOST_LIST_VM_DETECTION_OUTPUT.RESPONSE.HOST_LIST.HOST.map(h,
h.DETECTION_LIST.DETECTION.map(v, {
?"xml": state.keep_xml ? optional.of(string(xml)) : optional.none(),
"message": h.with({"DETECTION_LIST": v}).encode_json(),
})
).flatten(),
Expand Down
11 changes: 6 additions & 5 deletions ...es/qualys_vmdr/data_stream/asset_host_detection/elasticsearch/ingest_pipeline/default.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,11 +50,6 @@ processors:
field: vulnerability.scanner.vendor
tag: set_vulnerability_scanner_vendor
value: Qualys
- rename:
field: xml
target_field: event.original
ignore_missing: true
ignore_failure: true
- json:
field: message
tag: json_message
Expand Down Expand Up @@ -1687,6 +1682,12 @@ processors:
- append:
field: error.message
value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
- rename:
field: message
target_field: event.original
ignore_missing: true
ignore_failure: true
if: ctx.tags?.contains('preserve_original_event') == true
- remove:
tag: remove_json
field:
Expand Down
20 changes: 10 additions & 10 deletions packages/qualys_vmdr/data_stream/asset_host_detection/sample_event.json
View file
Original file line number Diff line number Diff line change
@@ -1,34 +1,34 @@
{
"@timestamp": "2024-09-12T13:03:16.792Z",
"@timestamp": "2024-09-25T21:44:26.325Z",
"agent": {
"ephemeral_id": "c4aefbcf-119f-4228-973a-57ecff367d67",
"id": "85d6cfe5-6af1-495f-a383-a783a5f1ebe6",
"name": "elastic-agent-14313",
"ephemeral_id": "f8145b5b-4d53-444a-bd44-2f296cf357e6",
"id": "efcbf604-6e25-41db-a21e-22c8227e0663",
"name": "elastic-agent-93250",
"type": "filebeat",
"version": "8.15.1"
"version": "8.13.0"
},
"data_stream": {
"dataset": "qualys_vmdr.asset_host_detection",
"namespace": "21446",
"namespace": "88572",
"type": "logs"
},
"ecs": {
"version": "8.11.0"
},
"elastic_agent": {
"id": "85d6cfe5-6af1-495f-a383-a783a5f1ebe6",
"id": "efcbf604-6e25-41db-a21e-22c8227e0663",
"snapshot": false,
"version": "8.15.1"
"version": "8.13.0"
},
"event": {
"agent_id_status": "verified",
"category": [
"host"
],
"dataset": "qualys_vmdr.asset_host_detection",
"ingested": "2024-09-12T13:03:19Z",
"ingested": "2024-09-25T21:44:29Z",
"kind": "alert",
"original": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!DOCTYPE HOST_LIST_VM_DETECTION_OUTPUT SYSTEM \"https://qualysapi.qualys.com/api/2.0/fo/asset/host/vm/detection/dtd/output.dtd\">\n<HOST_LIST_VM_DETECTION_OUTPUT>\n <RESPONSE>\n <DATETIME>2023-07-03T06:51:41Z</DATETIME>\n <HOST_LIST>\n <HOST>\n <ID>12048633</ID>\n <IP>10.50.2.111</IP>\n <TRACKING_METHOD>IP</TRACKING_METHOD>\n <OS>\n <![CDATA[Windows 2016/2019/10]]>\n </OS>\n <DNS>\n <![CDATA[adfssrvr.adfs.local]]>\n </DNS>\n <DNS_DATA>\n <HOSTNAME>\n <![CDATA[adfssrvr]]>\n </HOSTNAME>\n <DOMAIN>\n <![CDATA[adfs.local]]>\n </DOMAIN>\n <FQDN>\n <![CDATA[adfssrvr.adfs.local]]>\n </FQDN>\n </DNS_DATA>\n <NETBIOS>\n <![CDATA[ADFSSRVR]]>\n </NETBIOS>\n <LAST_SCAN_DATETIME>2023-07-03T06:25:17Z</LAST_SCAN_DATETIME>\n <LAST_VM_SCANNED_DATE>2023-07-03T06:23:47Z</LAST_VM_SCANNED_DATE>\n <LAST_VM_SCANNED_DURATION>1113</LAST_VM_SCANNED_DURATION>\n <LAST_PC_SCANNED_DATE>2023-06-28T09:58:12Z</LAST_PC_SCANNED_DATE>\n <DETECTION_LIST>\n <DETECTION>\n <UNIQUE_VULN_ID>5555555555</UNIQUE_VULN_ID>\n <QID>197595</QID>\n <TYPE>Confirmed</TYPE>\n <SEVERITY>3</SEVERITY>\n <SSL>0</SSL>\n <RESULTS><![CDATA[Package Installed Version Required Version\nlinux-cloud-tools-4.4.0 1074-aws_4.4.0-1074.84 1092\nlinux-aws-tools-4.4.0 1074_4.4.0-1074.84 1092\nlinux-aws-headers-4.4.0 1074_4.15.0-1126.135 1092\nlinux-tools-4.4.0 1074-aws_4.4.0-1074.84 1092\nlinux-aws-cloud-tools-4.4.0 1074_4.4.0-1074.84 1092]]></RESULTS>\n <STATUS>Active</STATUS>\n <FIRST_FOUND_DATETIME>2021-02-05T04:50:45Z</FIRST_FOUND_DATETIME>\n <LAST_FOUND_DATETIME>2024-03-08T20:15:41Z</LAST_FOUND_DATETIME>\n <QDS severity=\"LOW\">35</QDS>\n <QDS_FACTORS>\n <QDS_FACTOR name=\"CVSS\"><![CDATA[7.7]]></QDS_FACTOR>\n <QDS_FACTOR name=\"CVSS_version\"><![CDATA[v3.x]]></QDS_FACTOR>\n <QDS_FACTOR name=\"epss\"><![CDATA[0.00232]]></QDS_FACTOR>\n <QDS_FACTOR name=\"CVSS_vector\"><![CDATA[AV:N/AC:L/PR:L/UI:N/S:C/C:N/I:N/A:H]]></QDS_FACTOR>\n </QDS_FACTORS>\n <TIMES_FOUND>5393</TIMES_FOUND>\n <LAST_TEST_DATETIME>2024-03-08T20:15:41Z</LAST_TEST_DATETIME>\n <LAST_UPDATE_DATETIME>2024-03-08T20:15:41Z</LAST_UPDATE_DATETIME>\n <LAST_FIXED_DATETIME>2022-12-14T06:52:57Z</LAST_FIXED_DATETIME>\n <IS_IGNORED>0</IS_IGNORED>\n <IS_DISABLED>0</IS_DISABLED>\n <AFFECT_RUNNING_KERNEL>0</AFFECT_RUNNING_KERNEL>\n <LAST_PROCESSED_DATETIME>2024-03-08T20:15:41Z</LAST_PROCESSED_DATETIME>\n </DETECTION>\n <DETECTION>\n <UNIQUE_VULN_ID>6666666666</UNIQUE_VULN_ID>\n <QID>197597</QID>\n <TYPE>Confirmed</TYPE>\n <SEVERITY>5</SEVERITY>\n <SSL>0</SSL>\n <RESULTS><![CDATA[Package Installed Version Required Version\nlinux-image-4.15.0 1027-aws_4.15.0-1126.135 1047\nlinux-headers-4.15.0 1027-aws_4.15.0-1126.135 1047\nlinux-modules-4.15.0 1027-aws_4.15.0-1126.135 1047\nlinux-aws-headers-4.15.0 1027_4.15.0-1126.135 1047]]></RESULTS>\n <STATUS>Active</STATUS>\n <FIRST_FOUND_DATETIME>2021-02-05T04:50:45Z</FIRST_FOUND_DATETIME>\n <LAST_FOUND_DATETIME>2024-03-08T20:15:41Z</LAST_FOUND_DATETIME>\n <QDS severity=\"CRITICAL\">95</QDS>\n <QDS_FACTORS>\n <QDS_FACTOR name=\"RTI\"><![CDATA[local]]></QDS_FACTOR>\n <QDS_FACTOR name=\"exploit_maturity\"><![CDATA[weaponized,poc]]></QDS_FACTOR>\n <QDS_FACTOR name=\"CISA_vuln\"><![CDATA[YES]]></QDS_FACTOR>\n <QDS_FACTOR name=\"CVSS\"><![CDATA[7.8]]></QDS_FACTOR>\n <QDS_FACTOR name=\"CVSS_version\"><![CDATA[v3.x]]></QDS_FACTOR>\n <QDS_FACTOR name=\"epss\"><![CDATA[0.00052]]></QDS_FACTOR>\n <QDS_FACTOR name=\"trending\"><![CDATA[02222024,02162024,02262024,02152024,02012024,02252024,02212024,02282024,02102024,02062024,02082024,02042024,02052024]]></QDS_FACTOR>\n <QDS_FACTOR name=\"CVSS_vector\"><![CDATA[AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H]]></QDS_FACTOR>\n <QDS_FACTOR name=\"mitigation_controls\"><![CDATA[18436,18437]]></QDS_FACTOR>\n </QDS_FACTORS>\n <TIMES_FOUND>5393</TIMES_FOUND>\n <LAST_TEST_DATETIME>2024-03-08T20:15:41Z</LAST_TEST_DATETIME>\n <LAST_UPDATE_DATETIME>2024-03-08T20:15:41Z</LAST_UPDATE_DATETIME>\n <LAST_FIXED_DATETIME>2022-12-14T06:52:57Z</LAST_FIXED_DATETIME>\n <IS_IGNORED>0</IS_IGNORED>\n <IS_DISABLED>0</IS_DISABLED>\n <AFFECT_RUNNING_KERNEL>0</AFFECT_RUNNING_KERNEL>\n <LAST_PROCESSED_DATETIME>2024-03-08T20:15:41Z</LAST_PROCESSED_DATETIME>\n </DETECTION>\n </DETECTION_LIST>\n </HOST>\n </HOST_LIST>\n <WARNING>\n <CODE>1980</CODE>\n <TEXT>1000 record limit exceeded. Use URL to get next batch of results.</TEXT>\n <URL><![CDATA[http://qualys_vmdr:8090/api/2.0/fo/asset/host/vm/detection/?action=list&truncation_limit=1000&id_min=5641289]]></URL>\n </WARNING>\n </RESPONSE>\n</HOST_LIST_VM_DETECTION_OUTPUT>",
"original": "{\"DETECTION_LIST\":{\"AFFECT_RUNNING_KERNEL\":\"0\",\"FIRST_FOUND_DATETIME\":\"2021-02-05T04:50:45Z\",\"IS_DISABLED\":\"0\",\"IS_IGNORED\":\"0\",\"LAST_FIXED_DATETIME\":\"2022-12-14T06:52:57Z\",\"LAST_FOUND_DATETIME\":\"2024-03-08T20:15:41Z\",\"LAST_PROCESSED_DATETIME\":\"2024-03-08T20:15:41Z\",\"LAST_TEST_DATETIME\":\"2024-03-08T20:15:41Z\",\"LAST_UPDATE_DATETIME\":\"2024-03-08T20:15:41Z\",\"QDS\":{\"#text\":\"35\",\"severity\":\"LOW\"},\"QDS_FACTORS\":{\"QDS_FACTOR\":[{\"#text\":\"7.7\",\"name\":\"CVSS\"},{\"#text\":\"v3.x\",\"name\":\"CVSS_version\"},{\"#text\":\"0.00232\",\"name\":\"epss\"},{\"#text\":\"AV:N/AC:L/PR:L/UI:N/S:C/C:N/I:N/A:H\",\"name\":\"CVSS_vector\"}]},\"QID\":\"197595\",\"RESULTS\":\"Package Installed Version Required Version\\nlinux-cloud-tools-4.4.0 1074-aws_4.4.0-1074.84 1092\\nlinux-aws-tools-4.4.0 1074_4.4.0-1074.84 1092\\nlinux-aws-headers-4.4.0 1074_4.15.0-1126.135 1092\\nlinux-tools-4.4.0 1074-aws_4.4.0-1074.84 1092\\nlinux-aws-cloud-tools-4.4.0 1074_4.4.0-1074.84 1092\",\"SEVERITY\":\"3\",\"SSL\":\"0\",\"STATUS\":\"Active\",\"TIMES_FOUND\":\"5393\",\"TYPE\":\"Confirmed\",\"UNIQUE_VULN_ID\":\"5555555555\"},\"DNS\":\"\",\"DNS_DATA\":{\"DOMAIN\":\"\",\"FQDN\":\"\",\"HOSTNAME\":\"\"},\"ID\":\"12048633\",\"IP\":\"10.50.2.111\",\"LAST_PC_SCANNED_DATE\":\"2023-06-28T09:58:12Z\",\"LAST_SCAN_DATETIME\":\"2023-07-03T06:25:17Z\",\"LAST_VM_SCANNED_DATE\":\"2023-07-03T06:23:47Z\",\"LAST_VM_SCANNED_DURATION\":\"1113\",\"NETBIOS\":\"\",\"OS\":\"\",\"TRACKING_METHOD\":\"IP\"}",
"type": [
"info"
]
Expand Down
1 change: 0 additions & 1 deletion packages/qualys_vmdr/data_stream/knowledge_base/agent/stream/input.yml.hbs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@ program: |
has(body.doc.KNOWLEDGE_BASE_VULN_LIST_OUTPUT.RESPONSE.VULN_LIST)
?
body.doc.KNOWLEDGE_BASE_VULN_LIST_OUTPUT.RESPONSE.VULN_LIST.VULN.map(e, {
?"xml": state.keep_xml ? optional.of(string(xml)) : optional.none(),
"message": e.with({
"CVE_LIST": e.?CVE_LIST.CVE.orValue([]).map(c, c.ID),
}).encode_json()
Expand Down
11 changes: 6 additions & 5 deletions packages/qualys_vmdr/data_stream/knowledge_base/elasticsearch/ingest_pipeline/default.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,6 @@ processors:
field: event.type
tag: set_event_type
value: [info]
- rename:
field: xml
target_field: event.original
ignore_missing: true
ignore_failure: true
- json:
field: message
tag: json_message
Expand Down Expand Up @@ -650,6 +645,12 @@ processors:
tag: set_vulnerability_severity
copy_from: json.SEVERITY_LEVEL
ignore_empty_value: true
- rename:
field: message
target_field: event.original
ignore_missing: true
ignore_failure: true
if: ctx.tags?.contains('preserve_original_event') == true
- remove:
tag: remove_json
field:
Expand Down
18 changes: 9 additions & 9 deletions packages/qualys_vmdr/data_stream/knowledge_base/sample_event.json
View file
Original file line number Diff line number Diff line change
@@ -1,24 +1,24 @@
{
"@timestamp": "2023-06-29T12:20:46.000Z",
"agent": {
"ephemeral_id": "4457f1c5-914c-421b-9e5b-8d65febb3a2d",
"id": "20adbc1d-c590-4efe-a604-f426ab6cf7f2",
"name": "elastic-agent-78527",
"ephemeral_id": "4e6d92f6-8a28-471c-a03f-8c2685171b7b",
"id": "dc86e78e-6670-441f-acdd-99309474050f",
"name": "elastic-agent-65730",
"type": "filebeat",
"version": "8.15.1"
"version": "8.13.0"
},
"data_stream": {
"dataset": "qualys_vmdr.knowledge_base",
"namespace": "14856",
"namespace": "47901",
"type": "logs"
},
"ecs": {
"version": "8.11.0"
},
"elastic_agent": {
"id": "20adbc1d-c590-4efe-a604-f426ab6cf7f2",
"id": "dc86e78e-6670-441f-acdd-99309474050f",
"snapshot": false,
"version": "8.15.1"
"version": "8.13.0"
},
"event": {
"agent_id_status": "verified",
Expand All @@ -27,9 +27,9 @@
],
"dataset": "qualys_vmdr.knowledge_base",
"id": "11830",
"ingested": "2024-09-12T13:05:24Z",
"ingested": "2024-09-25T21:49:31Z",
"kind": "alert",
"original": "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n<!DOCTYPE KNOWLEDGE_BASE_VULN_LIST_OUTPUT SYSTEM \"https://qualysapi.qualys.com/api/2.0/fo/knowledge_base/vuln/knowledge_base_vuln_list_output.dtd\">\n<KNOWLEDGE_BASE_VULN_LIST_OUTPUT>\n <RESPONSE>\n <DATETIME>2023-07-06T15:02:16Z</DATETIME>\n <VULN_LIST>\n <VULN>\n <QID>11830</QID>\n <VULN_TYPE>Vulnerability</VULN_TYPE>\n <SEVERITY_LEVEL>2</SEVERITY_LEVEL>\n <TITLE>\n <![CDATA[HTTP Security Header Not Detected]]>\n </TITLE>\n <CVE_LIST>\n <CVE>\n <ID><![CDATA[CVE-2022-31629]]></ID>\n <URL><![CDATA[http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-31629]]></URL>\n </CVE>\n <CVE>\n <ID><![CDATA[CVE-2022-31628]]></ID>\n <URL><![CDATA[http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-31628]]></URL>\n </CVE>\n </CVE_LIST>\n <CATEGORY>CGI</CATEGORY>\n <LAST_SERVICE_MODIFICATION_DATETIME>2023-06-29T12:20:46Z</LAST_SERVICE_MODIFICATION_DATETIME>\n <PUBLISHED_DATETIME>2017-06-05T21:34:49Z</PUBLISHED_DATETIME>\n <PATCHABLE>0</PATCHABLE>\n <SOFTWARE_LIST>\n <SOFTWARE>\n <PRODUCT>\n <![CDATA[None]]>\n </PRODUCT>\n <VENDOR>\n <![CDATA[multi-vendor]]>\n </VENDOR>\n </SOFTWARE>\n </SOFTWARE_LIST>\n <DIAGNOSIS>\n <![CDATA[This QID reports the absence of the following]]>\n </DIAGNOSIS>\n <CONSEQUENCE>\n <![CDATA[Depending on the vulnerability being exploited, an unauthenticated remote attacker could conduct cross-site scripting, clickjacking or MIME-type sniffing attacks.]]>\n </CONSEQUENCE>\n <SOLUTION>\n <![CDATA[<B>Note:</B> To better debug the results of this QID]]>\n </SOLUTION>\n <PCI_FLAG>1</PCI_FLAG>\n <THREAT_INTELLIGENCE>\n <THREAT_INTEL id=\"8\">\n <![CDATA[No_Patch]]>\n </THREAT_INTEL>\n </THREAT_INTELLIGENCE>\n <DISCOVERY>\n <REMOTE>1</REMOTE>\n </DISCOVERY>\n </VULN>\n </VULN_LIST>\n </RESPONSE>\n</KNOWLEDGE_BASE_VULN_LIST_OUTPUT>",
"original": "{\"CATEGORY\":\"CGI\",\"CONSEQUENCE\":\"\",\"CVE_LIST\":[\"CVE-2022-31629\",\"CVE-2022-31628\"],\"DIAGNOSIS\":\"\",\"DISCOVERY\":{\"REMOTE\":\"1\"},\"LAST_SERVICE_MODIFICATION_DATETIME\":\"2023-06-29T12:20:46Z\",\"PATCHABLE\":\"0\",\"PCI_FLAG\":\"1\",\"PUBLISHED_DATETIME\":\"2017-06-05T21:34:49Z\",\"QID\":\"11830\",\"SEVERITY_LEVEL\":\"2\",\"SOFTWARE_LIST\":{\"SOFTWARE\":[{\"PRODUCT\":\"\",\"VENDOR\":\"\"}]},\"SOLUTION\":\"\",\"THREAT_INTELLIGENCE\":{\"THREAT_INTEL\":[{\"id\":\"8\"}]},\"TITLE\":\"\",\"VULN_TYPE\":\"Vulnerability\"}",
"type": [
"info"
]
Expand Down
16 changes: 8 additions & 8 deletions packages/qualys_vmdr/data_stream/user_activity/sample_event.json
View file
Original file line number Diff line number Diff line change
@@ -1,24 +1,24 @@
{
"@timestamp": "2024-01-18T12:45:24.000Z",
"agent": {
"ephemeral_id": "875168b0-f16d-41de-95c3-91523d61558f",
"id": "d7c96817-7b92-4a82-a129-723e4fe5a829",
"name": "elastic-agent-95908",
"ephemeral_id": "8541dd66-de0a-4e54-a66e-3f9dc02867df",
"id": "3acf31e6-1468-482c-b38b-d3b7397270dd",
"name": "elastic-agent-32349",
"type": "filebeat",
"version": "8.15.1"
"version": "8.13.0"
},
"data_stream": {
"dataset": "qualys_vmdr.user_activity",
"namespace": "55195",
"namespace": "28709",
"type": "logs"
},
"ecs": {
"version": "8.11.0"
},
"elastic_agent": {
"id": "d7c96817-7b92-4a82-a129-723e4fe5a829",
"id": "3acf31e6-1468-482c-b38b-d3b7397270dd",
"snapshot": false,
"version": "8.15.1"
"version": "8.13.0"
},
"event": {
"action": "request",
Expand All @@ -27,7 +27,7 @@
"api"
],
"dataset": "qualys_vmdr.user_activity",
"ingested": "2024-09-12T13:07:54Z",
"ingested": "2024-09-25T21:52:05Z",
"kind": "event",
"original": "{\"Action\":\"request\",\"Date\":\"2024-01-18T12:45:24Z\",\"Details\":\"API: /api/2.0/fo/activity_log/index.php\",\"Module\":\"auth\",\"User IP\":\"10.113.195.136\",\"User Name\":\"john\",\"User Role\":\"Reader\"}",
"provider": "auth",
Expand Down
Loading

0 comments on commit 72195af

Please sign in to comment.