From 19a1972e2a95c43ca42588725666dc77befd4a86 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 11 Apr 2023 23:53:24 +0000 Subject: [PATCH 01/76] remove obsolete tests and data --- .../shadowserver/scan_rdpeudp.csv.license | 2 - .../parsers/shadowserver/test_blocklist.py | 103 ------- .../shadowserver/test_compromised_website.py | 88 ------ .../parsers/shadowserver/test_device_id.py | 116 -------- .../test_event4_ddos_participant.py | 131 --------- .../test_event4_honeypot_darknet.py | 106 ------- .../shadowserver/test_event4_honeypot_ddos.py | 148 ---------- .../test_event4_honeypot_ddos_target.py | 150 ---------- .../test_event4_honeypot_http_scan.py | 109 -------- .../shadowserver/test_event4_ip_spoofer.py | 182 ------------ .../test_event4_microsoft_sinkhole.py | 135 --------- .../test_event4_microsoft_sinkhole_http.py | 202 -------------- .../shadowserver/test_event4_sinkhole.py | 73 ----- .../shadowserver/test_event4_sinkhole_dns.py | 127 --------- .../shadowserver/test_event4_sinkhole_http.py | 189 ------------- .../test_event4_sinkhole_http_referer.py | 213 --------------- .../shadowserver/test_event6_sinkhole_http.py | 146 ---------- .../shadowserver/test_honeypot_brute_force.py | 72 ----- .../shadowserver/test_honeypot_ddos_amp.py | 91 ------ .../parsers/shadowserver/test_malware_url.py | 107 -------- .../parsers/shadowserver/test_phish_url.py | 106 ------- .../test_population_http_proxy.py | 130 --------- .../parsers/shadowserver/test_sandbox_conn.py | 99 ------- .../parsers/shadowserver/test_sandbox_dns.py | 95 ------- .../parsers/shadowserver/test_sandbox_url.py | 104 ------- .../parsers/shadowserver/test_scan_adb.py | 98 ------- .../parsers/shadowserver/test_scan_afp.py | 106 ------- .../parsers/shadowserver/test_scan_amqp.py | 144 ---------- .../parsers/shadowserver/test_scan_ard.py | 111 -------- .../parsers/shadowserver/test_scan_chargen.py | 110 -------- .../test_scan_cisco_smart_install.py | 82 ------ .../parsers/shadowserver/test_scan_coap.py | 121 -------- .../parsers/shadowserver/test_scan_couchdb.py | 128 --------- .../parsers/shadowserver/test_scan_cwmp.py | 103 ------- .../parsers/shadowserver/test_scan_db2.py | 91 ------ .../shadowserver/test_scan_ddos_middlebox.py | 119 -------- .../parsers/shadowserver/test_scan_dns.py | 91 ------ .../parsers/shadowserver/test_scan_docker.py | 159 ----------- .../test_scan_dvr_dhcpdiscover.py | 178 ------------ .../shadowserver/test_scan_elasticsearch.py | 126 --------- .../shadowserver/test_scan_exchange.py | 149 ---------- .../parsers/shadowserver/test_scan_ftp.py | 120 -------- .../parsers/shadowserver/test_scan_hadoop.py | 94 ------- .../parsers/shadowserver/test_scan_http.py | 100 ------- .../shadowserver/test_scan_http_proxy.py | 118 -------- .../shadowserver/test_scan_http_vulnerable.py | 125 --------- .../parsers/shadowserver/test_scan_ics.py | 125 --------- .../parsers/shadowserver/test_scan_ipmi.py | 106 ------- .../parsers/shadowserver/test_scan_ipp.py | 79 ------ .../parsers/shadowserver/test_scan_isakmp.py | 105 ------- .../shadowserver/test_scan_kubernetes.py | 214 --------------- .../shadowserver/test_scan_ldap_tcp.py | 154 ----------- .../shadowserver/test_scan_ldap_udp.py | 162 ----------- .../parsers/shadowserver/test_scan_mdns.py | 127 --------- .../shadowserver/test_scan_memcached.py | 130 --------- .../parsers/shadowserver/test_scan_mongodb.py | 103 ------- .../parsers/shadowserver/test_scan_mqtt.py | 89 ------ .../shadowserver/test_scan_mqtt_anon.py | 173 ------------ .../parsers/shadowserver/test_scan_mssql.py | 123 --------- .../parsers/shadowserver/test_scan_mysql.py | 258 ------------------ .../parsers/shadowserver/test_scan_nat_pmp.py | 116 -------- .../parsers/shadowserver/test_scan_netbios.py | 121 -------- .../shadowserver/test_scan_netis_router.py | 107 -------- .../parsers/shadowserver/test_scan_ntp.py | 161 ----------- .../shadowserver/test_scan_ntpmonitor.py | 108 -------- .../shadowserver/test_scan_portmapper.py | 120 -------- .../shadowserver/test_scan_postgres.py | 199 -------------- .../parsers/shadowserver/test_scan_qotd.py | 119 -------- .../parsers/shadowserver/test_scan_quic.py | 118 -------- .../parsers/shadowserver/test_scan_radmin.py | 236 ---------------- .../parsers/shadowserver/test_scan_rdp.py | 117 -------- .../parsers/shadowserver/test_scan_rdpeudp.py | 109 -------- .../parsers/shadowserver/test_scan_redis.py | 107 -------- .../parsers/shadowserver/test_scan_rsync.py | 116 -------- .../parsers/shadowserver/test_scan_sip.py | 124 --------- .../parsers/shadowserver/test_scan_slp.py | 137 ---------- .../parsers/shadowserver/test_scan_smb.py | 124 --------- .../shadowserver/test_scan_smb_json.py | 123 --------- .../shadowserver/test_scan_smtp_vulnerable.py | 92 ------- .../parsers/shadowserver/test_scan_snmp.py | 120 -------- .../parsers/shadowserver/test_scan_socks.py | 107 -------- .../parsers/shadowserver/test_scan_ssdp.py | 136 --------- .../parsers/shadowserver/test_scan_ssh.py | 182 ------------ .../parsers/shadowserver/test_scan_ssl.py | 218 --------------- .../shadowserver/test_scan_ssl_freak.py | 136 --------- .../shadowserver/test_scan_ssl_poodle.py | 91 ------ .../parsers/shadowserver/test_scan_stun.py | 146 ---------- .../shadowserver/test_scan_synfulknock.py | 117 -------- .../parsers/shadowserver/test_scan_telnet.py | 87 ------ .../parsers/shadowserver/test_scan_tftp.py | 121 -------- .../shadowserver/test_scan_ubiquiti.py | 124 --------- .../parsers/shadowserver/test_scan_vnc.py | 86 ------ .../shadowserver/test_scan_ws_discovery.py | 119 -------- .../parsers/shadowserver/test_scan_xdmcp.py | 117 -------- .../bots/parsers/shadowserver/test_special.py | 106 ------- .../parsers/shadowserver/test_testdata.py | 81 ------ .../shadowserver/testdata/blocklist.csv | 4 - .../testdata/blocklist.csv.license | 2 - .../testdata/botnet_drone.csv.license | 2 - .../testdata/caida_ip_spoofer.csv.license | 2 - .../testdata/compromised_website.csv | 4 - .../testdata/compromised_website.csv.license | 2 - .../shadowserver/testdata/darknet.csv.license | 2 - .../testdata/ddos_amplification.csv.license | 2 - .../shadowserver/testdata/device_id.csv | 4 - .../testdata/device_id.csv.license | 2 - .../testdata/drone_brute_force.csv.license | 2 - .../testdata/event4_ddos_participant.csv | 4 - .../event4_ddos_participant.csv.license | 2 - .../testdata/event4_honeypot_brute_force.csv | 7 - .../event4_honeypot_brute_force.csv.license | 2 - .../testdata/event4_honeypot_darknet.csv | 9 - .../event4_honeypot_darknet.csv.license | 2 - .../testdata/event4_honeypot_ddos.csv | 4 - .../testdata/event4_honeypot_ddos.csv.license | 2 - .../testdata/event4_honeypot_ddos_amp.csv | 6 - .../event4_honeypot_ddos_amp.csv.license | 2 - .../testdata/event4_honeypot_ddos_target.csv | 4 - .../event4_honeypot_ddos_target.csv.license | 2 - .../testdata/event4_honeypot_http_scan.csv | 3 - .../event4_honeypot_http_scan.csv.license | 2 - .../testdata/event4_ip_spoofer.csv | 7 - .../testdata/event4_ip_spoofer.csv.license | 2 - .../testdata/event4_microsoft_sinkhole.csv | 7 - .../event4_microsoft_sinkhole.csv.license | 2 - .../event4_microsoft_sinkhole_http.csv | 6 - ...event4_microsoft_sinkhole_http.csv.license | 2 - .../shadowserver/testdata/event4_sinkhole.csv | 4 - .../testdata/event4_sinkhole.csv.license | 2 - .../testdata/event4_sinkhole_dns.csv | 4 - .../testdata/event4_sinkhole_dns.csv.license | 2 - .../testdata/event4_sinkhole_http.csv | 6 - .../testdata/event4_sinkhole_http.csv.license | 2 - .../testdata/event4_sinkhole_http_referer.csv | 6 - .../event4_sinkhole_http_referer.csv.license | 2 - .../testdata/event6_sinkhole_http.csv | 4 - .../testdata/event6_sinkhole_http.csv.license | 2 - .../testdata/hp_http_scan.csv.license | 2 - .../testdata/hp_ics_scan.csv.license | 2 - .../shadowserver/testdata/malware_url.csv | 4 - .../testdata/malware_url.csv.license | 2 - .../testdata/outdated_dnssec_key.csv.license | 2 - .../shadowserver/testdata/phish_url.csv | 4 - .../testdata/phish_url.csv.license | 2 - .../testdata/population_http_proxy.csv | 4 - .../population_http_proxy.csv.license | 2 - .../shadowserver/testdata/sandbox_conn.csv | 4 - .../testdata/sandbox_conn.csv.license | 2 - .../shadowserver/testdata/sandbox_dns.csv | 4 - .../testdata/sandbox_dns.csv.license | 2 - .../shadowserver/testdata/sandbox_url.csv | 4 - .../testdata/sandbox_url.csv.license | 2 - .../shadowserver/testdata/scan_adb.csv | 3 - .../testdata/scan_adb.csv.license | 2 - .../shadowserver/testdata/scan_afp.csv | 3 - .../testdata/scan_afp.csv.license | 2 - .../shadowserver/testdata/scan_amqp.csv | 4 - .../testdata/scan_amqp.csv.license | 2 - .../shadowserver/testdata/scan_ard.csv | 4 - .../testdata/scan_ard.csv.license | 2 - .../shadowserver/testdata/scan_chargen.csv | 4 - .../testdata/scan_chargen.csv.license | 2 - .../testdata/scan_cisco_smart_install.csv | 3 - .../scan_cisco_smart_install.csv.license | 2 - .../shadowserver/testdata/scan_coap.csv | 4 - .../testdata/scan_coap.csv.license | 2 - .../shadowserver/testdata/scan_couchdb.csv | 4 - .../testdata/scan_couchdb.csv.license | 2 - .../shadowserver/testdata/scan_cwmp.csv | 3 - .../testdata/scan_cwmp.csv.license | 2 - .../shadowserver/testdata/scan_db2.csv | 3 - .../testdata/scan_db2.csv.license | 2 - .../testdata/scan_ddos_middlebox.csv | 4 - .../testdata/scan_ddos_middlebox.csv.license | 2 - .../shadowserver/testdata/scan_dns.csv | 101 ------- .../testdata/scan_dns.csv.license | 2 - .../shadowserver/testdata/scan_docker.csv | 4 - .../testdata/scan_docker.csv.license | 2 - .../testdata/scan_dvr_dhcpdiscover.csv | 4 - .../scan_dvr_dhcpdiscover.csv.license | 2 - .../testdata/scan_elasticsearch.csv | 4 - .../testdata/scan_elasticsearch.csv.license | 2 - .../shadowserver/testdata/scan_exchange.csv | 8 - .../testdata/scan_exchange.csv.license | 2 - .../shadowserver/testdata/scan_ftp.csv | 3 - .../testdata/scan_ftp.csv.license | 2 - .../shadowserver/testdata/scan_hadoop.csv | 3 - .../testdata/scan_hadoop.csv.license | 2 - .../shadowserver/testdata/scan_http.csv | 3 - .../testdata/scan_http.csv.license | 2 - .../shadowserver/testdata/scan_http_proxy.csv | 4 - .../testdata/scan_http_proxy.csv.license | 2 - .../testdata/scan_http_vulnerable.csv | 4 - .../testdata/scan_http_vulnerable.csv.license | 2 - .../shadowserver/testdata/scan_ics.csv | 4 - .../testdata/scan_ics.csv.license | 2 - .../shadowserver/testdata/scan_ipmi.csv | 96 ------- .../testdata/scan_ipmi.csv.license | 2 - .../shadowserver/testdata/scan_ipp.csv | 2 - .../testdata/scan_ipp.csv.license | 2 - .../shadowserver/testdata/scan_isakmp.csv | 3 - .../testdata/scan_isakmp.csv.license | 2 - .../shadowserver/testdata/scan_kubernetes.csv | 4 - .../testdata/scan_kubernetes.csv.license | 2 - .../shadowserver/testdata/scan_ldap_tcp.csv | 4 - .../testdata/scan_ldap_tcp.csv.license | 2 - .../shadowserver/testdata/scan_ldap_udp.csv | 4 - .../testdata/scan_ldap_udp.csv.license | 2 - .../shadowserver/testdata/scan_mdns.csv | 4 - .../testdata/scan_mdns.csv.license | 2 - .../shadowserver/testdata/scan_memcached.csv | 4 - .../testdata/scan_memcached.csv.license | 2 - .../shadowserver/testdata/scan_mongodb.csv | 11 - .../testdata/scan_mongodb.csv.license | 2 - .../shadowserver/testdata/scan_mqtt.csv | 2 - .../testdata/scan_mqtt.csv.license | 2 - .../shadowserver/testdata/scan_mqtt_anon.csv | 4 - .../testdata/scan_mqtt_anon.csv.license | 2 - .../shadowserver/testdata/scan_mssql.csv | 4 - .../testdata/scan_mssql.csv.license | 2 - .../shadowserver/testdata/scan_mysql.csv | 4 - .../testdata/scan_mysql.csv.license | 2 - .../shadowserver/testdata/scan_nat_pmp.csv | 4 - .../testdata/scan_nat_pmp.csv.license | 2 - .../shadowserver/testdata/scan_netbios.csv | 4 - .../testdata/scan_netbios.csv.license | 2 - .../testdata/scan_netis_router.csv | 4 - .../testdata/scan_netis_router.csv.license | 2 - .../shadowserver/testdata/scan_ntp.csv | 4 - .../testdata/scan_ntp.csv.license | 2 - .../shadowserver/testdata/scan_ntpmonitor.csv | 4 - .../testdata/scan_ntpmonitor.csv.license | 2 - .../shadowserver/testdata/scan_portmapper.csv | 4 - .../testdata/scan_portmapper.csv.license | 2 - .../shadowserver/testdata/scan_postgres.csv | 4 - .../testdata/scan_postgres.csv.license | 2 - .../shadowserver/testdata/scan_qotd.csv | 4 - .../testdata/scan_qotd.csv.license | 2 - .../shadowserver/testdata/scan_quic.csv | 4 - .../testdata/scan_quic.csv.license | 2 - .../shadowserver/testdata/scan_radmin.csv | 10 - .../testdata/scan_radmin.csv.license | 2 - .../shadowserver/testdata/scan_rdp.csv | 3 - .../testdata/scan_rdp.csv.license | 2 - .../shadowserver/testdata/scan_rdpeudp.csv | 4 - .../testdata/scan_rdpeudp.csv.license | 2 - .../shadowserver/testdata/scan_redis.csv | 94 ------- .../testdata/scan_redis.csv.license | 2 - .../shadowserver/testdata/scan_rsync.csv | 4 - .../testdata/scan_rsync.csv.license | 2 - .../shadowserver/testdata/scan_sip.csv | 4 - .../testdata/scan_sip.csv.license | 2 - .../shadowserver/testdata/scan_slp.csv | 4 - .../testdata/scan_slp.csv.license | 2 - .../shadowserver/testdata/scan_smb.csv | 4 - .../testdata/scan_smb.csv.license | 2 - .../testdata/scan_smtp_vulnerable.csv | 3 - .../testdata/scan_smtp_vulnerable.csv.license | 2 - .../shadowserver/testdata/scan_snmp.csv | 4 - .../testdata/scan_snmp.csv.license | 2 - .../shadowserver/testdata/scan_socks.csv | 4 - .../testdata/scan_socks.csv.license | 2 - .../shadowserver/testdata/scan_ssdp.csv | 4 - .../testdata/scan_ssdp.csv.license | 2 - .../shadowserver/testdata/scan_ssh.csv | 4 - .../testdata/scan_ssh.csv.license | 2 - .../shadowserver/testdata/scan_ssl.csv | 4 - .../testdata/scan_ssl.csv.license | 2 - .../shadowserver/testdata/scan_ssl_freak.csv | 46 ---- .../testdata/scan_ssl_freak.csv.license | 2 - .../shadowserver/testdata/scan_ssl_poodle.csv | 32 --- .../testdata/scan_ssl_poodle.csv.license | 2 - .../shadowserver/testdata/scan_stun.csv | 4 - .../testdata/scan_stun.csv.license | 2 - .../testdata/scan_synfulknock.csv | 4 - .../testdata/scan_synfulknock.csv.license | 2 - .../shadowserver/testdata/scan_telnet.csv | 3 - .../testdata/scan_telnet.csv.license | 2 - .../shadowserver/testdata/scan_tftp.csv | 4 - .../testdata/scan_tftp.csv.license | 2 - .../shadowserver/testdata/scan_ubiquiti.csv | 4 - .../testdata/scan_ubiquiti.csv.license | 2 - .../shadowserver/testdata/scan_vnc.csv | 3 - .../testdata/scan_vnc.csv.license | 2 - .../testdata/scan_ws_discovery.csv | 4 - .../testdata/scan_ws_discovery.csv.license | 2 - .../shadowserver/testdata/scan_xdmcp.csv | 4 - .../testdata/scan_xdmcp.csv.license | 2 - .../testdata/sinkhole_http_drone.csv.license | 2 - .../parsers/shadowserver/testdata/special.csv | 4 - .../shadowserver/testdata/special.csv.license | 2 - 291 files changed, 12939 deletions(-) delete mode 100644 intelmq/tests/bots/parsers/shadowserver/scan_rdpeudp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_blocklist.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_compromised_website.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_device_id.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_ddos_participant.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_darknet.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos_target.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_http_scan.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_ip_spoofer.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole_http.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_dns.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http_referer.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event6_sinkhole_http.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_honeypot_brute_force.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_honeypot_ddos_amp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_malware_url.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_phish_url.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_population_http_proxy.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_sandbox_conn.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_sandbox_dns.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_sandbox_url.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_adb.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_afp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_amqp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ard.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_chargen.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_cisco_smart_install.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_coap.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_couchdb.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_cwmp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_db2.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ddos_middlebox.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_dns.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_docker.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_dvr_dhcpdiscover.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_elasticsearch.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_exchange.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ftp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_hadoop.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_http.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_http_proxy.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_http_vulnerable.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ics.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ipmi.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ipp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_isakmp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_kubernetes.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_tcp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_udp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_mdns.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_memcached.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_mongodb.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt_anon.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_mssql.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_mysql.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_nat_pmp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_netbios.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_netis_router.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ntp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ntpmonitor.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_portmapper.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_postgres.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_qotd.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_quic.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_radmin.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_rdp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_rdpeudp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_redis.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_rsync.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_sip.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_slp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_smb.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_smb_json.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_smtp_vulnerable.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_snmp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_socks.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ssdp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ssh.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ssl.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_freak.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_poodle.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_stun.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_synfulknock.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_telnet.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_tftp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ubiquiti.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_vnc.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ws_discovery.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_xdmcp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_special.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_testdata.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/botnet_drone.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/caida_ip_spoofer.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/darknet.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/ddos_amplification.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/drone_brute_force.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/hp_http_scan.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/hp_ics_scan.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/outdated_dnssec_key.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sinkhole_http_drone.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/special.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/special.csv.license diff --git a/intelmq/tests/bots/parsers/shadowserver/scan_rdpeudp.csv.license b/intelmq/tests/bots/parsers/shadowserver/scan_rdpeudp.csv.license deleted file mode 100644 index 043ed079f1..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/scan_rdpeudp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Sebastian Waldbauer -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/test_blocklist.py b/intelmq/tests/bots/parsers/shadowserver/test_blocklist.py deleted file mode 100644 index 48509eea0e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_blocklist.py +++ /dev/null @@ -1,103 +0,0 @@ -# SPDX-FileCopyrightText: 2020 Thomas Hungenberg -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/blocklist.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = { - 'feed.name': 'Block Listed IP Addresses', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-blocklist-test-geo.csv", -} -EVENTS = [{ - '__type': 'Event', - 'feed.name': 'Block Listed IP Addresses', - "classification.identifier": "blacklisted-ip", - "classification.taxonomy": "other", - "classification.type": "blacklist", - "extra.naics": 517311, - "extra.reason": "Malicious Host AA", - "extra.source": "Alien Vault", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 5678, - "source.geolocation.cc": "XX", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.134", - "source.reverse_dns": "host.local", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T07:00:19+00:00" -}, -{ - '__type': 'Event', - 'feed.name': 'Block Listed IP Addresses', - "classification.identifier": "blacklisted-ip", - "classification.taxonomy": "other", - "classification.type": "blacklist", - "extra.naics": 517311, - "extra.reason": "Malicious Host AA", - "extra.source": "Alien Vault", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 5678, - "source.geolocation.cc": "XX", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.171", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T07:00:19+00:00" -}, -{ - '__type': 'Event', - 'feed.name': 'Block Listed IP Addresses', - "classification.identifier": "blacklisted-ip", - "classification.taxonomy": "other", - "classification.type": "blacklist", - "extra.naics": 517311, - "extra.reason": "Malicious Host AA", - "extra.source": "Alien Vault", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - "source.asn": 5678, - "source.geolocation.cc": "XX", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.network": "198.123.245.0/24", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T07:00:19+00:00" -},] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_compromised_website.py b/intelmq/tests/bots/parsers/shadowserver/test_compromised_website.py deleted file mode 100644 index 53c5b247b1..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_compromised_website.py +++ /dev/null @@ -1,88 +0,0 @@ -# SPDX-FileCopyrightText: 2017 Sebastian Wagner -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/compromised_website.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "ShadowServer Compromised Website", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-compromised_website-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'ShadowServer Compromised Website', - 'classification.taxonomy': 'intrusions', - 'classification.type': 'system-compromise', - 'classification.identifier': 'compromised-website', - 'extra.server': 'Microsoft-IIS/7.5', - 'extra.system': 'WINNT', - 'extra.detected_since': '2015-05-09 05:51:12', - 'protocol.application': 'http', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 64496, - 'source.geolocation.cc': 'AT', - 'source.geolocation.city': 'VIENNA', - 'source.geolocation.region': 'WIEN', - 'source.ip': '203.0.113.1', - 'source.port': 80, - 'source.url': 'http://example.com/header.php', - 'source.fqdn': 'example.com', - 'source.reverse_dns': 'example.com', - 'malware.name': 'hacked-webserver-stealrat-t1', - 'event_description.text': 'spam', - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2017-01-16T00:43:48+00:00'}, - {'__type': 'Event', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'classification.identifier': 'compromised-website', - 'classification.taxonomy': 'intrusions', - 'classification.type': 'system-compromise', - 'event_description.text': 'phishing', - 'feed.name': 'ShadowServer Compromised Website', - 'malware.name': 'phishing', - 'protocol.application': 'http', - 'source.asn': 64496, - 'source.fqdn': 'example.com', - 'source.geolocation.cc': 'AT', - 'source.geolocation.city': 'GRAZ', - 'source.geolocation.region': 'STEIERMARK', - 'source.ip': '203.0.113.1', - 'source.port': 80, - 'source.url': 'http://example.com/', - 'time.source': '2018-04-09T15:43:41+00:00'}, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_device_id.py b/intelmq/tests/bots/parsers/shadowserver/test_device_id.py deleted file mode 100644 index e8954e03c1..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_device_id.py +++ /dev/null @@ -1,116 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/device_id.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Device ID', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-device_id-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'device-id', - 'classification.taxonomy' : 'other', - 'classification.type' : 'undetermined', - 'extra.device_model' : 'FortiGate', - 'extra.device_type' : 'firewall', - 'extra.device_vendor' : 'Fortinet', - 'extra.naics' : 517311, - 'feed.name' : 'Device ID', - 'extra.tag' : 'ssl,vpn', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 2116, - 'source.geolocation.cc' : 'NO', - 'source.geolocation.city' : 'TROMVIK', - 'source.geolocation.region' : 'TROMS OG FINNMARK', - 'source.ip' : '88.84.0.0', - 'source.port' : 10443, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:42+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'device-id', - 'classification.taxonomy' : 'other', - 'classification.type' : 'undetermined', - 'extra.device_model' : 'FortiGate', - 'extra.device_type' : 'firewall', - 'extra.device_vendor' : 'Fortinet', - 'feed.name' : 'Device ID', - 'extra.tag' : 'ssl,vpn', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 27843, - 'source.geolocation.cc' : 'PE', - 'source.geolocation.city' : 'LIMA', - 'source.geolocation.region' : 'METROPOLITANA DE LIMA', - 'source.ip' : '170.231.0.0', - 'source.port' : 10443, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:42+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'device-id', - 'classification.taxonomy' : 'other', - 'classification.type' : 'undetermined', - 'extra.device_model' : 'FortiGate', - 'extra.device_type' : 'firewall', - 'extra.device_vendor' : 'Fortinet', - 'extra.naics' : 517311, - 'feed.name' : 'Device ID', - 'extra.tag' : 'ssl,vpn', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 4181, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'MILWAUKEE', - 'source.geolocation.region' : 'WISCONSIN', - 'source.ip' : '96.60.0.0', - 'source.port' : 10443, - 'source.reverse_dns' : '96-60-66-218.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:42+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_ddos_participant.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_ddos_participant.py deleted file mode 100644 index badc53a736..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_ddos_participant.py +++ /dev/null @@ -1,131 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_ddos_participant.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'DDoS Participant', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-event4_ddos_participant-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'ddos-participant', - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'destination.asn': 65534, - 'destination.geolocation.cc': 'ZZ', - 'destination.geolocation.city': 'City', - 'destination.geolocation.region': 'Region', - 'destination.ip': '172.16.0.1', - 'destination.port': 443, - 'destination.reverse_dns': 'node01.example.net', - 'extra.application': 'https', - 'extra.domain': 'www.example.com', - 'extra.http_method': 'GET', - 'extra.http_path': '/??=GovpfOoaWYlk', - 'feed.name': 'DDoS Participant', - 'malware.name': 'ddos-participant', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 38055, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'ddos-participant', - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'destination.asn': 65534, - 'destination.geolocation.cc': 'ZZ', - 'destination.geolocation.city': 'City', - 'destination.geolocation.region': 'Region', - 'destination.ip': '172.16.0.2', - 'destination.port': 53, - 'destination.reverse_dns': 'node02.example.net', - 'extra.application': 'dns', - 'feed.name': 'DDoS Participant', - 'malware.name': 'ddos-participant', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 53, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'ddos-participant', - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'destination.asn': 65534, - 'destination.geolocation.cc': 'ZZ', - 'destination.geolocation.city': 'City', - 'destination.geolocation.region': 'Region', - 'destination.ip': '172.16.0.3', - 'destination.port': 53, - 'destination.reverse_dns': 'node03.example.net', - 'extra.application': 'dns', - 'extra.device_model': 'Exchange', - 'extra.device_type': 'email', - 'extra.device_vendor': 'Microsoft', - 'feed.name': 'DDoS Participant', - 'malware.name': 'ddos-participant', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 53, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_darknet.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_darknet.py deleted file mode 100644 index 1d020f4737..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_darknet.py +++ /dev/null @@ -1,106 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_honeypot_darknet.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "ShadowServer Darknet", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_honeypot_darknet.csv", - } -EVENTS = [{'__type': 'Event', - 'classification.identifier': 'mirai', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'destination.port': 23, - 'extra.source.naics': 518210, - 'extra.tag': 'mirai', - 'protocol.transport': 'tcp', - 'feed.name': 'ShadowServer Darknet', - 'malware.name': 'mirai', - 'source.asn': 9829, - 'source.geolocation.cc': 'IN', - 'source.geolocation.city': 'CHENGANNUR', - 'source.geolocation.region': 'KERALA', - 'source.ip': '61.3.1.2', - 'source.port': 4717, - 'time.source': '2021-03-07T00:00:00+00:00', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - }, - {'__type': 'Event', - 'classification.identifier': 'mirai', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'destination.port': 23, - 'protocol.transport': 'tcp', - 'extra.source.naics': 517311, - 'extra.tag': 'mirai', - 'feed.name': 'ShadowServer Darknet', - 'malware.name': 'mirai', - 'source.asn': 4766, - 'source.geolocation.cc': 'KR', - 'source.geolocation.city': 'PYEONGCHANG-EUP', - 'source.geolocation.region': 'GANGWON-DO', - 'source.ip': '211.218.3.4', - 'source.port': 4405, - 'time.source': '2021-03-07T00:00:00+00:00', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - }, - {'__type': 'Event', - 'classification.identifier': 'mirai', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'extra.tag': 'mirai', - 'destination.port': 23, - 'protocol.transport': 'tcp', - 'feed.name': 'ShadowServer Darknet', - 'malware.name': 'mirai', - 'source.asn': 266915, - 'source.geolocation.cc': 'BR', - 'source.geolocation.city': 'VITORIA DA CONQUISTA', - 'source.geolocation.region': 'BAHIA', - 'source.ip': '45.225.5.6', - 'source.port': 59777, - 'source.reverse_dns': 'static-45-225-x-x.example.net', - 'time.source': '2021-03-07T00:00:00+00:00', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - }, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos.py deleted file mode 100644 index c62a610faf..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos.py +++ /dev/null @@ -1,148 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_honeypot_ddos.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Honeypot DDoS Events', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-event4_honeypot_ddos-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'honeypot-ddos', - 'classification.taxonomy' : 'availability', - 'classification.type' : 'ddos', - 'destination.asn' : 65534, - 'destination.geolocation.cc' : 'ZZ', - 'destination.geolocation.city' : 'City', - 'destination.geolocation.region' : 'Region', - 'destination.ip' : '172.16.0.1', - 'destination.port' : 88, - 'destination.reverse_dns' : 'node01.example.net', - 'extra.application' : 'mirai', - 'extra.attack' : 'atk10', - 'extra.dst_netmask' : '32', - 'extra.dst_network' : '121.12.110.28/32', - 'extra.duration' : 30, - 'extra.family' : 'mirai', - 'extra.packet_length' : 1440, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'mirai', - 'feed.name' : 'Honeypot DDoS Events', - 'malware.name' : 'ddos', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 61234, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'honeypot-ddos', - 'classification.taxonomy' : 'availability', - 'classification.type' : 'ddos', - 'destination.asn' : 65534, - 'destination.geolocation.cc' : 'ZZ', - 'destination.geolocation.city' : 'City', - 'destination.geolocation.region' : 'Region', - 'destination.ip' : '172.16.0.2', - 'destination.port' : 80, - 'destination.reverse_dns' : 'node02.example.net', - 'extra.application' : 'mirai', - 'extra.attack' : 'atk10', - 'extra.dst_netmask' : '32', - 'extra.dst_network' : '180.97.183.94/32', - 'extra.duration' : 30, - 'extra.family' : 'mirai', - 'extra.packet_length' : 1440, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'mirai', - 'feed.name' : 'Honeypot DDoS Events', - 'malware.name' : 'ddos', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 61234, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'honeypot-ddos', - 'classification.taxonomy' : 'availability', - 'classification.type' : 'ddos', - 'destination.asn' : 65534, - 'destination.geolocation.cc' : 'ZZ', - 'destination.geolocation.city' : 'City', - 'destination.geolocation.region' : 'Region', - 'destination.ip' : '172.16.0.3', - 'destination.reverse_dns' : 'node03.example.net', - 'extra.application' : 'mirai', - 'extra.attack' : 'atk7', - 'extra.destination.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.dst_netmask' : '32', - 'extra.dst_network' : '104.237.138.135/32', - 'extra.duration' : 10, - 'extra.family' : 'mirai', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'mirai', - 'feed.name' : 'Honeypot DDoS Events', - 'malware.name' : 'ddos', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 6379, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos_target.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos_target.py deleted file mode 100644 index f379d1c882..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos_target.py +++ /dev/null @@ -1,150 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_honeypot_ddos_target.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Honeypot DDoS Target Events', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-event4_honeypot_ddos_target-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'honeypot-ddos-target', - 'classification.taxonomy' : 'availability', - 'classification.type' : 'ddos', - 'destination.asn' : 65534, - 'destination.geolocation.cc' : 'ZZ', - 'destination.geolocation.city' : 'City', - 'destination.geolocation.region' : 'Region', - 'destination.ip' : '172.16.0.1', - 'destination.port' : 80, - 'destination.reverse_dns' : 'node01.example.net', - 'extra.application' : 'mirai', - 'extra.attack' : 'atk0', - 'extra.dst_netmask' : '32', - 'extra.dst_network' : '115.238.198.85/32', - 'extra.duration' : 30, - 'extra.family' : 'mirai', - 'extra.packet_length' : 1440, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'mirai', - 'feed.name' : 'Honeypot DDoS Target Events', - 'malware.name' : 'ddos', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 61234, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'honeypot-ddos-target', - 'classification.taxonomy' : 'availability', - 'classification.type' : 'ddos', - 'destination.asn' : 65534, - 'destination.geolocation.cc' : 'ZZ', - 'destination.geolocation.city' : 'City', - 'destination.geolocation.region' : 'Region', - 'destination.ip' : '172.16.0.2', - 'destination.port' : 43437, - 'destination.reverse_dns' : 'node02.example.net', - 'extra.application' : 'mirai', - 'extra.attack' : 'atk0', - 'extra.destination.sector' : 'Information', - 'extra.dst_netmask' : '32', - 'extra.dst_network' : '52.184.50.250/32', - 'extra.duration' : 30, - 'extra.family' : 'mirai', - 'extra.packet_length' : 1440, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'mirai', - 'feed.name' : 'Honeypot DDoS Target Events', - 'malware.name' : 'ddos', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 61234, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'honeypot-ddos-target', - 'classification.taxonomy' : 'availability', - 'classification.type' : 'ddos', - 'destination.asn' : 65534, - 'destination.geolocation.cc' : 'ZZ', - 'destination.geolocation.city' : 'City', - 'destination.geolocation.region' : 'Region', - 'destination.ip' : '172.16.0.3', - 'destination.port' : 80, - 'destination.reverse_dns' : 'node03.example.net', - 'extra.application' : 'mirai', - 'extra.attack' : 'atk10', - 'extra.dst_netmask' : '32', - 'extra.dst_network' : '211.99.102.216/32', - 'extra.duration' : 30, - 'extra.family' : 'mirai', - 'extra.packet_length' : 1440, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'mirai', - 'feed.name' : 'Honeypot DDoS Target Events', - 'malware.name' : 'ddos', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 61234, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_http_scan.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_http_scan.py deleted file mode 100644 index bcf268ba7d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_http_scan.py +++ /dev/null @@ -1,109 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Mikk Margus Möll -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_honeypot_http_scan.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Honeypot-HTTP-Scan', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2021-08-01T12:00:00+00:00", - "extra.file_name": "2021-08-01-event4_honeypot_http_scan.csv", - } - -EVENTS = [{'__type': 'Event', - 'feed.name': 'Honeypot-HTTP-Scan', - 'classification.identifier': 'honeypot-http-scan', - 'classification.taxonomy': 'information-gathering', - 'classification.type': 'scanner', - 'destination.asn': 5678, - 'destination.geolocation.cc': 'UK', - 'destination.geolocation.city': 'MAIDENHEAD', - 'destination.geolocation.region': 'WINDSOR AND MAIDENHEAD', - 'destination.ip': '109.87.65.43', - 'destination.port': 80, - 'extra.http_url': '/js/ueditor/wwwroot/way-board.cgi', - 'extra.destination.naics': 518210, - 'protocol.transport': 'tcp', - 'protocol.application': 'http', - 'extra.public_source': 'CAPRICA-EU', - 'extra.request_raw': 'R0VUIC9qcy91ZWRpdG9yL3d3d3Jvb3Qvd2F5LWJvYXJkLmNnaSBIVFRQLzEuMHJuQWNjZXB0OiB0ZXh0L2h0bWwsYXBwbGljYXRpb24veGh0bWwreG1sLGFwcGxpY2F0aW9uL3htbDtxPTAuOSwqLyo7cT0wLjhybkFjY2VwdC1FbmNvZGluZzogZ3ppcCwgZGVmbGF0ZXJuQWNjZXB0LUxhbmd1YWdlOiBlbi1VUyxlbjtxPTAuNXJuQ29ubmVjdGlvbjogY2xvc2VybkRudDogMXJuSG9zdDogMTA5Ljg3LjY1LjQzcm5PcmlnaW46IGh0dHA6Ly8xMDkuODcuNjUuNDNyblJlZmVyZXI6IGh0dHA6Ly8xMDkuODcuNjUuNDMvcm5Vc2VyLUFnZW50OiBNb3ppbGxhLzUuMCAoV2luZG93cyBOVCA2LjE7IFdPVzY0KSBBcHBsZVdlYktpdC81MzcuMzYgKEtIVE1MLCBsaWtlIEdlY2tvKSBDaHJvbWUvNTMuMC4yNzg1LjEwNCBTYWZhcmkvNTM3LjM2IENvcmUvMS41My4zMDg0LjQwMCBRUUJyb3dzZXIvOS42LjExMzQ2LjQwMA==', - 'extra.source.naics': 518210, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.version': '3.1.3-dev', - 'malware.name': 'http-scan', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 1234, - 'source.geolocation.cc': 'EE', - 'source.geolocation.city': 'TALLINN', - 'source.geolocation.region': 'HARJUMAA', - 'source.ip': '191.23.45.67', - 'source.port': 36455, - 'source.reverse_dns': '191-23-45-67-host.example.com', - 'time.observation': '2021-08-01T12:00:00+00:00', - 'time.source': '2021-08-01T00:24:08+00:00'}, - {'__type': 'Event', - 'feed.name': 'Honeypot-HTTP-Scan', - 'classification.identifier': 'honeypot-http-scan', - 'classification.taxonomy': 'information-gathering', - 'classification.type': 'scanner', - 'destination.asn': 23456, - 'destination.geolocation.cc': 'UA', - 'destination.geolocation.city': 'KHARKIV', - 'destination.geolocation.region': "KHARKIVS'KA OBLAST'", - 'destination.ip': '82.41.20.10', - 'destination.port': 8080, - 'extra.http_url': '/', - 'extra.method': 'GET', - 'protocol.transport': 'tcp', - 'protocol.application': 'http', - 'extra.public_source': 'CAPRICA-EU', - 'extra.request_raw': 'R0VUIC8gSFRUUC8xLjENCkhvc3Q6IDgyLjQxLjIwLjEwOjgwODANCkFjY2VwdDogdGV4dC9odG1sLGFwcGxpY2F0aW9uL3hodG1sK3htbCxhcHBsaWNhdGlvbi94bWw7cT0wLjksKi8qO3E9MC44DQpBY2NlcHQtRW5jb2Rpbmc6IGRlZmxhdGUsIGd6aXAsIGlkZW50aXR5DQpBY2NlcHQtTGFuZ3VhZ2U6IGVuLVVTO3E9MC42LGVuO3E9MC40DQpVc2VyLUFnZW50OiBNb3ppbGxhLzUuMCAoV2luZG93cyBOVCA1LjE7IHJ2OjkuMC4xKSBHZWNrby8yMDEwMDEwMSBGaXJlZm94LzkuMC4xDQoNCg==', - 'extra.url_scheme': 'http', - 'extra.user_agent': 'Mozilla/5.0 (Windows NT 5.1; rv:9.0.1) Gecko/20100101 Firefox/9.0.1', - 'malware.name': 'http-scan', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn': 12345, - 'source.geolocation.cc': 'EE', - 'source.geolocation.city': 'TALLINN', - 'source.geolocation.region': 'HARJUMAA', - 'source.ip': '45.67.89.123', - 'source.port': 58610, - 'time.observation': '2021-08-01T12:00:00+00:00', - 'time.source': '2021-08-01T05:21:59+00:00'}, -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_ip_spoofer.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_ip_spoofer.py deleted file mode 100644 index d21fb10c5b..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_ip_spoofer.py +++ /dev/null @@ -1,182 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), "testdata/event4_ip_spoofer.csv")) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = { - "feed.name": "CAIDA", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2020-08-19T00:00:00+00:00", - "extra.file_name": "2020-08-19-event4_ip_spoofer.csv", -} - -EVENTS = [ - { - "__type": "Event", - "feed.name": "CAIDA", - "time.source": "2021-03-28T00:42:59+00:00", - "source.ip": "98.191.250.0", - - "source.asn": 22898, - - "source.geolocation.cc": "US", - "source.geolocation.region": "OKLAHOMA", - "source.geolocation.city": "OKLAHOMA CITY", - "source.network": "98.191.250.0/24", - "source.reverse_dns": 'ip-98.191.250.0.atlinkservices.com', - "extra.routedspoof": "received", - "extra.session": '1112907', - "extra.nat": True, - "extra.public_source": "caida", - "extra.source.naics": 517311, - "extra.version": 'ipv4', - "protocol.transport": 'tcp', - "extra.infection": 'ip-spoofer', - - "classification.identifier": "ip-spoofer", - "classification.taxonomy": "fraud", - "classification.type": "masquerade", - - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[1]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "CAIDA", - "time.source": "2021-03-28T01:36:22+00:00", - "source.ip": "191.7.16.0", - - "source.asn": 262485, - - "source.geolocation.cc": "BR", - "source.geolocation.region": "RIO DE JANEIRO", - "source.geolocation.city": "NOVA IGUACU", - "source.network": "191.7.16.0/24", - "extra.routedspoof": "received", - "extra.session": '1112914', - "extra.nat": False, - "extra.public_source": "caida", - "extra.version": 'ipv4', - "protocol.transport": 'tcp', - "extra.infection": 'ip-spoofer', - - "classification.identifier": "ip-spoofer", - "classification.taxonomy": "fraud", - "classification.type": "masquerade", - - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[2]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "CAIDA", - "time.source": "2021-03-28T02:10:58+00:00", - "source.ip": "202.53.160.0", - - "source.asn": 23923, - - "source.geolocation.cc": "BD", - "source.geolocation.region": "DHAKA", - "source.geolocation.city": "DHAKA", - "source.network": "202.53.160.0/24", - "extra.routedspoof": "received", - "extra.session": '1112931', - "extra.nat": True, - "extra.public_source": "caida", - "extra.version": 'ipv4', - "protocol.transport": 'tcp', - "extra.infection": 'ip-spoofer', - - "classification.identifier": "ip-spoofer", - "classification.taxonomy": "fraud", - "classification.type": "masquerade", - - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[3]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "CAIDA", - "time.source": "2021-03-28T03:41:51+00:00", - "source.ip": "87.121.75.0", - - "source.asn": 134697, - - "source.geolocation.cc": "AU", - "source.geolocation.region": "QUEENSLAND", - "source.geolocation.city": "BRISBANE", - "source.network": "87.121.75.0/24", - "extra.routedspoof": "received", - "extra.session": '1112953', - "extra.nat": True, - "extra.public_source": "caida", - "extra.version": 'ipv4', - "protocol.transport": 'tcp', - "extra.infection": 'ip-spoofer', - - "classification.identifier": "ip-spoofer", - "classification.taxonomy": "fraud", - "classification.type": "masquerade", - - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[4]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "CAIDA", - "time.source": "2021-03-28T06:07:17+00:00", - "source.ip": "189.201.194.0", - - "source.asn": 262944, - - "source.network": "189.201.194.0/24", - "source.geolocation.cc": 'MX', - "source.geolocation.city": 'SALTILLO', - "source.geolocation.region": 'COAHUILA', - "source.reverse_dns": 'ip-189-201-194-0.slw.spectro.mx', - "extra.routedspoof": "received", - "extra.session": '1113015', - "extra.nat": True, - "extra.public_source": "caida", - "extra.version": 'ipv4', - "protocol.transport": 'tcp', - "extra.infection": 'ip-spoofer', - - "classification.identifier": "ip-spoofer", - "classification.taxonomy": "fraud", - "classification.type": "masquerade", - - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[5]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, -] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == "__main__": - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole.py deleted file mode 100644 index f008fd18e1..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole.py +++ /dev/null @@ -1,135 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_microsoft_sinkhole.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "ShadowServer Microsoft Sinkhole", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_microsoft_sinkhole.csv", - } -EVENTS = [{'__type': 'Event', - 'classification.identifier': 'b68-zeroaccess-2-32bit', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.geolocation.cc': 'HK', - 'destination.geolocation.city': 'HONG KONG', - 'destination.geolocation.region': 'HONG KONG', - 'destination.ip': '168.63.134.179', - 'destination.port': 16464, - 'extra.destination.naics': 334111, - 'extra.tag': 'b68-zeroaccess-2-32bit', - 'extra.infection': 'b68-zeroaccess-2-32bit', - 'extra.destination.sector': 'Information', - 'extra.public_source': 'MSDCU', - 'extra.source.naics': 517311, - 'extra.tag': 'b68-zeroaccess-2-32bit', - 'feed.name': 'ShadowServer Microsoft Sinkhole', - 'malware.name': 'zeroaccess', - 'protocol.transport': 'tcp', - 'source.asn': 7303, - 'source.geolocation.cc': 'AR', - 'source.geolocation.city': 'CASEROS', - 'source.geolocation.region': 'BUENOS AIRES', - 'source.ip': '190.229.1.2', - 'source.port': 52955, - 'time.source': '2021-06-07T00:00:00+00:00', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - }, - {'__type': 'Event', - 'classification.identifier': 'b68-zeroaccess-2-32bit', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.geolocation.cc': 'IE', - 'destination.geolocation.city': 'DUBLIN', - 'destination.geolocation.region': 'DUBLIN', - 'destination.ip': '52.169.3.4', - 'destination.port': 16464, - 'extra.destination.naics': 334111, - 'extra.tag': 'b68-zeroaccess-2-32bit', - 'extra.infection': 'b68-zeroaccess-2-32bit', - 'extra.destination.sector': 'Information', - 'extra.public_source': 'MSDCU', - 'extra.source.naics': 517311, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'feed.name': 'ShadowServer Microsoft Sinkhole', - 'malware.name': 'zeroaccess', - 'protocol.transport': 'tcp', - 'source.asn': 5769, - 'source.geolocation.cc': 'CA', - 'source.geolocation.city': 'LAVAL', - 'source.geolocation.region': 'QUEBEC', - 'source.ip': '96.20.3.4', - 'source.port': 16464, - 'time.source': '2021-06-07T00:00:00+00:00', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - }, - {'__type': 'Event', - 'classification.identifier': 'b68-zeroaccess-2-32bit', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.geolocation.cc': 'HK', - 'destination.geolocation.city': 'HONG KONG', - 'destination.geolocation.region': 'HONG KONG', - 'destination.ip': '168.63.134.179', - 'destination.port': 16464, - 'extra.tag': 'b68-zeroaccess-2-32bit', - 'extra.infection': 'b68-zeroaccess-2-32bit', - 'extra.destination.naics': 334111, - 'extra.destination.sector': 'Information', - 'extra.public_source': 'MSDCU', - 'extra.source.naics': 517311, - 'feed.name': 'ShadowServer Microsoft Sinkhole', - 'malware.name': 'zeroaccess', - 'protocol.transport': 'tcp', - 'source.asn': 8151, - 'source.geolocation.cc': 'MX', - 'source.geolocation.city': 'MEXICO CITY', - 'source.geolocation.region': "CIUDAD DE MEXICO", - 'source.ip': '187.222.5.6', - 'source.port': 55049, - 'time.source': '2021-06-07T00:00:00+00:00', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - }, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole_http.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole_http.py deleted file mode 100644 index 2f8c3d8e2e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole_http.py +++ /dev/null @@ -1,202 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_microsoft_sinkhole_http.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'HTTP Microsoft Sinkhole IPv4', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_microsoft_sinkhole_http.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'HTTP Microsoft Sinkhole IPv4', - 'classification.identifier': 'necurs', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'ASHBURN', - 'destination.geolocation.region': 'VIRGINIA', - 'destination.ip': '40.121.206.97', - 'destination.port': 80, - 'destination.url': 'http://40.121.206.97/locator.php', - 'extra.destination.naics': 334111, - 'extra.destination.sector': 'Information', - 'extra.public_source': 'MSDCU', - 'extra.infection': 'necurs', - 'extra.tag': 'necurs', - 'protocol.application': 'http', - 'malware.name': 'necurs', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 8386, - 'source.geolocation.cc': 'TR', - 'source.geolocation.city': 'KEPEZ', - 'source.geolocation.region': 'ANTALYA', - 'source.ip': '31.206.1.2', - 'source.port': 49245, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-06-07T00:00:00+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Microsoft Sinkhole IPv4', - 'classification.identifier': 'caphaw', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.fqdn': '3fo8jrthz3y.rgk.cc', - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'REDMOND', - 'destination.geolocation.region': 'WASHINGTON', - 'destination.ip': '204.95.99.204', - 'destination.port': 443, - 'destination.url': 'http://3fo8jrthz3y.rgk.cc/index.php', - 'protocol.application': 'http', - 'extra.infection': 'caphaw', - 'extra.tag': 'caphaw', - 'extra.destination.naics': 334111, - 'extra.destination.sector': 'Information', - 'extra.http_agent': 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.0.6103)', - 'extra.http_referer': 'null', - 'extra.public_source': 'MSDCU', - 'extra.source.naics': 517312, - 'malware.name': 'caphaw', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn': 28573, - 'source.geolocation.cc': 'BR', - 'source.geolocation.city': 'SAO PAULO', - 'source.geolocation.region': 'SAO PAULO', - 'source.ip': '177.140.3.4', - 'source.port': 35919, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-06-07T00:00:00+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Microsoft Sinkhole IPv4', - 'classification.identifier': 'necurs', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'ASHBURN', - 'destination.geolocation.region': 'VIRGINIA', - 'destination.ip': '40.121.206.97', - 'destination.port': 80, - 'destination.url': 'http://40.121.206.97/locator.php', - 'protocol.application': 'http', - 'extra.tag': 'necurs', - 'extra.infection': 'necurs', - 'extra.destination.naics': 334111, - 'extra.destination.sector': 'Information', - 'extra.public_source': 'MSDCU', - 'extra.source.naics': 517311, - 'malware.name': 'necurs', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn': 132199, - 'source.geolocation.cc': 'PH', - 'source.geolocation.city': 'MANDAUE', - 'source.geolocation.region': 'CEBU', - 'source.ip': '180.190.5.6', - 'source.port': 49264, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-06-07T00:00:01+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Microsoft Sinkhole IPv4', - 'classification.identifier': 'necurs', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.geolocation.cc': 'US', - 'destination.ip': '40.121.206.97', - 'destination.geolocation.city': 'ASHBURN', - 'destination.geolocation.region': 'VIRGINIA', - 'destination.port': 80, - 'destination.url': 'http://40.121.206.97/news/stream.php', - 'extra.destination.naics': 334111, - 'extra.destination.sector': 'Information', - 'extra.public_source': 'MSDCU', - 'malware.name': 'necurs', - 'extra.tag': 'necurs', - 'extra.infection': 'necurs', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[4]])), - 'source.asn': 37129, - 'source.geolocation.cc': 'KE', - 'source.geolocation.city': 'NAIROBI', - 'source.geolocation.region': 'NAIROBI CITY', - 'source.ip': '197.157.7.8', - 'source.port': 55307, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-06-07T00:00:01+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Microsoft Sinkhole IPv4', - 'classification.identifier': 'necurs', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'ASHBURN', - 'destination.geolocation.region': 'VIRGINIA', - 'destination.ip': '40.121.206.97', - 'destination.port': 80, - 'destination.url': 'http://40.121.206.97/locator.php', - 'extra.destination.naics': 334111, - 'extra.destination.sector': 'Information', - 'extra.public_source': 'MSDCU', - 'extra.source.naics': 517311, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'malware.name': 'necurs', - 'protocol.application': 'http', - 'extra.tag': 'necurs', - 'extra.infection': 'necurs', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[5]])), - 'source.asn': 812, - 'source.geolocation.cc': 'CA', - 'source.geolocation.city': 'OTTAWA', - 'source.geolocation.region': 'ONTARIO', - 'source.ip': '174.114.9.10', - 'source.port': 59000, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-06-07T00:00:01+00:00'}] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole.py deleted file mode 100644 index 2bb8aa6980..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole.py +++ /dev/null @@ -1,73 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_sinkhole.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "ShadowServer Sinkhole", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_sinkhole.csv", - } -EVENTS = [{'__type': 'Event', - 'classification.identifier': 'victorygate.b', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 28753, - 'destination.geolocation.cc': 'DE', - 'destination.geolocation.city': 'FRANKFURT AM MAIN', - 'destination.geolocation.region': 'HESSEN', - 'destination.ip': '178.162.1.2', - 'destination.port': 4455, - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.public_source': 'eset', - 'feed.name': 'ShadowServer Sinkhole', - 'malware.name': 'victorygate.b', - 'extra.infection': 'victorygate.b', - 'protocol.transport': 'tcp', - 'source.asn': 12252, - 'source.geolocation.cc': 'PE', - 'source.geolocation.city': 'LIMA', - 'source.geolocation.region': 'METROPOLITANA DE LIMA', - 'source.ip': '190.113.1.2', - 'source.port': 17409, - 'time.source': '2021-03-04T00:00:00+00:00', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - }, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_dns.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_dns.py deleted file mode 100644 index cf3bdb1623..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_dns.py +++ /dev/null @@ -1,127 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/event4_sinkhole_dns.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "Sinkhole DNS", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_sinkhole_dns-test-geo.csv", - } - -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'sinkholedns', - 'extra.tag' : 'msexchange', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.count' : 1, - 'extra.infection' : 'calypso', - 'extra.dns_query' : 'YolkIsh.COM', - 'extra.dns_query_type' : 'A', - 'extra.naics' : 518210, - 'extra.sector' : 'Communications, Service Provider, and Hosting Service', - 'feed.name' : 'Sinkhole DNS', - 'malware.name' : 'calypso', - 'protocol.application' : 'dns', - 'protocol.transport' : 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 8220, - 'source.geolocation.cc' : 'DE', - 'source.geolocation.city' : 'FRANKFURT AM MAIN', - 'source.geolocation.region' : 'HESSEN', - 'source.ip' : '217.110.0.0', - 'source.port' : 29614, - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2022-01-06T00:00:02+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'sinkholedns', - 'extra.tag' : 'rat', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.count' : 1, - 'extra.infection' : 'orcus', - 'extra.dns_query' : 'verble.rocks', - 'extra.dns_query_type' : 'A', - 'extra.naics' : 518210, - 'feed.name' : 'Sinkhole DNS', - 'malware.name' : 'orcus', - 'protocol.application' : 'dns', - 'protocol.transport' : 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 40934, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'ASHBURN', - 'source.geolocation.region' : 'VIRGINIA', - 'source.ip' : '209.66.0.0', - 'source.port' : 46189, - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2022-01-06T00:00:02+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'sinkholedns', - 'extra.tag' : 'msexchange', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.count' : 1, - 'extra.infection' : 'calypso', - 'extra.dns_query' : 'RAwFuNS.COM', - 'extra.dns_query_type' : 'A', - 'extra.naics' : 518210, - 'extra.sector' : 'Communications, Service Provider, and Hosting Service', - 'feed.name' : 'Sinkhole DNS', - 'malware.name' : 'calypso', - 'protocol.application' : 'dns', - 'protocol.transport' : 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 8220, - 'source.geolocation.cc' : 'DE', - 'source.geolocation.city' : 'FRANKFURT AM MAIN', - 'source.geolocation.region' : 'HESSEN', - 'source.ip' : '217.110.0.0', - 'source.port' : 3590, - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2022-01-06T00:00:02+00:00' -} -] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http.py deleted file mode 100644 index 60cd6b6efb..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http.py +++ /dev/null @@ -1,189 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_sinkhole_http.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'HTTP Sinkhole IPv4', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_sinkhole_http.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'HTTP Sinkhole IPv4', - 'classification.identifier': 'avalanche-andromeda', - 'extra.tag': 'avalanche-andromeda', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 6939, - 'destination.fqdn': 'differentia.ru', - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'FREMONT', - 'destination.geolocation.region': 'CALIFORNIA', - 'destination.ip': '184.105.1.2', - 'destination.port': 80, - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting Service', - 'malware.name': 'andromeda', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 134707, - 'source.geolocation.cc': 'PH', - 'source.geolocation.city': 'DEL PILAR', - 'source.geolocation.region': 'NUEVA ECIJA', - 'source.ip': '103.196.1.2', - 'source.port': 60902, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-03-04T00:00:00+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Sinkhole IPv4', - 'classification.identifier': 'avalanche-andromeda', - 'extra.tag': 'avalanche-andromeda', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 6939, - 'destination.fqdn': 'differentia.ru', - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'FREMONT', - 'destination.geolocation.region': 'CALIFORNIA', - 'destination.ip': '184.105.3.4', - 'destination.port': 80, - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting Service', - 'malware.name': 'andromeda', - 'extra.source.naics': 517311, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn': 8708, - 'source.geolocation.cc': 'RO', - 'source.geolocation.city': 'CONSTANTA', - 'source.geolocation.region': 'CONSTANTA', - 'source.ip': '5.14.3.4', - 'source.port': 55002, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-03-04T00:00:00+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Sinkhole IPv4', - 'classification.identifier': 'avalanche-andromeda', - 'extra.tag': 'avalanche-andromeda', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 6939, - 'destination.fqdn': 'disorderstatus.ru', - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'FREMONT', - 'destination.geolocation.region': 'CALIFORNIA', - 'destination.ip': '184.105.5.6', - 'destination.port': 80, - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.source.naics': 517311, - 'malware.name': 'andromeda', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn': 9299, - 'source.geolocation.cc': 'PH', - 'source.geolocation.city': 'CEBU', - 'source.geolocation.region': 'CEBU', - 'source.ip': '49.145.5.6', - 'source.port': 31350, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-03-04T00:00:00+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Sinkhole IPv4', - 'classification.identifier': 'avalanche-andromeda', - 'extra.tag': 'avalanche-andromeda', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 6939, - 'destination.fqdn': 'differentia.ru', - 'destination.geolocation.cc': 'US', - 'destination.ip': '184.105.7.8', - 'destination.geolocation.city': 'FREMONT', - 'destination.geolocation.region': 'CALIFORNIA', - 'destination.port': 80, - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.source.naics': 517311, - 'malware.name': 'andromeda', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[4]])), - 'source.asn': 8048, - 'source.geolocation.cc': 'VE', - 'source.geolocation.city': 'VALENCIA', - 'source.geolocation.region': 'CARABOBO', - 'source.ip': '200.44.7.8', - 'source.port': 28063, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-03-04T00:00:00+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Sinkhole IPv4', - 'classification.identifier': 'avalanche-andromeda', - 'extra.tag': 'avalanche-andromeda', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 6939, - 'destination.fqdn': 'differentia.ru', - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'FREMONT', - 'destination.geolocation.region': 'CALIFORNIA', - 'destination.ip': '184.105.9.10', - 'destination.port': 80, - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting Service', - 'malware.name': 'andromeda', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[5]])), - 'source.asn': 17072, - 'source.geolocation.cc': 'MX', - 'source.geolocation.city': 'JUAREZ', - 'source.geolocation.region': 'CHIHUAHUA', - 'source.ip': '187.189.9.10', - 'source.port': 45335, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-03-04T00:00:00+00:00'}] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http_referer.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http_referer.py deleted file mode 100644 index b1ccacd311..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http_referer.py +++ /dev/null @@ -1,213 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Mikk Margus Möll -# -# SPDX-License-Identifier: AGPL-3.0-or-later -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_sinkhole_http_referer.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Sinkhole-Events-HTTP-Referer IPv4', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2021-03-05T00:00:00+00:00", - "extra.file_name": "2021-03-04-event4_sinkhole_http_referer.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Sinkhole-Events-HTTP-Referer IPv4', - 'classification.identifier': 'sinkhole-http-referer', - 'extra.tag': 'kovter', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'destination.asn': 60781, - 'destination.fqdn': '12106.mobapptrack.com', - 'destination.geolocation.cc': 'NL', - 'destination.geolocation.city': 'AMSTERDAM', - 'destination.geolocation.region': 'NOORD-HOLLAND', - 'destination.ip': '85.17.31.82', - 'destination.port': 80, - 'destination.url': 'http://12106.mobapptrack.com/favicon.ico', - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.event_id': '1614816002', - 'malware.name': 'kovter', - 'extra.http_referer': 'http://12106.mobapptrack.com/click/redirect?feed_id=12106&sub_id=7&q=8A5491983C8FBE7743E2D2C36E45EBC4-18307118D2626C9BD756B3F09D14BB910E381EE4', - 'extra.http_referer_asn': 28753, - 'extra.http_referer_city': 'FRANKFURT AM MAIN', - 'extra.http_referer_geo': 'DE', - 'extra.http_referer_hostname': '12106.mobapptrack.com', - 'extra.http_referer_ip': '178.162.203.211', - 'extra.http_referer_naics': 518210, - 'extra.http_referer_port': 80, - 'extra.http_referer_region': 'HESSEN', - 'extra.http_referer_sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-03-04T00:00:02+00:00'}, - {'__type': 'Event', - 'classification.identifier': 'sinkhole-http-referer', - 'extra.tag': 'sunburst', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'destination.asn': 28753, - 'destination.fqdn': 'freescanonline.com', - 'destination.geolocation.cc': 'DE', - 'destination.geolocation.city': 'FRANKFURT AM MAIN', - 'destination.geolocation.region': 'HESSEN', - 'destination.port': 80, - 'destination.url': 'http://freescanonline.com/animalally.com', - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'destination.ip': '178.162.1.2', - 'extra.event_id': '1614816011', - 'malware.name': 'sunburst', - 'extra.http_referer': 'http://x.noizm.com/jump.php?u=http://freescanonline.com/animalally.com', - 'extra.http_referer_asn': 9370, - 'extra.http_referer_city': 'OSAKA', - 'extra.http_referer_geo': 'JP', - 'extra.http_referer_hostname': 'x.noizm.com', - 'extra.http_referer_naics': 518210, - 'extra.http_referer_port': 80, - 'extra.http_referer_ip': '59.106.1.2', - 'extra.http_referer_region': 'OSAKA', - 'extra.http_referer_sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'protocol.transport': 'tcp', - 'feed.name': 'Sinkhole-Events-HTTP-Referer IPv4', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'time.source': '2021-03-04T00:00:11+00:00'}, - {'__type': 'Event', - 'classification.identifier': 'sinkhole-http-referer', - 'extra.tag': 'kovter', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'destination.asn': 28753, - 'destination.fqdn': 'rxrtb.bid', - 'destination.geolocation.cc': 'DE', - 'destination.geolocation.city': 'FRANKFURT AM MAIN', - 'destination.geolocation.region': 'HESSEN', - 'destination.port': 80, - 'destination.url': 'http://rxrtb.bid/getjs?r=0.6393021999392658', - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'destination.ip': '178.162.1.2', - 'extra.event_id': '1614816012', - 'malware.name': 'kovter', - 'extra.http_referer': 'http://x.blogspot.com/', - 'extra.http_referer_ip': '142.250.3.4', - 'extra.http_referer_asn': 15169, - 'extra.http_referer_city': 'MOUNTAIN VIEW', - 'extra.http_referer_geo': 'US', - 'extra.http_referer_hostname': 'x.blogspot.com', - 'extra.http_referer_naics': 519130, - 'extra.http_referer_port': 80, - 'extra.http_referer_region': 'CALIFORNIA', - 'extra.http_referer_sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'protocol.transport': 'tcp', - 'feed.name': 'Sinkhole-Events-HTTP-Referer IPv4', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'time.source': '2021-03-04T00:00:12+00:00'}, - {'__type': 'Event', - 'classification.identifier': 'sinkhole-http-referer', - 'extra.tag': 'sunburst', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'destination.asn': 60781, - 'destination.fqdn': 'freescanonline.com', - 'destination.geolocation.cc': 'NL', - 'destination.geolocation.city': 'AMSTERDAM', - 'destination.geolocation.region': 'NOORD-HOLLAND', - 'destination.ip': '5.79.71.225', - 'destination.port': 80, - 'destination.url': 'http://freescanonline.com/personalationmall.com', - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'extra.event_id': '1614816013', - 'malware.name': 'sunburst', - 'extra.http_referer': 'http://www.example.com/teams/default.asp?u=EKL&t=c&s=lacrosse&p=remote&url=http://freescanonline.com/personalationmall.com', - 'extra.http_referer_asn': 14618, - 'extra.http_referer_city': 'ASHBURN', - 'extra.http_referer_geo': 'US', - 'extra.http_referer_hostname': 'www.example.com', - 'extra.http_referer_ip': '34.232.5.6', - 'extra.http_referer_naics': 454110, - 'extra.http_referer_port': 80, - 'extra.http_referer_region': 'VIRGINIA', - 'extra.http_referer_sector': 'Retail Trade', - 'protocol.transport': 'tcp', - 'feed.name': 'Sinkhole-Events-HTTP-Referer IPv4', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[4]])), - 'time.source': '2021-03-04T00:00:13+00:00'}, - {'__type': 'Event', - 'classification.identifier': 'sinkhole-http-referer', - 'extra.tag': 'sunburst', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'destination.asn': 60781, - 'destination.fqdn': 'freescanonline.com', - 'destination.geolocation.cc': 'NL', - 'destination.geolocation.city': 'AMSTERDAM', - 'destination.geolocation.region': 'NOORD-HOLLAND', - 'destination.port': 80, - 'destination.url': 'http://freescanonline.com/raftcomply.com', - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'destination.ip': '5.79.1.2', - 'extra.event_id': '1614816086', - 'malware.name': 'sunburst', - 'extra.http_referer': 'http://x.communes.jp/?url=http://freescanonline.com/raftcomply.com', - 'extra.http_referer_asn': 2516, - 'extra.http_referer_city': 'SAPPORO', - 'extra.http_referer_geo': 'JP', - 'extra.http_referer_hostname': 'x.communes.jp', - 'extra.http_referer_ip': '210.172.7.8', - 'extra.http_referer_naics': 517312, - 'extra.http_referer_port': 80, - 'extra.http_referer_region': 'HOKKAIDO', - 'extra.http_referer_sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'protocol.transport': 'tcp', - 'feed.name': 'Sinkhole-Events-HTTP-Referer IPv4', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[5]])), - 'time.source': '2021-03-04T00:01:26+00:00'}] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event6_sinkhole_http.py b/intelmq/tests/bots/parsers/shadowserver/test_event6_sinkhole_http.py deleted file mode 100644 index d6ff35dc11..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event6_sinkhole_http.py +++ /dev/null @@ -1,146 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/event6_sinkhole_http.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "Sinkhole-Events-HTTP IPv6", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-event6_sinkhole_http-test-geo.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : '3ve', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'infected-system', - 'destination.asn' : 6939, - 'destination.fqdn' : 'devps.net', - 'destination.geolocation.cc' : 'US', - 'destination.geolocation.city' : 'FREMONT', - 'destination.geolocation.region' : 'CALIFORNIA', - 'destination.ip' : '2001:470:1:332::fe', - 'destination.port' : 80, - 'destination.url' : 'http://devps.net/QKMSvF2hl11j%2fbMkyPbF5EpHYhd6VWTG4u19K3Rt7JGU3lMYRqpq8wPYEuOGKKeidKW3pefVfKSjBnL0cXizZbmuWWu8AQNRqw5g9Ny5vZtiv638XKoWwCLuUOTISTV%2fLcpcS1%2f22NjWqgXkHGISAuyVtafqyCC%2f5cA0eYg9Me8VzAIFDdTArogQOdYhElf2xluhEFPsstGQ%2bwrM4VmKHJpzyjD7Y%2fN%2bQV3wnZNdVkEVk1k2iKBJkotYv3ajgYWr56xxCbY5vE1IpZBRNhhaUDNZo0kJgi%2b6knXZ4m7JHt%2fGtJeP%2bNTxHSUL2ELlTIiT3ENlPYD6FdH6ZBxT1OneW%2f0ih%2fcN7vctb5B5Qwa1ez7ZjN2QxgBYkFDDHHTs42ej5eF2BysWAQDSUr%2fcySyGxcfPveIpfQEdrynGKR6z3OYqkFnP%2bYRDQp2rt1qt0FwCB4L9cg05TQlSSTJVGfPDrtcqjvKY4c9hWwSHtE8jMRpeCYO4Es%2bWgwr5DjzMicmuZo%2f4Ycr16jpN7xlDJdJ8iCFZxbSGgVC7ksVlGE8wlfWPI4KTuX5U5s61eNWPTlAC%2fOGb8grtw%2ffzizoIX9D6ZUMvslGLQIp%2fvNmNQkZy8HhNoV6Lns%2figITP%2fpN0H8h9HjUTl9qn65xFOEVpc0motSy8alcTPtTRKq5Jvc4Ao0x3N%2fvCB1v4Epx7XC0UpFbw8TrYEvAczEfGsGM', - 'extra.destination.naics' : 518210, - 'extra.destination.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.http_agent' : 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET4.0C; .NET4.0E)', - 'extra.infection' : 'boaxxe', - 'extra.tag' : '3ve', - 'feed.name' : 'Sinkhole-Events-HTTP IPv6', - 'malware.name' : 'boaxxe', - 'protocol.application' : 'http', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 7713, - 'source.geolocation.cc' : 'ID', - 'source.geolocation.city' : 'JAKARTA', - 'source.geolocation.region' : 'JAKARTA RAYA', - 'source.ip' : '2001:448a:1082:4d9b:7491:bf9e:3d5f:a634', - 'source.port' : 49431, - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2022-03-02T09:14:19+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : '3ve', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'infected-system', - 'destination.asn' : 6939, - 'destination.fqdn' : 'devps.net', - 'destination.geolocation.cc' : 'US', - 'destination.geolocation.city' : 'FREMONT', - 'destination.geolocation.region' : 'CALIFORNIA', - 'destination.ip' : '2001:470:1:332::ef', - 'destination.port' : 80, - 'destination.url' : 'http://devps.net/QKMSvF2hl11j%2fbMkyPbF5EpHYhd6VWTG4u19K3Rt7JGU3lMYRqpq8wPYEuOGKKeidKW3pefVfKSjBnL0cXizZbmuWWu8AQNRqw5g9Ny5vZtiv638XKoWwCLuUOTISTV%2fLcpcS1%2f22NjWqgXkHGISAuyVtafqyCC%2f5cA0eYg9Me8VzAIFDdTArogQOdYhElf2xluhEFPsstGQ%2bwrM4VmKHJpzyjD7Y%2fN%2bQV3wnZNdVkEVk1k2iKBJkotYv3ajgYWr56xxCbY5vE1IpZBRNhhaUDNZo0kJgi%2b6knXZ4m7JHt%2fGtJeP%2bNTxHSUL2ELlTIiT3ENlPYD6FdH6ZBxT1OneW%2f0ih%2fcN7vctb5B5Qwa1ez7ZjN2QxgBYkFDDHHTs42ej5eF2BysWAQDSUr%2fcySyGxcfPveIpfQEdrynGKR6z3OYqkFnP%2bYRDQp2rt1qt0FwCB4L9cg05TQlSSTJVGfPDrtcqjvKY4c9hWwSHtE8jMRpeCYO4Es%2bWgwr5DjzMicmuZo%2f4Ycr16jpN7xlDJdJ8iCFZxbSGgVC7ksVlGE8wlfWPI4KTuX5U5s61eNWPTlAC%2fOGb8grtw%2ffzizoIX9D6ZUMvslGLQIp%2fvNmNQkZy8HhNoV6Lns%2figITP%2fpN0H8h9HjUTl9qn65xFOEVpc0motSy8alcTPtTRKq5Jvc4Ao0x3N%2fvCB1v4Epx7XC0UpFbw8TrYEvAczEfGsGM', - 'extra.destination.naics' : 518210, - 'extra.destination.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.http_agent' : 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET4.0C; .NET4.0E)', - 'extra.infection' : 'boaxxe', - 'extra.tag' : '3ve', - 'feed.name' : 'Sinkhole-Events-HTTP IPv6', - 'malware.name' : 'boaxxe', - 'protocol.application' : 'http', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 7713, - 'source.geolocation.cc' : 'ID', - 'source.geolocation.city' : 'JAKARTA', - 'source.geolocation.region' : 'JAKARTA RAYA', - 'source.ip' : '2001:448a:1082:4d9b:7491:bf9e:3d5f:a634', - 'source.port' : 49460, - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2022-03-02T09:15:10+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : '3ve', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'infected-system', - 'destination.asn' : 6939, - 'destination.fqdn' : 'devps.net', - 'destination.geolocation.cc' : 'US', - 'destination.geolocation.city' : 'FREMONT', - 'destination.geolocation.region' : 'CALIFORNIA', - 'destination.ip' : '2001:470:1:332::fe', - 'destination.port' : 80, - 'destination.url' : 'http://devps.net/WMoUNCvuKGzdqSCeQcadP1%2f0B%2f3bzpOmyKBU85Z25HVOhvDQUPFl%2fk8uOcLewS%2b1BsuHXalRAOIgGOYYs2igj6UX8FkdCAmDewWPvfDhPD45nwd2tx1lLf2IoIfuOtIpGR6bN5Q6hGpSBgfERqCa0ImHcwfcZ2EdO%2fWvg7R8H6SLcTiuUC0I4pzvlWt1CRLgLdIEU1hZ0nnFHIHhchb6D7ITEgBQ2chQDxy5TJMrGjm4Dac6dKl%2ft5uYhRhSjAHkLLtgrJjsqVtVbelTAkt5kdcqLlO09m1SH%2fvtAb%2fOvR2DbhBss7%2f64DG7g6cAnghNA6JrFn1uW7sw%2bnKH8koKQwzUjdSsbrQAvmg4r0KDDW8Diq64gfDzxFWkzCLOYifc%2fwlinXPCl7aJiNCoieDC1U98RNQg%2f5td4SZmJnDQ2%2f96CPbFeSpCez5WD1rCjrxLj1h2cqzIgkydEWACceWP9ztxc4QaObzEcgOGxbRckWC7H2aaLeT8jaYEYdKi1pwEKChSL3YdEt4ZIb2IFrWwzNaXEpQzFXf07f902OEdI9vVA1ZdEOBPG6rAIkzMdebfprfVyhKEWtrCd3Skg3COUFtRQks5jzG1nv4sVGijTfSgyn6xE9Taka668Nycik6nmHy8Huj3oC01j3tee%2f1Z3eI6tV7lgM5d3uFJ84slRGHUCwMfVozOGmZRwNo%2fz%2bA', - 'extra.destination.naics' : 518210, - 'extra.destination.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.http_agent' : 'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko', - 'extra.infection' : 'boaxxe', - 'extra.source.naics' : 517311, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : '3ve', - 'feed.name' : 'Sinkhole-Events-HTTP IPv6', - 'malware.name' : 'boaxxe', - 'protocol.application' : 'http', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 11427, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'GARLAND', - 'source.geolocation.region' : 'TEXAS', - 'source.ip' : '2603:8080:b20a:dc00:f06e:8304:71f6:27e2', - 'source.port' : 62932, - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2022-03-02T14:15:10+00:00' -} - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_honeypot_brute_force.py b/intelmq/tests/bots/parsers/shadowserver/test_honeypot_brute_force.py deleted file mode 100644 index c376a73fbd..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_honeypot_brute_force.py +++ /dev/null @@ -1,72 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/event4_honeypot_brute_force.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Honeypot-Brute-Force-Events', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_honeypot_brute_force.csv" - } -EVENTS = [{'__type': 'Event', - 'classification.identifier': 'ssh', - 'classification.taxonomy': 'intrusion-attempts', - 'classification.type': 'brute-force', - 'extra.client_version': "b'SSH-2.0-Go'", - 'destination.asn': 26832, - 'destination.geolocation.cc': 'CA', - 'destination.geolocation.city': 'MONTREAL', - 'destination.geolocation.region': 'QUEBEC', - 'destination.ip': '162.250.1.2', - 'destination.port': 22, - 'extra.application': 'ssh', - 'extra.end_time': '2021-03-27T00:00:01.710968+00:00', - 'extra.public_source': 'CAPRICA-EU', - 'extra.start_time': '2021-03-27T00:00:00.521730+00:00', - 'malware.name': 'ssh-brute-force', - 'feed.name': 'Honeypot-Brute-Force-Events', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 209588, - 'source.geolocation.cc': 'NL', - 'source.geolocation.city': 'AMSTERDAM', - 'source.geolocation.region': 'NOORD-HOLLAND', - 'source.ip': '141.98.1.2', - 'source.port': 30123, - 'time.source': '2021-03-27T00:00:00+00:00'}, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_honeypot_ddos_amp.py b/intelmq/tests/bots/parsers/shadowserver/test_honeypot_ddos_amp.py deleted file mode 100644 index e95e59dcb3..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_honeypot_ddos_amp.py +++ /dev/null @@ -1,91 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/event4_honeypot_ddos_amp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Amplification DDoS Victim', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2019-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_honeypot_ddos_amp.csv" - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Amplification DDoS Victim', - 'classification.identifier': 'amplification-ddos-victim', - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'time.observation': '2019-01-01T00:00:00+00:00', - 'time.source': '2021-03-28T00:00:02+00:00', - 'source.ip': '107.141.1.2', - 'destination.port': 389, - 'source.reverse_dns': '192-0-2-10.example.net', - 'source.asn': 7018, - 'source.geolocation.cc': 'US', - 'source.geolocation.region': 'VISALIA', - 'source.geolocation.city': 'VISALIA', - 'source.geolocation.region': 'CALIFORNIA', - 'extra.end_time': '2021-03-28T00:20:22+00:00', - 'extra.public_source': 'CISPA', - 'extra.source.naics': 517311, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'malware.name': 'ddos-amplification', - 'source.reverse_dns': '107-141-x-x.lightspeed.frsnca.sbcglobal.net', - }, - {'__type': 'Event', - 'feed.name': 'Amplification DDoS Victim', - 'classification.identifier': 'amplification-ddos-victim', - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'time.observation': '2019-01-01T00:00:00+00:00', - 'time.source': '2021-03-28T00:00:02+00:00', - 'source.ip': '74.59.3.4', - 'destination.port': 389, - 'source.reverse_dns': 'modemcablex-x-59-74.mc.videotron.ca', - 'source.asn': 5769, - 'source.geolocation.cc': 'CA', - 'source.geolocation.city': 'CHICOUTIMI', - 'source.geolocation.region': 'QUEBEC', - 'extra.end_time': '2021-03-28T00:13:50+00:00', - 'extra.public_source': 'CISPA', - 'extra.source.naics': 517311, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'malware.name': 'ddos-amplification', - }, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_malware_url.py b/intelmq/tests/bots/parsers/shadowserver/test_malware_url.py deleted file mode 100644 index b19b200b5f..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_malware_url.py +++ /dev/null @@ -1,107 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/malware_url.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Malware URL', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-malware_url-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'malware-url', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'extra.application' : 'http', - 'source.url' : 'http://41.86.0.0:50008/Mozi.m', - 'feed.name' : 'Malware URL', - 'malware.hash.sha256' : '12013662c71da69de977c04cd7021f13a70cf7bed4ca6c82acbc100464d4b0ef', - 'malware.name' : 'cve-2016-10372', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 37203, - 'source.geolocation.cc' : 'LR', - 'source.geolocation.city' : 'MONROVIA', - 'source.geolocation.region' : 'MONTSERRADO', - 'source.ip' : '41.86.0.0', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-07T00:02:07+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'malware-url', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'extra.application' : 'http', - 'extra.source.naics' : 517311, - 'source.url' : 'http://42.225.0.0:38173/Mozi.m', - 'feed.name' : 'Malware URL', - 'malware.name' : 'cve-2018-10562', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 4837, - 'source.geolocation.cc' : 'CN', - 'source.geolocation.city' : 'ZHUMADIAN', - 'source.geolocation.region' : 'HENAN SHENG', - 'source.ip' : '42.225.0.0', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-07T00:03:14+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'malware-url', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'extra.application' : 'http', - 'extra.source.naics' : 517311, - 'source.url' : 'http://211.52.0.0:53029/Mozi.m', - 'feed.name' : 'Malware URL', - 'malware.name' : 'cve-2018-10562', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 4766, - 'source.geolocation.cc' : 'KR', - 'source.geolocation.city' : 'SAGOK-MYEON', - 'source.geolocation.region' : 'CHUNGCHEONGNAM-DO', - 'source.ip' : '211.52.0.0', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-07T00:10:26+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_phish_url.py b/intelmq/tests/bots/parsers/shadowserver/test_phish_url.py deleted file mode 100644 index 0783372f91..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_phish_url.py +++ /dev/null @@ -1,106 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/phish_url.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Phish URL', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-phish_url-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'phish-url', - 'classification.taxonomy' : 'fraud', - 'classification.type' : 'phishing', - 'source.fqdn' : 'priceless-pare.example.net', - 'extra.source' : 'openphish.com', - 'extra.source.naics' : 518210, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'source.url' : 'https://priceless-pare.example.net/Postal-/acec6/', - 'feed.name' : 'Phish URL', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'BUFFALO', - 'source.geolocation.region' : 'NEW YORK', - 'source.ip' : '172.245.0.0', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-02-01T08:00:07+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'phish-url', - 'classification.taxonomy' : 'fraud', - 'classification.type' : 'phishing', - 'source.fqdn' : 'mailyahooattt.example.net', - 'extra.source' : 'openphish.com', - 'extra.source.sector' : 'Professional, Scientific, and Technical Services', - 'source.url' : 'https://mailyahooattt.example.net/', - 'feed.name' : 'Phish URL', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'SAN FRANCISCO', - 'source.geolocation.region' : 'CALIFORNIA', - 'source.ip' : '199.34.0.0', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-02-01T08:00:07+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'phish-url', - 'classification.taxonomy' : 'fraud', - 'classification.type' : 'phishing', - 'source.fqdn' : 'www.example.net', - 'extra.source' : 'openphish.com', - 'extra.source.naics' : 519130, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'source.url' : 'https://www.example.net/viewer/vbid-730ec2b1-omsttuer', - 'feed.name' : 'Phish URL', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'DRAPER', - 'source.geolocation.region' : 'UTAH', - 'source.ip' : '216.58.0.0', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-02-01T08:00:07+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_population_http_proxy.py b/intelmq/tests/bots/parsers/shadowserver/test_population_http_proxy.py deleted file mode 100644 index e9f11a47c3..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_population_http_proxy.py +++ /dev/null @@ -1,130 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/population_http_proxy.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible HTTP Proxy', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-population_http_proxy-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'accessible-http-proxy', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'extra.connection': 'keep-alive', - 'extra.content_length': 3741, - 'extra.content_type': 'text/html;charset=utf-8', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 407, - 'extra.http_date': '2010-02-10T00:00:00+00:00', - 'extra.http_reason': 'Proxy Authentication Required', - 'extra.proxy_authenticate': 'Basic realm=\\\\"Squid proxy-caching web ' - 'server\\"\\""', - 'extra.server': 'squid/4.10', - 'feed.name': 'Accessible HTTP Proxy', - 'malware.name': 'http-connect-proxy-closed', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 3128, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-http-proxy', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'extra.connection': 'keep-alive', - 'extra.content_length': 3833, - 'extra.content_type': 'text/html;charset=utf-8', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 407, - 'extra.http_date': '2010-02-10T00:00:01+00:00', - 'extra.http_reason': 'Proxy Authentication Required', - 'extra.proxy_authenticate': 'Basic realm=\\\\"00:23:24:43:1c:34\\"\\""', - 'feed.name': 'Accessible HTTP Proxy', - 'malware.name': 'http-connect-proxy-closed', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 3128, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-http-proxy', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'extra.connection': 'keep-alive', - 'extra.content_length': 179, - 'extra.content_type': 'text/html;charset=utf-8', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 407, - 'extra.http_date': '2010-02-10T00:00:02+00:00', - 'extra.http_reason': 'Proxy Authentication Required', - 'extra.proxy_authenticate': 'Basic realm=\\\\"Proxy\\"\\""', - 'feed.name': 'Accessible HTTP Proxy', - 'malware.name': 'http-connect-proxy-closed', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 3128, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_sandbox_conn.py b/intelmq/tests/bots/parsers/shadowserver/test_sandbox_conn.py deleted file mode 100644 index c5da823465..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_sandbox_conn.py +++ /dev/null @@ -1,99 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/sandbox_conn.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Sandbox Connections', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-sandbox_conn-test.csv", - } -EVENTS = [ - { - '__type' : 'Event', - 'classification.identifier' : 'sandbox-conn', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'destination.fqdn' : 'time.windows.com', - 'feed.name' : 'Sandbox Connections', - 'malware.hash.md5' : 'b575ce6dcce6502a8431db5610135c25', - 'protocol.transport' : 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 8075, - 'source.geolocation.cc' : 'US', - 'source.ip' : '40.119.6.228', - 'source.port' : 123, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:00:03+00:00' -}, - { - '__type' : 'Event', - 'classification.identifier' : 'sandbox-conn', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'feed.name' : 'Sandbox Connections', - 'malware.hash.md5' : 'c0d947f9a8685b0d9f3efdba966389c2', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 3356, - 'source.geolocation.cc' : 'US', - 'source.ip' : '8.252.70.126', - 'source.port' : 80, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:00:03+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'sandbox-conn', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'feed.name' : 'Sandbox Connections', - 'malware.hash.md5' : 'c0d947f9a8685b0d9f3efdba966389c2', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 8075, - 'source.geolocation.cc' : 'US', - 'source.ip' : '52.109.8.22', - 'source.port' : 443, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:00:03+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_sandbox_dns.py b/intelmq/tests/bots/parsers/shadowserver/test_sandbox_dns.py deleted file mode 100644 index 70cf1eee5e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_sandbox_dns.py +++ /dev/null @@ -1,95 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/sandbox_dns.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Sandbox DNS', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-sandbox_dns-test.csv", - } -EVENTS = [ - { - '__type' : 'Event', - 'classification.identifier' : 'sandbox-dns', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.request' : 'time.windows.com', - 'extra.response' : '40.119.6.228', - 'extra.dns_query_type' : 'A', - 'feed.name' : 'Sandbox DNS', - 'malware.hash.md5' : 'b575ce6dcce6502a8431db5610135c25', - 'protocol.application' : 'dns', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:00:02+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'sandbox-dns', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.request' : 'time.windows.com', - 'extra.response' : '40.119.6.228', - 'extra.dns_query_type' : 'A', - 'feed.name' : 'Sandbox DNS', - 'malware.hash.md5' : '807679198a39c80d3ca07e60fd51b581', - 'protocol.application' : 'dns', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:00:08+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'sandbox-dns', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.request' : 'client-office365-tas.msedge.net', - 'extra.response' : '13.107.5.88', - 'extra.dns_query_type' : 'A', - 'feed.name' : 'Sandbox DNS', - 'malware.hash.md5' : 'd97e973b9bf073bd3a217425259cea26', - 'protocol.application' : 'dns', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:00:20+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_sandbox_url.py b/intelmq/tests/bots/parsers/shadowserver/test_sandbox_url.py deleted file mode 100644 index 91b0154b84..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_sandbox_url.py +++ /dev/null @@ -1,104 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/sandbox_url.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Sandbox URL', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-sandbox_url-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'sandbox-url', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'destination.fqdn' : 'www.msftncsi.com', - 'extra.http_request_method' : 'GET', - 'destination.url' : 'http://www.msftncsi.com/ncsi.txt', - 'extra.user_agent' : 'Microsoft NCSI', - 'feed.name' : 'Sandbox URL', - 'malware.hash.md5' : '37514b54e679a5313334e830ad780ec7', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 20940, - 'source.geolocation.cc' : 'US', - 'source.ip' : '23.196.47.89', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:13+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'sandbox-url', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'destination.fqdn' : 'www.download.windowsupdate.com', - 'extra.http_request_method' : 'GET', - 'destination.url' : 'http://www.download.windowsupdate.com/msdownload/update/v3/static/trustedr/en/authrootstl.cab', - 'extra.user_agent' : 'Microsoft-CryptoAPI/6.1', - 'feed.name' : 'Sandbox URL', - 'malware.hash.md5' : '37514b54e679a5313334e830ad780ec7', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 15133, - 'source.geolocation.cc' : 'US', - 'source.ip' : '72.21.81.240', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:28+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'sandbox-url', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'destination.fqdn' : 'crl.microsoft.com', - 'extra.http_request_method' : 'GET', - 'destination.url' : 'http://crl.microsoft.com/pki/crl/products/MicTimStaPCA_2010-07-01.crl', - 'extra.user_agent' : 'Microsoft-CryptoAPI/6.1', - 'feed.name' : 'Sandbox URL', - 'malware.hash.md5' : 'e97ea2820c0d79f3f3ca241d4dcd1060', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 20940, - 'source.geolocation.cc' : 'US', - 'source.ip' : '23.56.4.57', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:08:24+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_adb.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_adb.py deleted file mode 100644 index 6bc6e61461..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_adb.py +++ /dev/null @@ -1,98 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_adb.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible ADB', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2018-07-30T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_adb-test-test.csv", - - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Accessible ADB', - 'time.observation': '2018-07-30T00:00:00+00:00', - 'time.source': '2018-07-26T02:07:16+00:00', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-adb', - 'protocol.application': 'adb', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 3462, - 'source.geolocation.cc': 'TW', - 'source.geolocation.city': 'TAOYUAN CITY', - 'source.geolocation.region': 'TAOYUAN COUNTY', - 'source.ip': '36.239.124.210', - 'source.port': 5555, - 'extra.name': 'hlteuc', - 'extra.model': 'SAMSUNG-SM-N900A', - 'extra.device': 'hlteatt', - 'extra.tag': 'adb', - 'extra.naics': 518210, - 'extra.sic': 737415, - 'source.reverse_dns': '36-239-124-210.dynamic-ip.hinet.net', - }, - {'__type': 'Event', - 'feed.name': 'Accessible ADB', - 'time.observation': '2018-07-30T00:00:00+00:00', - 'time.source': '2018-07-26T02:07:16+00:00', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-adb', - 'protocol.application': 'adb', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn': 3462, - 'source.geolocation.cc': 'TW', - 'source.geolocation.city': 'TAIPEI', - 'source.geolocation.region': 'TAIPEI CITY', - 'source.ip': '36.236.108.107', - 'source.port': 5555, - 'extra.name': 'marlin', - 'extra.model': 'Pixel XL', - 'extra.device': 'marlin', - 'extra.features': 'cmd,shell_v2', - 'extra.naics': 518210, - 'extra.sic': 737415, - 'extra.tag': 'adb', - 'source.reverse_dns': '36-236-108-107.dynamic-ip.hinet.net', - }, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_afp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_afp.py deleted file mode 100644 index cc30b1e4c0..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_afp.py +++ /dev/null @@ -1,106 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_afp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible AFP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2018-07-30T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_afp-test-test.csv", - - } -EVENTS = [{ - '__type': 'Event', - 'feed.name': 'Accessible AFP', - "classification.identifier": "accessible-afp", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.afp_versions": "AFP3.3,AFP3.2,AFP3.1", - "extra.flags": "SupportsCopyFile,SupportsChgPwd,SupportsServerMessages,SupportsServerSignature,SupportsTCP/IP,SupportsSrvrNotifications,SupportsReconnect,SupportsOpenDirectory,SupportsUTF8Servername,SupportsUUIDs,SupportsSuperClient", - "extra.machine_type": "TimeCapsule8,119", - "extra.naics": 517311, - "extra.network_address": "198.33.24.165:548,10.0.1.1:548,fe80:0008:0000:0000:6e70:9fff:fed4::548,fe80:0009:0000:0000:6e70:9fff:fed4::548,179.24.24.165 (DNS address),", - "extra.server_name": "airport-time-capsule-de-jack", - "extra.signature": "4338364e37364442463948350069672d", - "extra.tag": "afp", - "extra.uams": "DHCAST128,DHX2,SRP,Recon1", - "extra.utf8_servername": "AirPort Time Capsule de jack", - "protocol.application": "afp", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 6057, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.13.34.22", - "source.port": 548, - "source.reverse_dns": "host.local", - "time.observation": "2018-07-30T00:00:00+00:00", - "time.source": "2019-09-04T05:05:53+00:00" -}, -{ - '__type': 'Event', - 'feed.name': 'Accessible AFP', - "classification.identifier": "accessible-afp", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.afp_versions": "AFP3.3,AFP3.2,AFP3.1", - "extra.flags": "SupportsCopyFile,SupportsChgPwd,SupportsServerMessages,SupportsServerSignature,SupportsTCP/IP,SupportsSrvrNotifications,SupportsReconnect,SupportsOpenDirectory,SupportsUTF8Servername,SupportsUUIDs,SupportsSuperClient", - "extra.machine_type": "TimeCapsule8,119", - "extra.naics": 517311, - "extra.network_address": "0.0.0.1:548,10.0.1.1:548,198.33.42.1:548,fe80:000b:0000:0000:dea4:caff:feba::548,fe80:000c:0000:0000:dea4:caff:feba::548,fe80:000d:0000:0000:4c7d:ffff:fec7::548,0.0.0.1 (DNS address),", - "extra.server_name": "time-capsule-del-jack", - "extra.signature": "433836544b303147463948360069672d", - "extra.tag": "afp", - "extra.uams": "DHCAST128,DHX2,SRP,Recon1", - "extra.utf8_servername": "Time Capsule del Jack", - "protocol.application": "afp", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 6057, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.40.27.212", - "source.port": 548, - "source.reverse_dns": "host.local", - "time.observation": "2018-07-30T00:00:00+00:00", - "time.source": "2019-09-04T05:05:56+00:00" - },] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_amqp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_amqp.py deleted file mode 100644 index df707f30b0..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_amqp.py +++ /dev/null @@ -1,144 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_amqp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible AMQP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_amqp-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'accessible-amqp', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.capabilities' : 'publisher_confirms,exchange_exchange_bindings,basic.nack,consumer_cancel_notify,connection.blocked,consumer_priorities,authentication_failure_close,per_consumer_qos', - 'extra.class' : '10', - 'extra.cluster_name' : 'rabbit@iZuf63m0nnq9bwf7lhjxrkZ', - 'extra.locales' : 'en_US', - 'extra.mechanisms' : 'PLAIN AMQPLAIN', - 'extra.message_length' : 509, - 'extra.method' : '10', - 'extra.platform' : 'Erlang/OTP', - 'extra.product' : 'RabbitMQ', - 'extra.product_version' : '3.3.5', - 'extra.naics' : 518210, - 'extra.tag' : 'amqp', - 'extra.version_minor' : '9', - 'feed.name' : 'Accessible AMQP', - 'protocol.application' : 'amqp', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 37963, - 'source.geolocation.cc' : 'CN', - 'source.geolocation.city' : 'SHANGHAI', - 'source.geolocation.region' : 'SHANGHAI SHI', - 'source.ip' : '47.103.0.0', - 'source.port' : 5672, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T04:32:13+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'accessible-amqp', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.capabilities' : 'publisher_confirms,exchange_exchange_bindings,basic.nack,consumer_cancel_notify,connection.blocked,consumer_priorities,authentication_failure_close,per_consumer_qos,direct_reply_to', - 'extra.class' : '10', - 'extra.cluster_name' : 'rabbit@mtk-breizh', - 'extra.locales' : 'en_US', - 'extra.mechanisms' : 'AMQPLAIN PLAIN', - 'extra.message_length' : 509, - 'extra.method' : '10', - 'extra.platform' : 'Erlang/OTP 24.0.3', - 'extra.product' : 'RabbitMQ', - 'extra.product_version' : '3.8.19', - 'extra.naics' : 518210, - 'extra.tag' : 'amqp', - 'extra.version_minor' : '9', - 'feed.name' : 'Accessible AMQP', - 'protocol.application' : 'amqp', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 16276, - 'source.geolocation.cc' : 'DE', - 'source.geolocation.city' : 'SAARBRUCKEN', - 'source.geolocation.region' : 'SAARLAND', - 'source.ip' : '141.95.0.0', - 'source.port' : 5672, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T04:32:13+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'accessible-amqp', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.capabilities' : 'publisher_confirms,exchange_exchange_bindings,basic.nack,consumer_cancel_notify,connection.blocked,consumer_priorities,authentication_failure_close,per_consumer_qos,direct_reply_to', - 'extra.class' : '10', - 'extra.cluster_name' : 'rabbit@1397a0e9629b', - 'extra.locales' : 'en_US', - 'extra.mechanisms' : 'PLAIN AMQPLAIN', - 'extra.message_length' : 509, - 'extra.method' : '10', - 'extra.platform' : 'Erlang/OTP 24.2', - 'extra.product' : 'RabbitMQ', - 'extra.product_version' : '3.9.11', - 'extra.naics' : 454110, - 'extra.tag' : 'amqp', - 'extra.version_minor' : '9', - 'feed.name' : 'Accessible AMQP', - 'protocol.application' : 'amqp', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 14618, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'ASHBURN', - 'source.geolocation.region' : 'VIRGINIA', - 'source.ip' : '54.234.0.0', - 'source.port' : 5672, - 'source.reverse_dns' : 'ec2-54.234.0.0.compute-1.amazonaws.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T04:32:13+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ard.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ard.py deleted file mode 100644 index 4d8420c3bb..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ard.py +++ /dev/null @@ -1,111 +0,0 @@ -# SPDX-FileCopyrightText: 2020 Tomas Bellus -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ard.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible ARD', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2020-07-20T00:00:00+00:00", - "extra.file_name": "2020-01-01-scan_ard-test-test.csv", - - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'accessible-ard', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 201.2, - 'extra.machine_name': 'Macmini (radio)', - 'extra.response_size': 1006, - 'extra.tag': 'ard', - 'feed.name': 'Accessible ARD', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 3283, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-ard', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 201.2, - 'extra.machine_name': 'biuro-rip-org-pl', - 'extra.response_size': 1006, - 'extra.tag': 'ard', - 'feed.name': 'Accessible ARD', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 3283, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-ard', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 201.2, - 'extra.machine_name': '127.0.0.1', - 'extra.response_size': 1006, - 'extra.tag': 'ard', - 'feed.name': 'Accessible ARD', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 3283, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_chargen.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_chargen.py deleted file mode 100644 index 3b72baa8db..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_chargen.py +++ /dev/null @@ -1,110 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_chargen.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open Chargen', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_chargen-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-chargen', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 74.0, - 'extra.response_size': 74, - 'extra.tag': 'chargen', - 'feed.name': 'Open Chargen', - 'protocol.application': 'chargen', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 19, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-chargen', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 74.0, - 'extra.response_size': 74, - 'extra.tag': 'chargen', - 'feed.name': 'Open Chargen', - 'protocol.application': 'chargen', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 19, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-chargen', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 74.0, - 'extra.response_size': 74, - 'extra.sector': 'Government', - 'extra.tag': 'chargen', - 'feed.name': 'Open Chargen', - 'protocol.application': 'chargen', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 19, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_cisco_smart_install.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_cisco_smart_install.py deleted file mode 100644 index 46c963a79e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_cisco_smart_install.py +++ /dev/null @@ -1,82 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_cisco_smart_install.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible Cisco Smart Install', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_cisco_smart_install-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Accessible Cisco Smart Install', - 'classification.identifier': 'accessible-cisco-smart-install', - 'classification.type': 'vulnerable-system', - 'classification.taxonomy': 'vulnerable', - 'protocol.application': 'cisco-smart-install', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 8559, - 'source.geolocation.cc': 'AT', - 'source.geolocation.city': 'VIENNA', - 'source.geolocation.region': 'WIEN', - 'source.ip': '198.51.100.103', - 'source.port': 4786, - 'extra.tag': 'cisco-smart-install', - 'source.reverse_dns': '198-51-100-103.example.net', - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2017-11-18T08:42:45+00:00'}, - {'__type': 'Event', - 'feed.name': 'Accessible Cisco Smart Install', - 'classification.identifier': 'accessible-cisco-smart-install', - 'classification.type': 'vulnerable-system', - 'classification.taxonomy': 'vulnerable', - 'protocol.application': 'cisco-smart-install', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn': 35609, - 'source.geolocation.cc': 'AT', - 'source.geolocation.city': 'VIENNA', - 'source.geolocation.region': 'WIEN', - 'source.ip': '198.51.100.218', - 'source.port': 4786, - 'extra.tag': 'cisco-smart-install', - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2017-11-18T08:47:54+00:00'}, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_coap.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_coap.py deleted file mode 100644 index 773fc04d51..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_coap.py +++ /dev/null @@ -1,121 +0,0 @@ -# SPDX-FileCopyrightText: 2020 Thomas Hungenberg -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_coap.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible-CoAP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2020-06-29T00:00:00+00:00", - "extra.file_name": "2020-06-28-scan_coap-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'accessible-coap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 2.05, - 'extra.response': ',,', - 'extra.response_size': 43, - 'extra.tag': 'coap', - 'extra.version': '2', - 'feed.name': 'Accessible-CoAP', - 'protocol.application': 'coap', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 5683, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-coap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 5.38, - 'extra.response': ',,,,,,,,,', - 'extra.response_size': 113, - 'extra.tag': 'coap', - 'extra.version': '2', - 'feed.name': 'Accessible-CoAP', - 'protocol.application': 'coap', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 5683, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-coap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 113.5, - 'extra.response': '`EsjAy************************************************************|CoAP ' - 'RFC 7252 ' - '|************************************************************|This ' - 'server is using the Eclipse Californium (Cf) CoAP ' - 'framework|published under EPL+EDL: ' - 'http://www.eclipse.org/californium/||(c) 2014, 2015, 2016 ' - 'Institute for Pervasive Computing, ETH Zurich and ' - 'others|************************************************************', - 'extra.response_size': 454, - 'extra.tag': 'coap', - 'extra.version': '1', - 'feed.name': 'Accessible-CoAP', - 'protocol.application': 'coap', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 5683, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_couchdb.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_couchdb.py deleted file mode 100644 index 1bf6f321c6..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_couchdb.py +++ /dev/null @@ -1,128 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_couchdb.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible CouchDB Server', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_couchdb-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-couchdb', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.couchdb_message' : 'Welcome', - 'extra.couchdb_version' : '1.6.1', - 'extra.server_version' : 'CouchDB/1.6.1 (Erlang OTP/18)', - 'extra.tag' : 'couchdb', - 'extra.vendor' : 'Ubuntu 16.04', - 'extra.visible_databases' : '_replicator;_users;test;shops;god', - 'feed.name' : 'Accessible CouchDB Server', - 'protocol.application' : 'couchdb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 5984, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-couchdb', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.couchdb_message' : 'Welcome', - 'extra.couchdb_version' : '3.2.1', - 'extra.features' : 'access-ready,partitioned,pluggable-storage-engines,reshard,scheduler', - 'extra.git_sha' : '244d428af', - 'extra.server_version' : 'CouchDB/3.2.1 (Erlang OTP/23)', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'couchdb', - 'extra.vendor' : 'The Apache Software Foundation', - 'feed.name' : 'Accessible CouchDB Server', - 'protocol.application' : 'couchdb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 5984, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-couchdb', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.couchdb_message' : 'Welcome', - 'extra.couchdb_version' : '3.2.1', - 'extra.features' : 'access-ready,partitioned,pluggable-storage-engines,reshard,scheduler', - 'extra.git_sha' : '244d428af', - 'extra.server_version' : 'CouchDB/3.2.1 (Erlang OTP/20)', - 'extra.source.sector' : 'Retail Trade', - 'extra.tag' : 'couchdb', - 'extra.vendor' : 'The Apache Software Foundation', - 'feed.name' : 'Accessible CouchDB Server', - 'protocol.application' : 'couchdb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 5984, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_cwmp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_cwmp.py deleted file mode 100644 index b508b64508..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_cwmp.py +++ /dev/null @@ -1,103 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_cwmp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible CWMP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2018-07-30T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_cwmp-test-test.csv", - - } -EVENTS = [{ - '__type': 'Event', - 'feed.name': 'Accessible CWMP', - "classification.identifier": "open-cwmp", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.connection": "keep-alive", - "extra.content_length": 5678, - "extra.content_type": "text/html", - "extra.date": "Wed, 04 Sep 2019 07:42:37 GMT", - "extra.http": "HTTP/1.1", - "extra.http_code": 200, - "extra.http_reason": "OK", - "extra.naics": 517311, - "extra.server": "DNVRS-Webs", - "extra.tag": "cwmp", - "protocol.application": "cwmp", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 5678, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.142", - "source.port": 30005, - "time.observation": "2018-07-30T00:00:00+00:00", - "time.source": "2019-09-04T10:44:55+00:00" -}, -{ - '__type': 'Event', - 'feed.name': 'Accessible CWMP', - "classification.identifier": "open-cwmp", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.content_type": "text/html", - "extra.http": "HTTP/1.1", - "extra.http_code": 404, - "extra.http_reason": "Not Found", - "extra.naics": 517311, - "extra.server": "RomPager/4.07 UPnP/1.0", - "extra.tag": "cwmp", - "extra.transfer_encoding": "chunked", - "protocol.application": "cwmp", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 5678, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.162", - "source.port": 5678, - "source.reverse_dns": "localhost.localdomain", - "time.observation": "2018-07-30T00:00:00+00:00", - "time.source": "2019-09-04T11:06:50+00:00" - },] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_db2.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_db2.py deleted file mode 100644 index 423ebe8c53..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_db2.py +++ /dev/null @@ -1,91 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Sebastian Wagner -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_db2.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "ShadowServer Open-DB2-Discovery-Service", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_db2-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-db2-discovery-service', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 14.9, - 'extra.db2_hostname': 'NOWAK_SERWER', - 'extra.servername': 'node01.example.com', - 'extra.size': 298, - 'extra.tag': 'db2', - 'feed.name': 'ShadowServer Open-DB2-Discovery-Service', - 'protocol.application': 'db2', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 523, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-db2-discovery-service', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 14.9, - 'extra.db2_hostname': 'SPZOZ-DZIEWIN', - 'extra.servername': 'node02.example.com', - 'extra.size': 298, - 'extra.tag': 'db2', - 'feed.name': 'ShadowServer Open-DB2-Discovery-Service', - 'protocol.application': 'db2', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 523, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ddos_middlebox.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ddos_middlebox.py deleted file mode 100644 index 9038a79ef1..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ddos_middlebox.py +++ /dev/null @@ -1,119 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ddos_middlebox.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'DDoS Middlebox', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_ddos_middlebox-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ddos-middlebox', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.amplification' : 2, - 'extra.bytes' : 99, - 'extra.method' : 'SYN+ACK:PSH', - 'extra.source_port' : '49002', - 'feed.name' : 'DDoS Middlebox', - 'protocol.application' : 'ddos-middlebox', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 80, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ddos-middlebox', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.amplification' : 2, - 'extra.bytes' : 99, - 'extra.method' : 'SYN+ACK:PSH', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.source_port' : '41200', - 'feed.name' : 'DDoS Middlebox', - 'protocol.application' : 'ddos-middlebox', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 80, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ddos-middlebox', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.amplification' : 2, - 'extra.bytes' : 99, - 'extra.method' : 'SYN+ACK:PSH', - 'extra.source_port' : '47492', - 'feed.name' : 'DDoS Middlebox', - 'protocol.application' : 'ddos-middlebox', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 80, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_dns.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_dns.py deleted file mode 100644 index 3492f82cec..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_dns.py +++ /dev/null @@ -1,91 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_dns.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'DNS Open Resolvers', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2018-07-30T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_dns-test-test.csv", - } -EVENTS = [{ - '__type': 'Event', - 'feed.name': 'DNS Open Resolvers', - "classification.identifier": "dns-open-resolver", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.dns_version": "dnsmasq-2.66", - "extra.min_amplification": 4.619, - "extra.tag": "openresolver", - "protocol.application": "dns", - "protocol.transport": "udp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 25255, - "source.geolocation.cc": "AT", - "source.geolocation.city": "VIENNA", - "source.geolocation.region": "WIEN", - "source.ip": "198.51.100.179", - "source.port": 53, - "source.reverse_dns": "198-51-100-189.example.net", - "time.observation": "2018-07-30T00:00:00+00:00", - "time.source": "2018-04-14T00:14:34+00:00" -}, -{ - '__type': 'Event', - 'feed.name': 'DNS Open Resolvers', - "classification.identifier": "dns-open-resolver", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.dns_version": "dnsmasq-2.51", - "extra.min_amplification": 4.619, - "extra.tag": "openresolver", - "protocol.application": "dns", - "protocol.transport": "udp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 25255, - "source.geolocation.cc": "AT", - "source.geolocation.city": "VIENNA", - "source.geolocation.region": "WIEN", - "source.ip": "198.51.100.8", - "source.port": 53, - "source.reverse_dns": "198-51-100-111.example.net", - "time.observation": "2018-07-30T00:00:00+00:00", - "time.source": "2018-04-14T00:14:36+00:00" -},] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_docker.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_docker.py deleted file mode 100644 index 31d0e4417e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_docker.py +++ /dev/null @@ -1,159 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_docker.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible Docker Service', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_docker-test.csv", - } -EVENTS = [ - { - '__type' : 'Event', - 'classification.identifier' : 'open-docker', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.api_version' : '1.37', - 'extra.arch' : 'amd64', - 'extra.build_time' : '2018-05-09T22:18:36.000000000+00:00', - 'extra.content_type' : 'application/json; charset=UTF-8', - 'extra.date' : 'Fri, 06 May 2022 14:06:30 GMT', - 'extra.experimental' : 'false', - 'extra.git_commit' : 'f150324', - 'extra.go_version' : 'go1.9.5', - 'extra.http' : 'HTTP/1.1', - 'extra.http_code' : 200, - 'extra.http_reason' : 'OK', - 'extra.kernel_version' : '3.10.0-514.26.2.el7.x86_64', - 'extra.min_api_version' : '1.12', - 'extra.os.name' : 'linux', - 'extra.server' : 'Docker/18.05.0-ce (linux)', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'docker', - 'extra.version' : '18.05.0-ce', - 'feed.name' : 'Accessible Docker Service', - 'protocol.application' : 'docker', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 2375, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-docker', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.api_version' : '1.26', - 'extra.arch' : 'amd64', - 'extra.build_time' : '2022-03-02T15:25:43.414574467+00:00', - 'extra.content_type' : 'application/json', - 'extra.date' : 'Fri, 06 May 2022 14:08:07 GMT', - 'extra.experimental' : 'false', - 'extra.git_commit' : '7d71120/1.13.1', - 'extra.go_version' : 'go1.10.3', - 'extra.http' : 'HTTP/1.1', - 'extra.http_code' : 200, - 'extra.http_reason' : 'OK', - 'extra.kernel_version' : '3.10.0-693.2.2.el7.x86_64', - 'extra.min_api_version' : '1.12', - 'extra.os.name' : 'linux', - 'extra.pkg_version' : 'docker-1.13.1-209.git7d71120.el7.centos.x86_64', - 'extra.server' : 'Docker/1.13.1 (linux)', - 'extra.tag' : 'docker', - 'extra.version' : '1.13.1', - 'feed.name' : 'Accessible Docker Service', - 'protocol.application' : 'docker', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 2375, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-docker', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.api_version' : '1.37', - 'extra.arch' : 'amd64', - 'extra.build_time' : '2018-05-09T22:18:36.000000000+00:00', - 'extra.content_type' : 'application/json; charset=UTF-8', - 'extra.date' : 'Fri, 06 May 2022 14:08:06 GMT', - 'extra.experimental' : 'false', - 'extra.git_commit' : 'f150324', - 'extra.go_version' : 'go1.9.5', - 'extra.http' : 'HTTP/1.1', - 'extra.http_code' : 200, - 'extra.http_reason' : 'OK', - 'extra.kernel_version' : '3.10.0-514.26.2.el7.x86_64', - 'extra.min_api_version' : '1.12', - 'extra.os.name' : 'linux', - 'extra.server' : 'Docker/18.05.0-ce (linux)', - 'extra.tag' : 'docker', - 'extra.version' : '18.05.0-ce', - 'feed.name' : 'Accessible Docker Service', - 'protocol.application' : 'docker', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 2375, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_dvr_dhcpdiscover.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_dvr_dhcpdiscover.py deleted file mode 100644 index 01e68db94b..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_dvr_dhcpdiscover.py +++ /dev/null @@ -1,178 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_dvr_dhcpdiscover.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible DVR DHCPDiscover', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_dvr_dhcpdiscover-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-dvr-dhcpdiscover', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.alarm_input_channels': 0, - 'extra.alarm_output_channels': 0, - 'extra.amplification': 794.0, - 'extra.device_model': 'BCS-TIP3401IR-E-V', - 'extra.device_serial': '6J0E022PAG35073', - 'extra.device_type': 'IPC', - 'extra.device_vendor': 'General', - 'extra.device_version': '2.800.106F004.0.R', - 'extra.http_port': 80, - 'extra.internal_port': 37777, - 'extra.ipv4_address': '192.168.0.1', - 'extra.ipv4_dhcp_enable': False, - 'extra.ipv4_gateway': '192.168.0.240', - 'extra.ipv4_subnet_mask': '255.255.255.0', - 'extra.ipv6_address': 'fd09:4ab5:dae9:b078::1', - 'extra.ipv6_dhcp_enable': False, - 'extra.ipv6_gateway': 'fd09:4ab5:dae9:b078::ff', - 'extra.ipv6_link_local': 'fe80::3ac4:e8ff:fe03:b3e2/64', - 'extra.mac_address': '38:c4:e8:03:b3:e2', - 'extra.machine_name': '6J0E022PAG35073', - 'extra.manufacturer': 'General', - 'extra.method': 'client.notifyDevInfo', - 'extra.remote_video_input_channels': 0, - 'extra.response_size': 794, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.video_input_channels': 1, - 'extra.video_output_channels': 0, - 'feed.name': 'Accessible DVR DHCPDiscover', - 'protocol.application': 'dvrdhcpdiscover', - 'protocol.transport': 'udp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 37810, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, - {'__type': 'Event', - 'classification.identifier': 'open-dvr-dhcpdiscover', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.alarm_input_channels': 0, - 'extra.alarm_output_channels': 0, - 'extra.amplification': 761.0, - 'extra.device_model': 'HCVR', - 'extra.device_serial': '2K0488CPAGS0ND6', - 'extra.device_type': 'HCVR', - 'extra.device_vendor': 'Private', - 'extra.device_version': '3.210.1.4', - 'extra.http_port': 80, - 'extra.internal_port': 37777, - 'extra.ipv4_address': '192.168.0.2', - 'extra.ipv4_dhcp_enable': False, - 'extra.ipv4_gateway': '192.168.0.240', - 'extra.ipv4_subnet_mask': '255.255.255.0', - 'extra.ipv6_address': 'fd09:4ab5:dae9:b078::2', - 'extra.ipv6_gateway': 'fd09:4ab5:dae9:b078::ff', - 'extra.ipv6_link_local': 'fe80::3eef:8cff:fe18:a507/64', - 'extra.mac_address': '3c:ef:8c:18:a5:07', - 'extra.machine_name': 'HCVR', - 'extra.manufacturer': 'Private', - 'extra.method': 'client.notifyDevInfo', - 'extra.remote_video_input_channels': 9, - 'extra.response_size': 761, - 'extra.video_input_channels': 3, - 'extra.video_output_channels': 0, - 'feed.name': 'Accessible DVR DHCPDiscover', - 'protocol.application': 'dvrdhcpdiscover', - 'protocol.transport': 'udp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 37810, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-dvr-dhcpdiscover', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.alarm_input_channels': 0, - 'extra.alarm_output_channels': 0, - 'extra.amplification': 711.0, - 'extra.device_model': 'BCS-XVR0401-IV', - 'extra.device_serial': '5L034FAPAZA0E30', - 'extra.device_type': 'HCVR', - 'extra.device_vendor': 'General', - 'extra.device_version': '4.000.0000002.11', - 'extra.http_port': 80, - 'extra.internal_port': 37777, - 'extra.ipv4_address': '192.168.0.3', - 'extra.ipv4_dhcp_enable': False, - 'extra.ipv4_gateway': '192.168.0.240', - 'extra.ipv4_subnet_mask': '255.255.255.0', - 'extra.ipv6_address': 'fd09:4ab5:dae9:b078::3', - 'extra.ipv6_gateway': 'fd09:4ab5:dae9:b078::ff', - 'extra.ipv6_link_local': 'fe80::3ac4:e8ff:fe02:74da/64', - 'extra.mac_address': '38:c4:e8:02:74:da', - 'extra.machine_name': 'XVR', - 'extra.manufacturer': 'General', - 'extra.method': 'client.notifyDevInfo', - 'extra.remote_video_input_channels': 0, - 'extra.response_size': 711, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.video_input_channels': 4, - 'extra.video_output_channels': 0, - 'feed.name': 'Accessible DVR DHCPDiscover', - 'protocol.application': 'dvrdhcpdiscover', - 'protocol.transport': 'udp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 37810, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_elasticsearch.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_elasticsearch.py deleted file mode 100644 index 4e12a1b076..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_elasticsearch.py +++ /dev/null @@ -1,126 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_elasticsearch.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open Elasticsearch', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_elasticsearch-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-elasticsearch', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.build_hash': '90f439ff60a3c0f497f91663701e64ccd01edbb4', - 'extra.build_snapshot': False, - 'extra.build_timestamp': '2016-07-27T10:36:52Z', - 'extra.cluster_name': 'elasticsearch', - 'extra.lucene_version': '5.5.0', - 'extra.name': 'Red Skull', - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'elasticsearch', - 'extra.tagline': 'You Know, for Search', - 'extra.version': '2.3.5', - 'feed.name': 'Open Elasticsearch', - 'protocol.application': 'elasticsearch', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 9200, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, - {'__type': 'Event', - 'classification.identifier': 'open-elasticsearch', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.build_hash': 'bee86328705acaa9a6daede7140defd4d9ec56bd', - 'extra.build_snapshot': False, - 'extra.cluster_name': 'docker-cluster', - 'extra.lucene_version': '8.11.1', - 'extra.name': 'allinonepod', - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'elasticsearch', - 'extra.tagline': 'You Know, for Search', - 'extra.version': '7.17.0', - 'feed.name': 'Open Elasticsearch', - 'protocol.application': 'elasticsearch', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 9200, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-elasticsearch', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.build_hash': '79d65f6e357953a5b3cbcc5e2c7c21073d89aa29', - 'extra.build_snapshot': False, - 'extra.cluster_name': 'docker-cluster', - 'extra.lucene_version': '8.9.0', - 'extra.name': 'f547c2952610', - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'elasticsearch', - 'extra.tagline': 'You Know, for Search', - 'extra.version': '7.15.0', - 'feed.name': 'Open Elasticsearch', - 'protocol.application': 'elasticsearch', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 9200, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_exchange.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_exchange.py deleted file mode 100644 index aeeffa3c29..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_exchange.py +++ /dev/null @@ -1,149 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), "testdata/scan_exchange.csv")) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = { - "feed.name": "Shadowserver CVE-2021-26855", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2020-08-19T00:00:00+00:00", - "extra.file_name": "2020-08-19-scan_exchange.csv", -} - -EVENTS = [ - { - "__type": "Event", - "feed.name": "Shadowserver CVE-2021-26855", - "time.source": "2021-05-14T00:11:30+00:00", - "source.ip": "12.237.1.2", - "source.port": 443, - "source.asn": 7018, - "source.geolocation.cc": "US", - "source.geolocation.region": "CALIFORNIA", - "source.geolocation.city": "TURLOCK", - "source.reverse_dns": 'afs-exch-cas2.xxx.com', - "extra.version": '15.2.721', - "extra.source.sector": "Communications, Service Provider, and Hosting Service", - "extra.source.naics": 517311, - "classification.identifier": "vulnerable-exchange-server", - "extra.tag": "exchange;cve-2021-26855", - "classification.taxonomy": "vulnerable", - "classification.type": "infected-system", - "extra.servername": "AFS-EXCH2019", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[1]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Shadowserver CVE-2021-26855", - "time.source": "2021-05-14T00:11:37+00:00", - "source.ip": "98.153.3.4", - "source.port": 443, - "source.asn": 20001, - "source.geolocation.cc": "US", - "source.geolocation.region": "CALIFORNIA", - "source.geolocation.city": "LOS ANGELES", - "source.reverse_dns": 'rrcs-98-153-x-x.west.biz.rr.com', - "extra.version": '15.0.847', - "extra.source.sector": "Communications, Service Provider, and Hosting Service", - "extra.source.naics": 517311, - "extra.tag": "exchange;webshell", - "classification.taxonomy": "intrusions", - "classification.type": "system-compromise", - "classification.identifier": "exchange-server-webshell", - "extra.servername": "SSAMAIL", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[2]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Shadowserver CVE-2021-26855", - "time.source": "2021-05-14T00:11:38+00:00", - "source.ip": "206.210.5.6", - "source.port": 443, - "source.asn": 17054, - "source.geolocation.cc": "US", - "source.geolocation.region": "PENNSYLVANIA", - "source.geolocation.city": "PITTSBURGH", - "source.reverse_dns": 'webmail.xxx.com', - "extra.source.naics": 518210, - "extra.version": '15.0.1178', - "extra.servername": "OMNYXEXCH02", - "extra.tag": "exchange;webshell", - "classification.taxonomy": "intrusions", - "classification.type": "system-compromise", - "classification.identifier": "exchange-server-webshell", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[3]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Shadowserver CVE-2021-26855", - "time.source": "2021-05-14T00:11:38+00:00", - "source.ip": "12.33.7.8", - "source.port": 443, - "source.asn": 7018, - "source.geolocation.cc": "US", - "source.geolocation.region": "ARKANSAS", - "source.geolocation.city": "LITTLE ROCK", - "source.reverse_dns": 'mail.xxx.org', - "extra.version": '15.1.2176', - "extra.source.sector": "Communications, Service Provider, and Hosting Service", - "extra.source.naics": 921120, - "extra.servername": "MHASVR02", - "classification.identifier": "vulnerable-exchange-server", - "extra.tag": "exchange;cve-2021-26855", - "classification.taxonomy": "vulnerable", - "classification.type": "infected-system", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[4]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Shadowserver CVE-2021-26855", - "time.source": "2021-05-14T00:11:38+00:00", - "source.ip": "41.204.9.10", - "source.port": 443, - "source.asn": 21042, - "source.geolocation.cc": 'MG', - "source.geolocation.city": 'ANTANANARIVO', - "source.geolocation.region": 'ANTANANARIVO', - "source.reverse_dns": 'mail.xxx.mg', - "extra.servername": "SABMHQE0232", - "classification.identifier": "vulnerable-exchange-server", - "extra.tag": "exchange;cve-2021-26855", - "classification.taxonomy": "vulnerable", - "classification.type": "infected-system", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[5]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, -] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == "__main__": - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ftp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ftp.py deleted file mode 100644 index 33daefd75e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ftp.py +++ /dev/null @@ -1,120 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ftp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible FTP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2019-03-25T00:00:00+00:00", - "extra.file_name": "2019-03-25-scan_ftp-test-test.csv", - } -EVENTS = [{ - '__type': 'Event', - 'feed.name': 'Accessible FTP', - 'time.observation': '2019-03-25T00:00:00+00:00', - 'time.source': '2019-03-06T06:37:00+00:00', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-ftp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.ip': '61.126.3.70', - 'source.port': 21, - 'protocol.transport': 'tcp', - 'protocol.application': 'ftp', - 'source.reverse_dns': 'arcus-net.co.jp', - 'extra.tag': 'ftp', - 'source.asn': 4713, - 'source.geolocation.cc': 'JP', - 'source.geolocation.region': 'TOKYO', - 'source.geolocation.city': 'TOKYO', - 'extra.naics': 517311, - 'extra.sic': 737401, - 'extra.banner': '220 FTP Server ready.|', - 'extra.handshake': 'TLSv1.2', - 'extra.cipher_suite': 'TLS_RSA_WITH_AES_128_CBC_SHA', - 'extra.cert_length': 2048, - 'extra.subject_common_name': '*.bizmw.com', - 'extra.issuer_common_name': 'GlobalSign Organization Validation CA - SHA256 - G2', - 'extra.cert_issue_date': 'Jan 14 08:04:50 2015 GMT', - 'extra.cert_expiration_date': 'Jan 14 08:04:50 2020 GMT', - 'extra.sha1_fingerprint': 'D9:98:3F:2E:F9:D1:BE:9A:10:1E:DE:51:2C:C1:DF:01:18:0A:20:65', - 'extra.cert_serial_number': '1121DC7421AB7924C3B1D396AEA3707E9E29', - 'extra.ssl_version': 2, - 'extra.signature_algorithm': 'sha256WithRSAEncryption', - 'extra.key_algorithm': 'rsaEncryption', - 'extra.subject_organization_name': 'NTT Communications Corporation', - 'extra.subject_country': 'JP', - 'extra.subject_state_or_province_name': 'Tokyo', - 'extra.subject_locality_name': 'Minato-ku', - 'extra.issuer_organization_name': 'GlobalSign nv-sa', - 'extra.issuer_country': 'BE', - 'extra.sha256_fingerprint': '27:4A:8A:3A:A7:DF:82:D0:43:03:0E:6F:48:30:30:C9:24:77:11:1A:08:EF:F7:B9:74:0C:CE:40:87:03:D2:51', - 'extra.sha512_fingerprint': 'E5:93:8B:72:84:0F:35:52:8E:7A:6C:E3:EF:36:90:4C:F2:86:A7:4D:B2:DD:C0:C6:23:83:18:EF:DD:86:34:92:91:57:22:29:75:45:71:8B:3A:CD:F1:27:A9:CA:5F:70:5E:AC:15:A5:E6:63:FD:6F:BB:C5:E2:45:99:73:E9:E6', - 'extra.md5_fingerprint': 'D1:A7:BC:96:78:1D:16:D0:24:A8:62:7C:3A:95:5A:4A', - 'extra.cert_valid': False, - 'extra.self_signed': False, - 'extra.cert_expired': False, - 'extra.validation_level': 'OV', - 'extra.auth_tls_response': '234 AUTH TLS successful', - }, - { - '__type': 'Event', - 'feed.name': 'Accessible FTP', - 'time.observation': '2019-03-25T00:00:00+00:00', - 'time.source': '2019-03-06T06:37:00+00:00', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-ftp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.ip': '62.48.156.65', - 'source.port': 21, - 'protocol.transport': 'tcp', - 'protocol.application': 'ftp', - 'source.reverse_dns': 'dial-62-48-156-65.ptprime.net', - 'extra.tag': 'ftp', - 'source.asn': 15525, - 'source.geolocation.cc': 'PT', - 'source.geolocation.region': 'LISBOA', - 'source.geolocation.city': 'FRIELAS', - 'extra.banner': '220-================================================================| PT Empresas| Acesso Reservado| Acesso nao autorizado punido por lei: 109/91; 67/98| ----------------------------------------------------------------| HENNES & MAURITZ LDA - 149093| SITE: PT303 - Cascais Shopping| MORADA: | NIR: EWS1822940| ================================================================|220 FTP server ready, 1 active clients of 4 simultaneous clients allowed.|', - 'extra.auth_tls_response': '500 Syntax error, command unrecognized.', - 'extra.auth_ssl_response': '500 Syntax error, command unrecognized.' - } - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_hadoop.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_hadoop.py deleted file mode 100644 index 0b5794cb7b..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_hadoop.py +++ /dev/null @@ -1,94 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Sebastian Wagner -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_hadoop.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "ShadowServer Accessible-Hadoop", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_hadoop-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'ShadowServer Accessible-Hadoop', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-hadoop', - 'extra.version': '2.7.3, rbaa91f7c6bc9cb92be5982de4719c1c8af91ccff', - 'extra.server_type': 'namenode', - 'extra.clusterid': 'CID-64471a53-60cb-4302-9832-92f321f111fe', - 'extra.total_disk': 41567956992, - 'extra.used_disk': 53248, - 'extra.free_disk': 25160089600, - 'extra.livenodes': 'edmonton:50010', - 'protocol.application': 'hadoop', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 15296, - 'source.geolocation.cc': 'CA', - 'source.geolocation.city': 'CALGARY', - 'source.geolocation.region': 'ALBERTA', - 'source.ip': '199.116.235.200', - 'source.port': 50070, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2017-09-13T02:06:05+00:00'}, - {'__type': 'Event', - 'feed.name': 'ShadowServer Accessible-Hadoop', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-hadoop', - 'extra.version': '2.7.1.2.4.0.0-169', - 'extra.naics': 334111, - 'extra.sic': 357101, - 'extra.server_type': 'datanode', - 'extra.clusterid': 'CID-771bae52-9e4f-4ec4-bc1a-c867585751f0', - 'extra.namenodeaddress': 'sandbox.hortonworks.com', - 'extra.volumeinfo': '/hadoop/hdfs/data/current', - 'protocol.application': 'hadoop', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn': 8075, - 'source.geolocation.cc': 'US', - 'source.geolocation.city': 'DES MOINES', - 'source.geolocation.region': 'IOWA', - 'source.ip': '104.43.235.92', - 'source.port': 50075, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2017-09-13T02:07:48+00:00'}, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_http.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_http.py deleted file mode 100644 index 793a95f221..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_http.py +++ /dev/null @@ -1,100 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_http.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible HTTP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-03-25-scan_http-test-test.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Accessible HTTP', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'classification.identifier': 'accessible-http', - 'extra.source.naics': 518111, - 'extra.source.sic': 737401, - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 200, - 'extra.http_reason': 'OK', - 'extra.content_type': 'text/html', - 'extra.server': 'lighttpd', - 'extra.transfer_encoding': 'chunked', - 'extra.http_date': '2018-04-19T00:02:28+00:00', - 'extra.tag': 'http', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.reverse_dns': 'c-75-74-78-113.hsd1.fl.comcast.net', - 'source.asn': 7922, - 'source.geolocation.cc': 'US', - 'source.geolocation.city': 'MIAMI', - 'source.geolocation.region': 'FLORIDA', - 'source.ip': '75.74.78.113', - 'source.port': 8080, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2018-04-19T00:02:26+00:00'}, - {'__type': 'Event', - 'feed.name': 'Accessible HTTP', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'classification.identifier': 'accessible-http', - 'extra.source.naics': 518210, - 'extra.source.sic': 737415, - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 200, - 'extra.http_reason': 'OK', - 'extra.content_type': 'text/html', - 'extra.content_length': 17729, - 'extra.http_date': '2018-04-19T02:02:28+00:00', - 'extra.tag': 'http', - 'protocol.transport': 'tcp', - 'protocol.application': 'http', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.reverse_dns': 'sto95-3-88-162-174-130.fbx.proxad.net', - 'source.asn': 12322, - 'source.geolocation.cc': 'FR', - 'source.geolocation.city': 'SAINT-OUEN-LAUMONE', - 'source.ip': '88.162.174.130', - 'source.port': 8080, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2018-04-19T00:02:26+00:00'}, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_http_proxy.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_http_proxy.py deleted file mode 100644 index dc5e94e5ec..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_http_proxy.py +++ /dev/null @@ -1,118 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_http_proxy.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open HTTP Proxy', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_http_proxy-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-http-proxy', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 200, - 'extra.http_date': '2010-02-10T00:00:00+00:00', - 'extra.http_reason': 'Connection established', - 'extra.tag': 'http-connect-proxy', - 'feed.name': 'Open HTTP Proxy', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 3128, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-http-proxy', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 200, - 'extra.http_date': '2010-02-10T00:00:01+00:00', - 'extra.http_reason': 'Connection established', - 'extra.tag': 'http-connect-proxy', - 'extra.via': 'HTTP/1.1 s_proxy_den1', - 'feed.name': 'Open HTTP Proxy', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 3128, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-http-proxy', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 200, - 'extra.http_date': '2010-02-10T00:00:02+00:00', - 'extra.http_reason': 'Connection established', - 'extra.tag': 'http-connect-proxy', - 'extra.via': 'HTTP/1.1 s_proxy_yvr', - 'feed.name': 'Open HTTP Proxy', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 3128, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_http_vulnerable.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_http_vulnerable.py deleted file mode 100644 index d15232eaf7..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_http_vulnerable.py +++ /dev/null @@ -1,125 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Mikk Margus Möll -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_http_vulnerable.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Vulnerable HTTP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2021-08-01T09:00:00+00:00", - "extra.file_name": "2021-08-01-scan_http_vulnerable-test-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'accessible-http', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.content_length': 149, - 'extra.content_type': 'text/html; charset=utf-8', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 401, - 'extra.http_date': '2010-02-10T00:00:00+00:00', - 'extra.http_reason': 'Unauthorized', - 'extra.server': 'TwistedWeb/19.7.0', - 'extra.set_cookie': 'TWISTED_SESSION=5473ad3faa3de66685fb3a53bffb390b4fcec2039893009a06caf38e1bec8aa8', - 'extra.tag': 'basic-auth,http', - 'extra.www_authenticate': 'Basic realm=\\\\"OpenWebif\\"\\""', - 'feed.name': 'Vulnerable HTTP', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 8080, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-http', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.content_length': 149, - 'extra.content_type': 'text/html; charset=utf-8', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 401, - 'extra.http_date': '2010-02-10T00:00:01+00:00', - 'extra.http_reason': 'Unauthorized', - 'extra.server': 'TwistedWeb/19.7.0', - 'extra.set_cookie': 'TWISTED_SESSION=d2460d37b7fdbdd6c27dd74423ead5704e553d4f2c230672313edc5602059e33', - 'extra.tag': 'basic-auth,http', - 'extra.www_authenticate': 'Basic realm=\\\\"OpenWebif\\"\\""', - 'feed.name': 'Vulnerable HTTP', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 80, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-http', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.detail': 'repositoryformatversion = 0;filemode = false;bare = ' - 'false;logallrefupdates = true;symlinks = false;ignorecase = ' - 'true', - 'extra.http_date': '2010-02-10T00:00:02+00:00', - 'extra.tag': 'git-config-file', - 'feed.name': 'Vulnerable HTTP', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 443, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ics.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ics.py deleted file mode 100644 index f673f40c80..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ics.py +++ /dev/null @@ -1,125 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ics.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Acessible ICS', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_ics-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ics', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.device_model' : 'device_model', - 'extra.device_type' : 'device_type', - 'extra.device_vendor' : 'Vendor 1', - 'extra.device_version' : 'device_version', - 'extra.raw_response' : 'dGVzdDE=', - 'extra.response_size' : 5, - 'extra.source.sector' : 'Sector', - 'feed.name' : 'Acessible ICS', - 'protocol.application' : 'modbus', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'CITY', - 'source.geolocation.region' : 'REGION', - 'source.ip' : '192.168.0.1', - 'source.port' : 502, - 'source.reverse_dns' : 'host1.example.net', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-03-02T00:34:22+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ics', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.device_model' : 'device_model', - 'extra.device_type' : 'device_type', - 'extra.device_vendor' : 'Vendor 2', - 'extra.device_version' : 'device_version', - 'extra.raw_response' : 'dGVzdDI=', - 'extra.response_size' : 5, - 'extra.source.sector' : 'Sector', - 'feed.name' : 'Acessible ICS', - 'protocol.application' : 'modbus', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64513, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'CITY', - 'source.geolocation.region' : 'REGION', - 'source.ip' : '192.168.0.2', - 'source.port' : 502, - 'source.reverse_dns' : 'host2.example.net', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-03-02T00:34:22+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ics', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.device_model' : 'device_model', - 'extra.device_type' : 'device_type', - 'extra.device_vendor' : 'Vendor 3', - 'extra.device_version' : 'device_version', - 'extra.raw_response' : 'dGVzdDM=', - 'extra.response_size' : 5, - 'extra.source.sector' : 'Sector', - 'feed.name' : 'Acessible ICS', - 'protocol.application' : 'modbus', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64514, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'CITY', - 'source.geolocation.region' : 'REGION', - 'source.ip' : '192.168.0.3', - 'source.port' : 502, - 'source.reverse_dns' : 'host3.example.net', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-03-02T00:34:22+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ipmi.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ipmi.py deleted file mode 100644 index 08a9082af9..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ipmi.py +++ /dev/null @@ -1,106 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_ipmi.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open IPMI', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ipmi-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Open IPMI', - "classification.identifier": "open-ipmi", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.anon_login": False, - "extra.defaultkg": "-", - "extra.ipmi_version": "1.5", - "extra.md2_auth": False, - "extra.md5_auth": True, - "extra.none_auth": True, - "extra.nulluser": True, - "extra.oem_auth": False, - "extra.passkey_auth": True, - "extra.permessage_auth": True, - "extra.tag": "ipmi", - "extra.userlevel_auth": True, - "extra.usernames": False, - "protocol.application": "ipmi", - "protocol.transport": "udp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 2914, - "source.geolocation.cc": "DE", - "source.geolocation.city": "BERLIN", - "source.geolocation.region": "BERLIN", - "source.ip": "198.51.100.4", - "source.port": 623, - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2016-07-24T00:09:42+00:00" - }, - {'__type': 'Event', - 'feed.name': 'Open IPMI', - "classification.identifier": "open-ipmi", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.anon_login": False, - "extra.defaultkg": "default", - "extra.ipmi_version": "2.0", - "extra.md2_auth": False, - "extra.md5_auth": False, - "extra.none_auth": False, - "extra.nulluser": False, - "extra.oem_auth": False, - "extra.passkey_auth": False, - "extra.permessage_auth": False, - "extra.tag": "ipmi", - "extra.userlevel_auth": True, - "extra.usernames": True, - "protocol.application": "ipmi", - "protocol.transport": "udp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 28753, - "source.geolocation.cc": "DE", - "source.geolocation.city": "FRANKFURT AM MAIN", - "source.geolocation.region": "HESSEN", - "source.ip": "198.51.100.182", - "source.port": 623, - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2016-07-24T00:09:43+00:00" - }, - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ipp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ipp.py deleted file mode 100644 index 9adc8485e0..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ipp.py +++ /dev/null @@ -1,79 +0,0 @@ -# SPDX-FileCopyrightText: 2020 Thomas Hungenberg -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_ipp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open-IPP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2020-06-09T00:00:00+00:00", - "extra.file_name": "2020-06-08-scan_ipp-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Open-IPP', - "classification.identifier": "open-ipp", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517311, - "extra.tag": "ipp", - "extra.ipp_version": "IPP/2.1", - "extra.cups_version": "CUPS/2.0", - "extra.printer_uris": "ipp://123.45.67.89:631/ipp/print", - "extra.printer_name": "NPI3F0D22", - "extra.printer_info": "HP Color LaserJet MFP M277dw", - "extra.printer_more_info": "http://123.45.67.89:631/hp/device/info_config_AirPrint.html?tab=Networking&menu=AirPrintStatus", - "extra.printer_make_and_model": "HP Color LaserJet MFP M277dw", - "extra.printer_firmware_name": "20191203", - "extra.printer_firmware_string_version": "20191203", - "extra.printer_firmware_version": "20191203", - "extra.printer_organization": "org", - "extra.printer_organization_unit": "unit", - "extra.printer_uuid": "urn:uuid:456e4238-4a44-4643-4c42-10e1813f0a18", - "extra.printer_wifi_ssid": "wifissid", - "protocol.application": "ipp", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 12345, - "source.geolocation.cc": "AA", - "source.geolocation.city": "CITY", - "source.geolocation.region": "REGION", - "source.ip": "123.45.67.89", - "source.port": 631, - 'source.reverse_dns': 'some.host.com', - "time.observation": "2020-06-09T00:00:00+00:00", - "time.source": "2020-06-08T11:30:14+00:00" - }, - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_isakmp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_isakmp.py deleted file mode 100644 index 3192f508f8..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_isakmp.py +++ /dev/null @@ -1,105 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_isakmp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Vulnerable ISAKMP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_isakmp-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Vulnerable ISAKMP', - "classification.identifier": "open-ike", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.domain_of_interpretation": '00', - "extra.exchange_type": '05', - "extra.flags": '00', - "extra.initiator_spi": "3e35c70729dfedef", - "extra.message_id": "00000000", - "extra.naics": 517311, - "extra.next_payload": '11', - "extra.next_payload2": '00', - "extra.notify_message_type": '14', - "extra.responder_spi": "253acab7cbfda607", - "extra.spi_size": 0, - "extra.tag": "isakmp-vulnerable", - "protocol.application": "ipsec", - "protocol.transport": "udp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 5678, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.42", - "source.port": 500, - "source.reverse_dns": "example.local", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T00:17:25+00:00" - }, - {'__type': 'Event', - 'feed.name': 'Vulnerable ISAKMP', - "classification.identifier": "open-ike", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.domain_of_interpretation": '00', - "extra.exchange_type": '05', - "extra.flags": '00', - "extra.initiator_spi": "3e35c70729dfedef", - "extra.message_id": "00000000", - "extra.next_payload": '11', - "extra.next_payload2": '00', - "extra.notify_message_type": '14', - "extra.responder_spi": "b274460e7adc1bf0", - "extra.spi_size": 0, - "extra.tag": "isakmp-vulnerable", - "protocol.application": "ipsec", - "protocol.transport": "udp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 20255, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.67", - "source.port": 500, - "source.reverse_dns": "example.local", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T00:17:28+00:00" - }, - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_kubernetes.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_kubernetes.py deleted file mode 100644 index 2bac336a79..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_kubernetes.py +++ /dev/null @@ -1,214 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_kubernetes.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible Kubernetes API Server', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_kubernetes-test.csv", - } -EVENTS = [ - { - '__type' : 'Event', - 'classification.identifier' : 'open-kubernetes', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.browser_error' : 'x509: failed to load system roots and no roots provided', - 'extra.browser_trusted' : False, - 'extra.build_date' : '2021-11-17T13:00:29Z', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.compiler' : 'gc', - 'extra.content_type' : 'application/json', - 'extra.date' : 'Tue, 10 May 2022 14:24:13 GMT', - 'extra.git_commit' : '2444b3347a2c45eb965b182fb836e1f51dc61b70', - 'extra.git_tree_state' : 'clean', - 'extra.git_version' : 'v1.20.13', - 'extra.go_version' : 'go1.15.15', - 'extra.handshake' : 'TLSv1.2', - 'extra.http' : 'HTTP/1.1', - 'extra.http_code' : 200, - 'extra.http_reason' : 'OK', - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.major' : '1', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.minor' : '20', - 'extra.platform' : 'linux/amd64', - 'extra.self_signed' : False, - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.tag' : 'kubernetes', - 'feed.name' : 'Accessible Kubernetes API Server', - 'protocol.application' : 'kubernetes', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 6443, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-kubernetes', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.browser_error' : 'x509: failed to load system roots and no roots provided', - 'extra.browser_trusted' : False, - 'extra.build_date' : '2022-02-25T06:26:46Z', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.compiler' : 'gc', - 'extra.content_type' : 'application/json', - 'extra.date' : 'Tue, 10 May 2022 14:24:12 GMT', - 'extra.git_commit' : '6f5a5295923a614a4202a7ad274b38b69f9ca8c0', - 'extra.git_tree_state' : 'clean', - 'extra.git_version' : 'v1.23.3+e419edf', - 'extra.go_version' : 'go1.17.5', - 'extra.handshake' : 'TLSv1.2', - 'extra.http' : 'HTTP/1.1', - 'extra.http_code' : 200, - 'extra.http_reason' : 'OK', - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.major' : '1', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.minor' : '23', - 'extra.platform' : 'linux/amd64', - 'extra.self_signed' : False, - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.source.sector' : 'Retail Trade', - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.tag' : 'kubernetes', - 'feed.name' : 'Accessible Kubernetes API Server', - 'protocol.application' : 'kubernetes', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 6443, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-kubernetes', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.browser_error' : 'x509: failed to load system roots and no roots provided', - 'extra.browser_trusted' : False, - 'extra.build_date' : '2020-05-08T07:29:59Z', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.compiler' : 'gc', - 'extra.content_type' : 'application/json', - 'extra.date' : 'Tue, 10 May 2022 14:24:12 GMT', - 'extra.git_commit' : '4f7ea78', - 'extra.git_version' : 'v1.16.9-aliyun.1', - 'extra.go_version' : 'go1.13.9', - 'extra.handshake' : 'TLSv1.2', - 'extra.http' : 'HTTP/1.1', - 'extra.http_code' : 200, - 'extra.http_reason' : 'OK', - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.major' : '1', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.minor' : '16+', - 'extra.platform' : 'linux/amd64', - 'extra.self_signed' : False, - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.tag' : 'kubernetes', - 'feed.name' : 'Accessible Kubernetes API Server', - 'protocol.application' : 'kubernetes', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 6443, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_tcp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_tcp.py deleted file mode 100644 index b6abf6eba9..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_tcp.py +++ /dev/null @@ -1,154 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_ldap_tcp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open LDAP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ldap_tcp-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.configuration_naming_context': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.default_naming_context': 'DC=ad,DC=example,DC=com', - 'extra.domain_controller_functionality': 7, - 'extra.ds_service_name': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.forest_functionality': 2, - 'extra.ldap_service_name': 'node01.example.com', - 'extra.naming_contexts': 'DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.root_domain_naming_context': 'DC=example,DC=com', - 'extra.schema_naming_context': 'CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.server_name': 'CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com', - 'extra.size': 0, - 'extra.subschema_subentry': 'CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.supported_control': '1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|', - 'extra.supported_ldap_policies': 'MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent', - 'extra.supported_ldap_version': '3|2', - 'extra.supported_sasl_mechanisms': 'GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5', - 'extra.tag': 'ldap-tcp', - 'feed.name': 'Open LDAP', - 'protocol.application': 'ldap', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.local_hostname': 'node01.example.com', - 'source.port': 389, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, - {'__type': 'Event', - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.configuration_naming_context': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.current_time': '20220821124435.0Z', - 'extra.default_naming_context': 'DC=ad,DC=example,DC=com', - 'extra.domain_controller_functionality': 7, - 'extra.domain_functionality': 7, - 'extra.ds_service_name': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.forest_functionality': 7, - 'extra.highest_committed_usn': 25029662, - 'extra.is_global_catalog_ready': True, - 'extra.is_synchronized': True, - 'extra.ldap_service_name': 'node02.example.com', - 'extra.naming_contexts': 'DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.root_domain_naming_context': 'DC=example,DC=com', - 'extra.schema_naming_context': 'CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.server_name': 'CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com', - 'extra.size': 0, - 'extra.subschema_subentry': 'CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.supported_capabilities': '1.2.840.113556.1.4.800|1.2.840.113556.1.4.1670|1.2.840.113556.1.4.1791|1.2.840.113556.1.4.1935|1.2.840.113556.1.4.2080|1.2.840.113556.1.4.2237', - 'extra.supported_control': '1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|1.2.840.113556.1.4.2206|1.2.840.113556.1.4.2211|1.2.840.113556.1.4.2239|1.2.840.113556.1.4.2255|1.2.840.113556.1.4.2256|1.2.840.113556.1.4.2309', - 'extra.supported_ldap_policies': 'MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent', - 'extra.supported_ldap_version': '3|2', - 'extra.supported_sasl_mechanisms': 'GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5', - 'extra.tag': 'ldap-tcp', - 'feed.name': 'Open LDAP', - 'protocol.application': 'ldap', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.local_hostname': 'node02.example.com', - 'source.port': 389, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.configuration_naming_context': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.current_time': '20220821124539.0Z', - 'extra.default_naming_context': 'DC=ad,DC=example,DC=com', - 'extra.ds_service_name': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.ldap_service_name': 'node03.example.com', - 'extra.naming_contexts': 'DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.root_domain_naming_context': 'DC=example,DC=com', - 'extra.schema_naming_context': 'CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.server_name': 'CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com', - 'extra.size': 0, - 'extra.subschema_subentry': 'CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.supported_control': '1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|', - 'extra.tag': 'ldap-tcp', - 'feed.name': 'Open LDAP', - 'protocol.application': 'ldap', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.local_hostname': 'node03.example.com', - 'source.port': 389, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_udp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_udp.py deleted file mode 100644 index aa4deefb87..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_udp.py +++ /dev/null @@ -1,162 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_ldap_udp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open LDAP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ldap_udp-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 58.42, - 'extra.configuration_naming_context': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.current_time': '20220821044533.0Z', - 'extra.default_naming_context': 'DC=ad,DC=example,DC=com', - 'extra.domain_controller_functionality': 7, - 'extra.domain_functionality': 7, - 'extra.ds_service_name': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.forest_functionality': 7, - 'extra.highest_committed_usn': 222537, - 'extra.is_global_catalog_ready': True, - 'extra.is_synchronized': True, - 'extra.ldap_service_name': 'node01.example.com', - 'extra.naming_contexts': 'DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.root_domain_naming_context': 'DC=example,DC=com', - 'extra.schema_naming_context': 'CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.server_name': 'CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com', - 'extra.size': 3038, - 'extra.subschema_subentry': 'CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.supported_capabilities': '1.2.840.113556.1.4.800|1.2.840.113556.1.4.1670|1.2.840.113556.1.4.1791|1.2.840.113556.1.4.1935|1.2.840.113556.1.4.2080|1.2.840.113556.1.4.2237', - 'extra.supported_control': '1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|1.2.840.113556.1.4.2206|1.2.840.113556.1.4.2211|1.2.840.113556.1.4.2239|1.2.840.113556.1.4.2255|1.2.840.113556.1.4.2256|1.2.840.113556.1.4.2309|1.2.840.113556.1.4.2330|1.2.840.113556.1.4.2354', - 'extra.supported_ldap_policies': 'MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent', - 'extra.supported_ldap_version': '3|2', - 'extra.supported_sasl_mechanisms': 'GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5', - 'extra.tag': 'ldap-udp', - 'feed.name': 'Open LDAP', - 'protocol.application': 'ldap', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.local_hostname': 'node01.example.com', - 'source.port': 389, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 58.88, - 'extra.configuration_naming_context': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.current_time': '20220821044948.0Z', - 'extra.default_naming_context': 'DC=ad,DC=example,DC=com', - 'extra.domain_controller_functionality': 7, - 'extra.domain_functionality': 7, - 'extra.ds_service_name': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.forest_functionality': 7, - 'extra.highest_committed_usn': 1478714, - 'extra.is_global_catalog_ready': True, - 'extra.is_synchronized': True, - 'extra.ldap_service_name': 'node02.example.com', - 'extra.naming_contexts': 'DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.root_domain_naming_context': 'DC=example,DC=com', - 'extra.schema_naming_context': 'CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.server_name': 'CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com', - 'extra.size': 3062, - 'extra.subschema_subentry': 'CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.supported_capabilities': '1.2.840.113556.1.4.800|1.2.840.113556.1.4.1670|1.2.840.113556.1.4.1791|1.2.840.113556.1.4.1935|1.2.840.113556.1.4.2080|1.2.840.113556.1.4.2237', - 'extra.supported_control': '1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|1.2.840.113556.1.4.2206|1.2.840.113556.1.4.2211|1.2.840.113556.1.4.2239|1.2.840.113556.1.4.2255|1.2.840.113556.1.4.2256|1.2.840.113556.1.4.2309|1.2.840.113556.1.4.2330|1.2.840.113556.1.4.2354', - 'extra.supported_ldap_policies': 'MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent', - 'extra.supported_ldap_version': '3|2', - 'extra.supported_sasl_mechanisms': 'GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5', - 'extra.tag': 'ldap-udp', - 'feed.name': 'Open LDAP', - 'protocol.application': 'ldap', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.local_hostname': 'node02.example.com', - 'source.port': 389, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 0.69, - 'extra.configuration_naming_context': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.default_naming_context': 'DC=ad,DC=example,DC=com', - 'extra.ds_service_name': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.ldap_service_name': 'node03.example.com', - 'extra.naming_contexts': 'DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.root_domain_naming_context': 'DC=example,DC=com', - 'extra.schema_naming_context': 'CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.server_name': 'CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com', - 'extra.size': 36, - 'extra.subschema_subentry': 'CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.tag': 'ldap-udp', - 'feed.name': 'Open LDAP', - 'protocol.application': 'ldap', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.local_hostname': 'node03.example.com', - 'source.port': 389, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_mdns.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_mdns.py deleted file mode 100644 index 9207aaf365..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_mdns.py +++ /dev/null @@ -1,127 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_mdns.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open mDNS', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_mdns-test-geo.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mdns', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.http_ipv4' : '192.168.0.1', - 'extra.http_ipv6' : 'fd09:4ab5:dae9:b078::1', - 'extra.services' : '_smb._tcp.local.; _device-info._tcp.local.; _http._tcp.local.; _dacp._tcp.local.;', - 'extra.tag' : 'mdns', - 'extra.workstation_ipv4' : '192.168.0.1', - 'extra.workstation_ipv6' : 'fd09:4ab5:dae9:b078::1', - 'feed.name' : 'Open mDNS', - 'protocol.application' : 'mdns', - 'protocol.transport' : 'udp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 5353, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mdns', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.http_ipv4' : '192.168.0.2', - 'extra.http_ipv6' : 'fd09:4ab5:dae9:b078::2', - 'extra.services' : '_home-assistant._tcp.local.;', - 'extra.tag' : 'mdns', - 'extra.workstation_ipv4' : '192.168.0.2', - 'extra.workstation_ipv6' : 'fd09:4ab5:dae9:b078::2', - 'feed.name' : 'Open mDNS', - 'protocol.application' : 'mdns', - 'protocol.transport' : 'udp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 5353, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mdns', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.http_info' : '\\\\\"vendor=Synology\\\"\\\" \\\"\\\"model=DS218+\\\"\\\" \\\"\\\"serial=17A0PCN482002\\\"\\\" \\\"\\\"version_major=6\\\"\\\" \\\"\\\"version_minor=2\\\"\\\" \\\"\\\"version_build=25556\\\"\\\" \\\"\\\"admin_port=5000\\\"\\\" \\\"\\\"secure_admin_port=5001\\\"\\\" \\\"\\\"mac_address=00:11:32:80:fd:b5\\\"\\\"\"', - 'extra.http_ipv4' : '192.168.0.3', - 'extra.http_ipv6' : 'fd09:4ab5:dae9:b078::3', - 'extra.http_name' : 'snmeijer.local.', - 'extra.http_port' : 5000, - 'extra.http_ptr' : 'snmeijer._http._tcp.local.', - 'extra.http_target' : 'snmeijer.local.', - 'extra.services' : '_webdav._tcp.local.; _adisk._tcp.local.; _smb._tcp.local.; _http._tcp.local.; _dacp._tcp.local.; _afpovertcp._tcp.local.; _device-info._tcp.local.;', - 'extra.tag' : 'mdns,iot', - 'extra.workstation_ipv4' : '192.168.0.3', - 'extra.workstation_ipv6' : 'fd09:4ab5:dae9:b078::3', - 'feed.name' : 'Open mDNS', - 'protocol.application' : 'mdns', - 'protocol.transport' : 'udp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 5353, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_memcached.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_memcached.py deleted file mode 100644 index b54fc0ea53..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_memcached.py +++ /dev/null @@ -1,130 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_memcached.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open Memcached', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_memcached-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-memcached', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 81.71, - 'extra.curr_connections': 243, - 'extra.pid': 1010, - 'extra.pointer_size': 64, - 'extra.response_size': 1144, - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'memcached', - 'extra.time': '2022-08-21 10:34:06', - 'extra.total_connections': 6106, - 'extra.uptime': 32908114, - 'extra.version': '1.4.15', - 'feed.name': 'Open Memcached', - 'protocol.application': 'memcached', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 50260, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-memcached', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 75.21, - 'extra.curr_connections': 9, - 'extra.pid': 5316, - 'extra.pointer_size': 64, - 'extra.response_size': 1053, - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'memcached', - 'extra.time': '2022-08-21 10:39:21', - 'extra.total_connections': 2962, - 'extra.uptime': 9618498, - 'extra.version': '1.4.13', - 'feed.name': 'Open Memcached', - 'protocol.application': 'memcached', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 11211, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, - {'__type': 'Event', - 'classification.identifier': 'open-memcached', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 31.57, - 'extra.curr_connections': 2, - 'extra.pid': 1460, - 'extra.pointer_size': 32, - 'extra.response_size': 442, - 'extra.tag': 'memcached', - 'extra.time': '2022-08-21 10:39:39', - 'extra.total_connections': 534, - 'extra.uptime': 1375159, - 'extra.version': '1.2.6', - 'feed.name': 'Open Memcached', - 'protocol.application': 'memcached', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 11211, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_mongodb.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_mongodb.py deleted file mode 100644 index 3ecf7b21f9..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_mongodb.py +++ /dev/null @@ -1,103 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_mongodb.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open MongoDB', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_mongodb-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Open MongoDB', - "classification.identifier": "open-mongodb", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.allocator": "tcmalloc", - "extra.bits": "64", - "extra.gitversion": "a2ddc68ba7c9cee17bfe69ed840383ec3506602b", - "extra.javascriptengine": "V8", - "extra.maxbsonobjectsize": "16777216", - "extra.ok": True, - "extra.sysinfo": "Linux ip-198-51-100-100 198.51.100.103-2.ec2.v1.2.fc8xen #1 SMP Fri Nov 20 17:48:28 EST 2009 x86_64 BOOST_LIB_VERSION=1_49", - "extra.tag": "mongodb", - "extra.version": "2.4.5", - "extra.visible_databases": "local | countly | admin", - "protocol.application": "mongodb", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 20773, - "source.geolocation.cc": "DE", - "source.geolocation.city": "WEEZE", - "source.geolocation.region": "NORDRHEIN-WESTFALEN", - "source.ip": "198.51.100.203", - "source.port": 27017, - "source.reverse_dns": "198-51-100-203.example.net", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2016-07-24T00:40:07+00:00" - }, - {'__type': 'Event', - 'feed.name': 'Open MongoDB', - "classification.identifier": "open-mongodb", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.allocator": "tcmalloc", - "extra.bits": "64", - "extra.gitversion": "d73c92b1c85703828b55c2916a5dd4ad46535f6a", - "extra.javascriptengine": "V8", - "extra.maxbsonobjectsize": "16777216", - "extra.ok": True, - "extra.sector": "Information Technology", - "extra.sysinfo": "Linux build5.ny.cbi.10gen.cc 2.6.32-431.3.1.el6.x86_64 #1 SMP Fri Jan 3 21:39:27 UTC 2014 x86_64 BOOST_LIB_VERSION=1_49", - "extra.tag": "mongodb", - "extra.version": "2.6.12", - "extra.visible_databases": "none visible", - "protocol.application": "mongodb", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 24940, - "source.geolocation.cc": "DE", - "source.geolocation.city": "GUNZENHAUSEN", - "source.geolocation.region": "BAYERN", - "source.ip": "198.51.100.42", - "source.port": 27017, - "source.reverse_dns": "198-51-100-208.example.net", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2016-07-24T00:40:07+00:00" - }, - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt.py deleted file mode 100644 index 45d19f9eea..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt.py +++ /dev/null @@ -1,89 +0,0 @@ -# SPDX-FileCopyrightText: 2020 Thomas Hungenberg -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_mqtt.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open-MQTT', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2020-03-15T00:00:00+00:00", - "extra.file_name": "2020-03-14-scan_mqtt-test-geo.csv", - } -EVENTS = [ - { - '__type' : 'Event', - 'classification.identifier' : 'open-mqtt', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.anonymous_access' : False, - 'extra.cert_expiration_date' : '2022-11-14 00:00:00', - 'extra.cert_issue_date' : '2020-08-12 00:00:00', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : '085699743A23114C9B6B8DC975A8AF42', - 'extra.cipher_suite' : 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', - 'extra.code' : 'Connection Refused, not authorized', - 'extra.hex_code' : '05', - 'extra.issuer_common_name' : 'Sectigo RSA Domain Validation Secure Server CA', - 'extra.issuer_country' : 'GB', - 'extra.issuer_locality_name' : 'Salford', - 'extra.issuer_organization_name' : 'Sectigo Limited', - 'extra.issuer_state_or_province_name' : 'Greater Manchester', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'DE:2C:98:30:27:2E:7D:C9:ED:A3:9D:AF:9E:CE:14:CC', - 'extra.raw_response' : '20020005', - 'extra.sha1_fingerprint' : '70:84:F1:6D:28:DA:B6:E6:27:60:13:8B:2C:93:52:B6:7B:4B:13:7B', - 'extra.sha256_fingerprint' : 'D2:D7:54:52:EB:86:4E:2D:34:4D:FC:CE:CD:CF:39:41:E1:06:5C:8B:B8:54:E6:0C:DF:FD:6E:E3:F1:B5:41:00', - 'extra.sha512_fingerprint' : '17:57:FB:88:9D:BE:A7:F0:29:A5:31:FC:79:DF:F7:8A:1C:D6:4A:DF:1B:4A:DC:BF:05:E7:E8:2F:79:9A:FA:FE:F7:E8:66:22:CB:B9:4C:72:F7:FB:6C:1D:59:8C:54:63:70:05:DE:7F:3C:2F:BA:B8:37:18:CE:29:6F:11:E8:AB', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.naics' : 454110, - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : '*.tracesafe.io', - 'extra.tag' : 'mqtt', - 'feed.name' : 'Open-MQTT', - 'protocol.application' : 'mqtt', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 12345, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'COLUMBUS', - 'source.geolocation.region' : 'OHIO', - 'source.ip' : '18.220.0.0', - 'source.port' : 8883, - 'source.reverse_dns' : '18-220-0-0.example.com', - 'time.observation' : '2020-03-15T00:00:00+00:00', - 'time.source' : '2022-02-07T12:56:53+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt_anon.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt_anon.py deleted file mode 100644 index 4618957240..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt_anon.py +++ /dev/null @@ -1,173 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_mqtt_anon.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open Anonymous MQTT', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_mqtt_anon-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mqtt-anon', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.cert_expiration_date' : '2030-05-06 08:07:05', - 'extra.cert_issue_date' : '2020-05-08 08:07:05', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : '02', - 'extra.cipher_suite' : 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', - 'extra.code' : 'Connection Accepted', - 'extra.hex_code' : '00', - 'extra.issuer_common_name' : 'RootCA', - 'extra.issuer_country' : 'CN', - 'extra.issuer_organization_name' : 'EMQ', - 'extra.issuer_state_or_province_name' : 'hangzhou', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'AB:A8:E0:2C:EF:AE:BF:9D:DD:FA:70:BA:2F:F2:CA:5C', - 'extra.raw_response' : '20020000', - 'extra.sha1_fingerprint' : '70:1A:1E:1F:EC:5F:7E:A9:12:32:B2:C9:8A:C9:EE:91:8E:0B:82:45', - 'extra.sha256_fingerprint' : '85:26:A2:F2:A2:50:CD:96:33:19:A6:2D:12:2E:97:6B:D3:06:3C:11:EA:01:B4:B7:25:2A:B7:4F:0A:8F:45:40', - 'extra.sha512_fingerprint' : '72:50:07:30:9A:6F:CB:FD:E2:80:69:02:65:62:77:16:C3:B4:0C:98:44:4E:D4:2C:AC:6B:AF:F8:9E:AB:51:C2:FA:A8:72:A3:45:DF:81:09:50:08:18:EB:03:34:FC:92:33:A7:12:46:FE:90:20:91:86:C5:4D:89:48:86:4C:CD', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.source.naics' : 518210, - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'Server', - 'extra.subject_country' : 'CN', - 'extra.subject_organization_name' : 'EMQ', - 'extra.subject_state_or_province_name' : 'hangzhou', - 'extra.tag' : 'mqtt,mqtt-anon', - 'feed.name' : 'Open Anonymous MQTT', - 'protocol.application' : 'mqtt', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 37963, - 'source.geolocation.cc' : 'CN', - 'source.geolocation.city' : 'SHENZHEN', - 'source.geolocation.region' : 'GUANGDONG SHENG', - 'source.ip' : '47.106.0.0', - 'source.port' : 8883, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:59:34+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mqtt-anon', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.cert_expiration_date' : '2022-03-06 13:48:03', - 'extra.cert_issue_date' : '2021-12-06 13:48:04', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : '06B25BEAD1F43266ABCFCDDE408D3544D04B', - 'extra.cipher_suite' : 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', - 'extra.code' : 'Connection Accepted', - 'extra.hex_code' : '00', - 'extra.issuer_common_name' : 'R3', - 'extra.issuer_country' : 'US', - 'extra.issuer_organization_name' : 'Lets Encrypt', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : '23:99:39:C6:77:D8:9F:55:90:FC:A5:FB:BA:72:8B:42', - 'extra.raw_response' : '20020000', - 'extra.sha1_fingerprint' : '20:0E:AC:E7:AF:07:8D:D3:16:7C:63:D1:B9:12:AD:1D:2C:F0:46:86', - 'extra.sha256_fingerprint' : 'DD:7A:4C:8A:1D:66:1D:7C:F5:17:04:5B:A0:B4:C4:E0:80:58:44:B4:DB:A7:5E:61:AE:43:9D:85:4C:9E:DC:83', - 'extra.sha512_fingerprint' : '55:B6:3D:56:A4:39:6E:99:B6:AF:72:AF:4D:3C:7C:C5:A8:C5:4F:A1:79:92:D0:46:8A:A2:9B:2A:48:0D:00:68:39:F0:B8:67:B4:E0:88:51:2A:D7:55:46:83:BD:ED:1E:09:6E:DB:3D:21:E2:AA:DB:42:6A:33:45:1A:2A:DB:4C', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.source.naics' : 518210, - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'example.com', - 'extra.tag' : 'mqtt,mqtt-anon', - 'feed.name' : 'Open Anonymous MQTT', - 'protocol.application' : 'mqtt', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 24940, - 'source.geolocation.cc' : 'DE', - 'source.geolocation.city' : 'WERNIGERODE', - 'source.geolocation.region' : 'SACHSEN-ANHALT', - 'source.ip' : '144.76.0.0', - 'source.port' : 8883, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:59:34+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mqtt-anon', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.cert_expiration_date' : '2030-08-05 16:51:57', - 'extra.cert_issue_date' : '2020-08-07 16:51:57', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'A71541EFAE529B03', - 'extra.cipher_suite' : 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA', - 'extra.code' : 'Connection Accepted', - 'extra.hex_code' : '00', - 'extra.issuer_common_name' : 'ClearView2Dev', - 'extra.issuer_organization_name' : 'Sohonet', - 'extra.issuer_organization_unit_name' : 'ClearView2Dev', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : '43:0D:A7:89:9E:76:8D:6E:D5:AD:95:CC:F2:91:87:56', - 'extra.raw_response' : '20020000', - 'extra.sha1_fingerprint' : '32:4B:66:98:FA:5B:D2:D1:F2:53:83:21:19:11:5A:A9:BE:85:56:16', - 'extra.sha256_fingerprint' : 'AE:0D:65:34:2F:51:F7:32:1E:DF:B1:DA:12:C7:6A:DE:42:B5:4B:FF:80:2C:E5:EF:99:F6:CC:01:4B:C9:77:68', - 'extra.sha512_fingerprint' : '44:C4:B8:19:FA:39:55:51:EC:E4:6D:C4:6D:0F:A5:46:BB:D5:F9:FD:A6:8D:DF:F3:2D:D2:92:6C:0B:D5:D3:25:CB:19:50:9D:A6:A4:D4:D3:2E:53:10:F5:8D:77:F7:90:F8:65:A7:79:AB:14:62:72:01:F3:EA:38:E2:68:C7:25', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.ssl_version' : 0, - 'extra.subject_common_name' : 'foo.example.com', - 'extra.subject_locality_name' : '<', - 'extra.subject_organization_name' : 'Sohonet', - 'extra.tag' : 'mqtt,mqtt-anon', - 'feed.name' : 'Open Anonymous MQTT', - 'protocol.application' : 'mqtt', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 5555, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'BURBANK', - 'source.geolocation.region' : 'CALIFORNIA', - 'source.ip' : '173.0.0.0', - 'source.port' : 8883, - 'source.reverse_dns' : 'example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:59:34+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_mssql.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_mssql.py deleted file mode 100644 index 0f12014e68..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_mssql.py +++ /dev/null @@ -1,123 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_mssql.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open MSSQL', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_mssql-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-mssql', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 310.0, - 'extra.instance_name': 'OPTIMA', - 'extra.named_pipe': '\\\\\\\\ERPOPTIMA\\\\pipe\\\\MSSQL$OPTIMA\\\\sql\\\\query', - 'extra.response_size': 310, - 'extra.tag': 'mssql', - 'extra.tcp_port': 49729, - 'extra.version': '13.2.5026.0', - 'feed.name': 'Open MSSQL', - 'protocol.application': 'mssql', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.local_hostname': 'ERPOPTIMA', - 'source.port': 1434, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-mssql', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 226.0, - 'extra.instance_name': 'MSSQLSERVER', - 'extra.response_size': 226, - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'mssql', - 'extra.tcp_port': 1433, - 'extra.version': '13.0.1601.5', - 'feed.name': 'Open MSSQL', - 'protocol.application': 'mssql', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.local_hostname': 'SERWER', - 'source.port': 1434, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-mssql', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 304.0, - 'extra.instance_name': 'INSERTGT', - 'extra.named_pipe': '\\\\\\\\ILONY\\\\pipe\\\\MSSQL$INSERTGT\\\\sql\\\\query', - 'extra.response_size': 304, - 'extra.tag': 'mssql', - 'extra.tcp_port': 49358, - 'extra.version': '10.50.2500.0', - 'feed.name': 'Open MSSQL', - 'protocol.application': 'mssql', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.local_hostname': 'ILONY', - 'source.port': 1434, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_mysql.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_mysql.py deleted file mode 100644 index 3e008f9502..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_mysql.py +++ /dev/null @@ -1,258 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_mysql.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible MySQL Server', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_mysql-test.csv", - } -EVENTS = [ - { - '__type' : 'Event', - 'classification.identifier' : 'open-mysql', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.browser_error' : 'x509: failed to load system roots and no roots provided', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.client_can_handle_expired_passwords' : True, - 'extra.client_compress' : True, - 'extra.client_connect_attrs' : True, - 'extra.client_connect_with_db' : True, - 'extra.client_deprecated_eof' : True, - 'extra.client_found_rows' : True, - 'extra.client_ignore_sigpipe' : True, - 'extra.client_ignore_space' : True, - 'extra.client_interactive' : True, - 'extra.client_local_files' : True, - 'extra.client_long_flag' : True, - 'extra.client_long_password' : True, - 'extra.client_multi_results' : True, - 'extra.client_multi_statements' : True, - 'extra.client_no_schema' : True, - 'extra.client_odbc' : True, - 'extra.client_plugin_auth' : True, - 'extra.client_plugin_auth_len_enc_client_data' : True, - 'extra.client_protocol_41' : True, - 'extra.client_ps_multi_results' : True, - 'extra.client_reserved' : True, - 'extra.client_secure_connection' : True, - 'extra.client_session_track' : False, - 'extra.client_ssl' : False, - 'extra.client_transactions' : False, - 'extra.error_code' : '1', - 'extra.error_id' : '1', - 'extra.error_message' : '1', - 'extra.handshake' : 'TLSv1.2', - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.mysql_protocol_version' : '10', - 'extra.server_version' : '5.7.37-0ubuntu0.18.04.1', - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.tag' : 'mysql', - 'feed.name' : 'Accessible MySQL Server', - 'protocol.application' : 'mysql', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 3306, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mysql', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.browser_error' : 'x509: failed to load system roots and no roots provided', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.client_can_handle_expired_passwords' : True, - 'extra.client_compress' : True, - 'extra.client_connect_attrs' : True, - 'extra.client_connect_with_db' : True, - 'extra.client_deprecated_eof' : True, - 'extra.client_found_rows' : True, - 'extra.client_ignore_sigpipe' : True, - 'extra.client_ignore_space' : True, - 'extra.client_interactive' : True, - 'extra.client_local_files' : True, - 'extra.client_long_flag' : True, - 'extra.client_long_password' : True, - 'extra.client_multi_results' : True, - 'extra.client_multi_statements' : True, - 'extra.client_no_schema' : True, - 'extra.client_odbc' : True, - 'extra.client_plugin_auth' : True, - 'extra.client_plugin_auth_len_enc_client_data' : True, - 'extra.client_protocol_41' : True, - 'extra.client_ps_multi_results' : True, - 'extra.client_reserved' : True, - 'extra.client_secure_connection' : True, - 'extra.client_session_track' : False, - 'extra.client_ssl' : False, - 'extra.client_transactions' : False, - 'extra.error_code' : '1', - 'extra.error_id' : '1', - 'extra.error_message' : '1', - 'extra.handshake' : 'TLSv1.2', - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.mysql_protocol_version' : '10', - 'extra.server_version' : '5.7.30-0ubuntu0.18.04.1-log', - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.tag' : 'mysql', - 'feed.name' : 'Accessible MySQL Server', - 'protocol.application' : 'mysql', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 3306, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mysql', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.browser_error' : 'x509: failed to load system roots and no roots provided', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.client_can_handle_expired_passwords' : True, - 'extra.client_compress' : True, - 'extra.client_connect_attrs' : True, - 'extra.client_connect_with_db' : True, - 'extra.client_deprecated_eof' : True, - 'extra.client_found_rows' : True, - 'extra.client_ignore_sigpipe' : True, - 'extra.client_ignore_space' : True, - 'extra.client_interactive' : True, - 'extra.client_local_files' : True, - 'extra.client_long_flag' : True, - 'extra.client_long_password' : True, - 'extra.client_multi_results' : True, - 'extra.client_multi_statements' : True, - 'extra.client_no_schema' : True, - 'extra.client_odbc' : True, - 'extra.client_plugin_auth' : True, - 'extra.client_plugin_auth_len_enc_client_data' : True, - 'extra.client_protocol_41' : True, - 'extra.client_ps_multi_results' : True, - 'extra.client_reserved' : True, - 'extra.client_secure_connection' : True, - 'extra.client_session_track' : False, - 'extra.client_ssl' : False, - 'extra.client_transactions' : False, - 'extra.error_code' : '1', - 'extra.error_id' : '1', - 'extra.error_message' : '1', - 'extra.handshake' : 'TLSv1.2', - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.mysql_protocol_version' : '10', - 'extra.server_version' : '8.0.23', - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.source.sector' : 'Retail Trade', - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.tag' : 'mysql', - 'feed.name' : 'Accessible MySQL Server', - 'protocol.application' : 'mysql', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 3306, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_nat_pmp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_nat_pmp.py deleted file mode 100644 index beeac2717f..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_nat_pmp.py +++ /dev/null @@ -1,116 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_nat_pmp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open NATPMP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_nat_pmp-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-natpmp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 6.0, - 'extra.external_ip': '192.168.0.1', - 'extra.opcode': '128', - 'extra.response_size': 12, - 'extra.tag': 'nat-pmp', - 'extra.uptime': 291278940, - 'feed.name': 'Open NATPMP', - 'protocol.application': 'natpmp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 5351, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-natpmp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 6.0, - 'extra.external_ip': '192.168.0.2', - 'extra.opcode': '128', - 'extra.response_size': 12, - 'extra.tag': 'nat-pmp', - 'extra.uptime': 768416, - 'feed.name': 'Open NATPMP', - 'protocol.application': 'natpmp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 5351, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, - {'__type': 'Event', - 'classification.identifier': 'open-natpmp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 6.0, - 'extra.external_ip': '192.168.0.3', - 'extra.opcode': '128', - 'extra.response_size': 12, - 'extra.tag': 'nat-pmp', - 'extra.uptime': 19629454, - 'feed.name': 'Open NATPMP', - 'protocol.application': 'natpmp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 5351, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_netbios.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_netbios.py deleted file mode 100644 index febe8305c1..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_netbios.py +++ /dev/null @@ -1,121 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_netbios.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Netbios', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_netbios-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-netbios-nameservice', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 4.58, - 'extra.mac_address': '00-00-00-00-00-00', - 'extra.machine_name': 'NBG6503', - 'extra.response_size': 229, - 'extra.tag': 'netbios', - 'feed.name': 'Netbios', - 'protocol.application': 'netbios-nameservice', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.account': 'NBG6503', - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 137, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-netbios-nameservice', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3.86, - 'extra.mac_address': '00-00-00-00-00-00', - 'extra.machine_name': 'NAS-OLD', - 'extra.response_size': 193, - 'extra.tag': 'netbios', - 'extra.workgroup': 'PRACOWNIAELN.', - 'feed.name': 'Netbios', - 'protocol.application': 'netbios-nameservice', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.account': 'NAS-OLD', - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 137, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-netbios-nameservice', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3.14, - 'extra.mac_address': '00-25-90-F0-64-64', - 'extra.machine_name': 'HR-SRV01', - 'extra.response_size': 157, - 'extra.sector': 'Government', - 'extra.tag': 'netbios', - 'extra.workgroup': 'HRSIGMA', - 'feed.name': 'Netbios', - 'protocol.application': 'netbios-nameservice', - 'protocol.transport': 'udp', - 'raw': 'InRpbWVzdGFtcCIsImlwIiwicHJvdG9jb2wiLCJwb3J0IiwiaG9zdG5hbWUiLCJ0YWciLCJtYWNfYWRkcmVzcyIsImFzbiIsImdlbyIsInJlZ2lvbiIsImNpdHkiLCJ3b3JrZ3JvdXAiLCJtYWNoaW5lX25hbWUiLCJ1c2VybmFtZSIsIm5haWNzIiwic2ljIiwic2VjdG9yIiwicmVzcG9uc2Vfc2l6ZSIsImFtcGxpZmljYXRpb24iCiIyMDEwLTAyLTEwIDAwOjAwOjAyIiwxOTIuMTY4LjAuMyx1ZHAsMTM3LG5vZGUwMy5leGFtcGxlLmNvbSxuZXRiaW9zLDAwLTI1LTkwLUYwLTY0LTY0LDY0NTEyLFpaLFJlZ2lvbixDaXR5LEhSU0lHTUEsSFItU1JWMDEsLDAsMCxHb3Zlcm5tZW50LDE1NywzLjE0', - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 137, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_netis_router.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_netis_router.py deleted file mode 100644 index 043cdf1aad..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_netis_router.py +++ /dev/null @@ -1,107 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Sebastian Wagner -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_netis_router.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_netis_router-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-netis', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 18.0, - 'extra.response': 'Login:', - 'extra.response_size': 18, - 'extra.tag': 'netis_vulnerability', - 'feed.name': 'Open-Netis', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 53413, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-netis', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 18.0, - 'extra.response': 'Login:', - 'extra.response_size': 18, - 'extra.tag': 'netis_vulnerability', - 'feed.name': 'Open-Netis', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 53413, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-netis', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 18.0, - 'extra.response': 'Login:', - 'extra.response_size': 18, - 'extra.tag': 'netis_vulnerability', - 'feed.name': 'Open-Netis', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 53413, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ntp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ntp.py deleted file mode 100644 index 85ef710d4e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ntp.py +++ /dev/null @@ -1,161 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_ntp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'NTP Version', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ntp-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'ntp-version', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 27.0, - 'extra.clock': '0xe6ac3809.363028e7', - 'extra.frequency': 2.018, - 'extra.jitter': 0.977, - 'extra.leap': 0.0, - 'extra.noise': '0.984', - 'extra.offset': 0.557, - 'extra.peer': 18986, - 'extra.poll': 10, - 'extra.precision': -10, - 'extra.refid': '81.15.252.130', - 'extra.reftime': '0xe6ac35ba.2d2e8f2b', - 'extra.response_size': 324, - 'extra.rootdelay': 17.685, - 'extra.rootdispersion': 61.254, - 'extra.stability': '0.027', - 'extra.state': '4', - 'extra.stratum': 4, - 'extra.system': 'UNIX', - 'extra.tag': 'ntpversion', - 'extra.version': '4', - 'feed.name': 'NTP Version', - 'protocol.application': 'ntp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 123, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'ntp-version', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 27.33, - 'extra.clk_wander': 0.007, - 'extra.clock': '0xE6AC3806.7DF3B7A0', - 'extra.frequency': -20.407, - 'extra.jitter': 8.776, - 'extra.leap': 0.0, - 'extra.mintc': '3', - 'extra.offset': -14.502, - 'extra.peer': 19244, - 'extra.precision': -10, - 'extra.refid': '10.48.21.21', - 'extra.reftime': '0xE6AC3431.B3B64790', - 'extra.response_size': 328, - 'extra.rootdelay': 32.25, - 'extra.rootdispersion': 105.778, - 'extra.sector': 'Transportation and Warehousing', - 'extra.stratum': 8, - 'extra.system': 'UNIX', - 'extra.tag': 'ntpversion', - 'extra.tc': 10, - 'extra.version': '4', - 'feed.name': 'NTP Version', - 'protocol.application': 'ntp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 123, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'ntp-version', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 27.0, - 'extra.clk_wander': 0.001, - 'extra.clock': '0xE6AC380A.5A1CAD00', - 'extra.frequency': -24.01, - 'extra.jitter': 2.343, - 'extra.leap': 0.0, - 'extra.mintc': '3', - 'extra.offset': 0.49, - 'extra.peer': 51892, - 'extra.precision': -10, - 'extra.refid': '172.28.0.1', - 'extra.reftime': '0xE6AC3020.0C49BA80', - 'extra.response_size': 324, - 'extra.rootdelay': 7.749, - 'extra.rootdispersion': 81.612, - 'extra.stratum': 4, - 'extra.system': 'UNIX', - 'extra.tag': 'ntpversion', - 'extra.tc': 10, - 'extra.version': '4', - 'feed.name': 'NTP Version', - 'protocol.application': 'ntp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 123, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ntpmonitor.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ntpmonitor.py deleted file mode 100644 index ff0e95f3ea..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ntpmonitor.py +++ /dev/null @@ -1,108 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_ntpmonitor.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'NTP Monitor', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ntpmonitor-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'ntp-monitor', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 55.33, - 'extra.packets': 2, - 'extra.size': 664, - 'feed.name': 'NTP Monitor', - 'protocol.application': 'ntp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 123, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'ntp-monitor', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3666.67, - 'extra.packets': 100, - 'extra.size': 44000, - 'feed.name': 'NTP Monitor', - 'protocol.application': 'ntp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 123, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'ntp-monitor', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3666.67, - 'extra.packets': 100, - 'extra.size': 44000, - 'feed.name': 'NTP Monitor', - 'protocol.application': 'ntp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 123, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_portmapper.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_portmapper.py deleted file mode 100644 index 11caec78a1..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_portmapper.py +++ /dev/null @@ -1,120 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_portmapper.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open Portmapper', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_portmapper-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-portmapper', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3.7, - 'extra.exports': '/mnt/export 192.168.0.0', - 'extra.programs': '100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp; ' - '100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp;', - 'extra.response_size': 148, - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'portmapper', - 'feed.name': 'Open Portmapper', - 'protocol.application': 'portmapper', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 111, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-portmapper', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3.7, - 'extra.exports': '/mnt/export 192.168.0.0', - 'extra.programs': '100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp; ' - '100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp;', - 'extra.response_size': 148, - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'portmapper', - 'feed.name': 'Open Portmapper', - 'protocol.application': 'portmapper', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 111, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-portmapper', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3.7, - 'extra.exports': '/mnt/export 192.168.0.0', - 'extra.programs': '100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp; ' - '100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp;', - 'extra.response_size': 148, - 'extra.sector': 'Government', - 'extra.tag': 'portmapper', - 'feed.name': 'Open Portmapper', - 'protocol.application': 'portmapper', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 111, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_postgres.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_postgres.py deleted file mode 100644 index 43a297f787..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_postgres.py +++ /dev/null @@ -1,199 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_postgres.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible-PostgreSQL', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_postgres-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-postgres', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.client_ssl' : False, - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.protocol_error_code' : '0A000', - 'extra.protocol_error_file' : 'postmaster.c', - 'extra.protocol_error_line' : '1798', - 'extra.protocol_error_message' : 'unsupported frontend protocol 255.255: server supports 1.0 to 3.0', - 'extra.protocol_error_routine' : 'ProcessStartupPacket', - 'extra.protocol_error_severity' : 'FATAL', - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.startup_error_code' : '28000', - 'extra.startup_error_file' : 'postmaster.c', - 'extra.startup_error_line' : 1893, - 'extra.startup_error_message' : 'no PostgreSQL user name specified in startup packet', - 'extra.startup_error_routine' : 'ProcessStartupPacket', - 'extra.startup_error_severity' : 'FATAL', - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.supported_protocols' : '1.0-3.0', - 'extra.tag' : 'postgres', - 'feed.name' : 'Accessible-PostgreSQL', - 'protocol.application' : 'postgres', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 5432, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-postgres', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.client_ssl' : False, - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.protocol_error_code' : '0A000', - 'extra.protocol_error_file' : 'postmaster.c', - 'extra.protocol_error_line' : '1798', - 'extra.protocol_error_message' : 'unsupported frontend protocol 255.255: server supports 1.0 to 3.0', - 'extra.protocol_error_routine' : 'ProcessStartupPacket', - 'extra.protocol_error_severity' : 'FATAL', - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.startup_error_code' : '28000', - 'extra.startup_error_file' : 'postmaster.c', - 'extra.startup_error_line' : 1893, - 'extra.startup_error_message' : 'no PostgreSQL user name specified in startup packet', - 'extra.startup_error_routine' : 'ProcessStartupPacket', - 'extra.startup_error_severity' : 'FATAL', - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.supported_protocols' : '1.0-3.0', - 'extra.tag' : 'postgres', - 'feed.name' : 'Accessible-PostgreSQL', - 'protocol.application' : 'postgres', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 5432, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-postgres', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.client_ssl' : False, - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.protocol_error_code' : '0A000', - 'extra.protocol_error_file' : 'postmaster.c', - 'extra.protocol_error_line' : '1798', - 'extra.protocol_error_message' : 'unsupported frontend protocol 255.255: server supports 1.0 to 3.0', - 'extra.protocol_error_routine' : 'ProcessStartupPacket', - 'extra.protocol_error_severity' : 'FATAL', - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.startup_error_code' : '28000', - 'extra.startup_error_file' : 'postmaster.c', - 'extra.startup_error_line' : 1893, - 'extra.startup_error_message' : 'no PostgreSQL user name specified in startup packet', - 'extra.startup_error_routine' : 'ProcessStartupPacket', - 'extra.startup_error_severity' : 'FATAL', - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.supported_protocols' : '1.0-3.0', - 'extra.tag' : 'postgres', - 'feed.name' : 'Accessible-PostgreSQL', - 'protocol.application' : 'postgres', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 5432, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_qotd.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_qotd.py deleted file mode 100644 index de52af6259..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_qotd.py +++ /dev/null @@ -1,119 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_qotd.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open QOTD', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_qotd-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-qotd', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 166.0, - 'extra.quote': '_The secret of being miserable is to have leisure to bother ' - 'about whether?? you are happy or not. The cure for it is ' - 'occupation._?? George Bernard Shaw (1856-1950)?', - 'extra.response_size': 166, - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'qotd', - 'feed.name': 'Open QOTD', - 'protocol.application': 'qotd', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 17, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-qotd', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 162.0, - 'extra.quote': '_Oh the nerves, the nerves; the mysteries of this machine ' - 'called man!?? Oh the little that unhinges it, poor creatures ' - 'that we are!_?? Charles Dickens (1812-70)?', - 'extra.response_size': 162, - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'qotd', - 'feed.name': 'Open QOTD', - 'protocol.application': 'qotd', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 17, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-qotd', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 162.0, - 'extra.quote': '_Oh the nerves, the nerves; the mysteries of this machine ' - 'called man!?? Oh the little that unhinges it, poor creatures ' - 'that we are!_?? Charles Dickens (1812-70)?', - 'extra.response_size': 162, - 'extra.tag': 'qotd', - 'feed.name': 'Open QOTD', - 'protocol.application': 'qotd', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 17, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_quic.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_quic.py deleted file mode 100644 index 23d11ce996..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_quic.py +++ /dev/null @@ -1,118 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_quic.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible QUIC Report', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_quic-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-quic', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.source.naics' : 517311, - 'extra.tag' : 'quic', - 'extra.version_field_1' : 'Q050', - 'extra.version_field_3' : 'Q046', - 'extra.version_field_4' : 'Q043', - 'feed.name' : 'Accessible QUIC Report', - 'protocol.transport' : 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 5607, - 'source.geolocation.cc' : 'UK', - 'source.geolocation.city' : 'LONDON', - 'source.geolocation.region' : 'LONDON', - 'source.ip' : '176.255.0.0', - 'source.port' : 443, - 'source.reverse_dns' : 'test1.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T14:31:17+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-quic', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.source.naics' : 517311, - 'extra.tag' : 'quic', - 'extra.version_field_1' : 'Q050', - 'extra.version_field_2' : 'Q046', - 'extra.version_field_4' : 'Q043', - 'feed.name' : 'Accessible QUIC Report', - 'protocol.transport' : 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 6327, - 'source.geolocation.cc' : 'CA', - 'source.geolocation.city' : 'MEACHAM', - 'source.geolocation.region' : 'SASKATCHEWAN', - 'source.ip' : '24.244.0.0', - 'source.port' : 443, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T14:31:17+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-quic', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.source.naics' : 517919, - 'extra.tag' : 'quic', - 'extra.version_field_2' : 'Q050', - 'extra.version_field_3' : 'Q046', - 'extra.version_field_4' : 'Q043', - 'feed.name' : 'Accessible QUIC Report', - 'protocol.transport' : 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 20940, - 'source.geolocation.cc' : 'JP', - 'source.geolocation.city' : 'OSAKA', - 'source.geolocation.region' : 'OSAKA', - 'source.ip' : '23.60.0.0', - 'source.port' : 443, - 'source.reverse_dns' : 'test3.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T14:31:17+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_radmin.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_radmin.py deleted file mode 100644 index 7c052c451c..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_radmin.py +++ /dev/null @@ -1,236 +0,0 @@ -# SPDX-FileCopyrightText: 2020 sinus-x -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), "testdata/scan_radmin.csv")) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = { - "feed.name": "Accessible Radmin", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2020-08-19T00:00:00+00:00", - "extra.file_name": "2020-08-19-scan_radmin-test-test.csv", -} - -EVENTS = [ - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517312, - "extra.tag": "radmin", - "extra.version": "Radmin (Details Unknown)", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.asn": 701, - "source.geolocation.cc": "US", - "source.geolocation.city": "BROOKLYN", - "source.geolocation.region": "NEW YORK", - "source.ip": "74.101.218.75", - "source.port": 4899, - "source.reverse_dns": "static-74-101-218-75.nycmny.fios.verizon.net", - "time.source": "2020-07-06T13:55:26+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[1]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.tag": "radmin", - "extra.version": "Radmin v3.X Radmin Authentication", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.asn": 56618, - "source.geolocation.cc": "RU", - "source.geolocation.city": "MURMANSK", - "source.geolocation.region": "MURMANSKAYA OBLAST", - "source.ip": "192.162.189.171", - "source.port": 4899, - "source.reverse_dns": "rubin.an.ru", - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[2]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517311, - "extra.tag": "radmin", - "extra.version": "Radmin (Details Unknown)", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.geolocation.cc": "CN", - "source.geolocation.city": "BEIJING", - "source.geolocation.region": "BEIJING SHI", - "source.asn": 4808, - "source.ip": "111.197.143.69", - "source.port": 4899, - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[3]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517311, - "extra.tag": "radmin", - "extra.version": "Radmin v3.X Radmin Authentication", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.geolocation.cc": "KR", - "source.geolocation.city": "DAEIN-DONG", - "source.geolocation.region": "GWANGJU-GWANGYEOKSI", - "source.asn": 4766, - "source.ip": "121.147.215.220", - "source.port": 4899, - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[4]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517311, - "extra.tag": "radmin", - "extra.version": "Radmin v3.X Radmin Authentication", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.geolocation.cc": "KR", - "source.geolocation.city": "DAEIN-DONG", - "source.geolocation.region": "GWANGJU-GWANGYEOKSI", - "source.asn": 4766, - "source.ip": "121.147.215.178", - "source.port": 4899, - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[5]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517312, - "extra.tag": "radmin", - "extra.version": "Radmin v3.X Radmin Authentication", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.geolocation.cc": "CN", - "source.geolocation.city": "CHONGQING", - "source.geolocation.region": "CHONGQING SHI", - "source.asn": 9808, - "source.ip": "183.230.5.219", - "source.port": 4899, - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[6]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.tag": "radmin", - "extra.version": "Radmin v3.X Radmin Authentication", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.geolocation.cc": "RU", - "source.geolocation.city": "MOSCOW", - "source.geolocation.region": "MOSKVA", - "source.asn": 34300, - "source.ip": "85.93.154.74", - "source.port": 4899, - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[7]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517311, - "extra.tag": "radmin", - "extra.version": "Radmin v3.X Radmin Authentication", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.geolocation.cc": "BE", - "source.geolocation.city": "BRASSCHAAT", - "source.geolocation.region": "ANTWERPEN", - "source.asn": 5432, - "source.ip": "81.246.135.247", - "source.port": 4899, - "source.reverse_dns": "247.135-246-81.adsl-dyn.isp.belgacom.be", - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[8]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517312, - "extra.tag": "radmin", - "extra.version": "Radmin v3.X Radmin Authentication", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.geolocation.cc": "ES", - "source.geolocation.city": "LAS PALMAS DE GRAN CANARIA", - "source.geolocation.region": "LAS PALMAS", - "source.asn": 12430, - "source.ip": "46.27.146.22", - "source.port": 4899, - "source.reverse_dns": "static-22-146-27-46.ipcom.comunitel.net", - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[9]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == "__main__": - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_rdp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_rdp.py deleted file mode 100644 index 28a4a02c23..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_rdp.py +++ /dev/null @@ -1,117 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_rdp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible RDP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_rdp-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Accessible RDP', - "classification.identifier": "open-rdp", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.bluekeep_vulnerable": False, - "extra.cert_expiration_date": "2019-10-29 02:22:06", - "extra.cert_issue_date": "2019-04-29 02:22:06", - "extra.cert_length": 5678, - "extra.cert_serial_number": "1EF2B37AF850C9BF4E88F18177001D6B", - "extra.cve20190708_vulnerable": False, - "extra.issuer_common_name": "KABESRV.KABE.local", - "extra.key_algorithm": "rsaEncryption", - "extra.md5_fingerprint": "BC:6E:C3:E2:98:22:EC:BA:5B:30:E2:53:FD:4A:9D:FF", - "extra.naics": 517311, - "extra.rdp_protocol": "RDP", - "extra.sha1_fingerprint": "EC:BB:4D:DB:9F:0C:D3:FF:5B:49:EA:B1:56:62:B6:A7:5D:60:54:42", - "extra.sha256_fingerprint": "B7:C9:F4:07:D5:C0:75:1D:EA:0C:40:E7:26:39:C2:30:C6:13:83:7E:18:46:D8:E9:4C:45:3F:88:1B:0B:70:76", - "extra.sha512_fingerprint": "08:AC:75:FA:EB:A3:2B:44:15:DE:6D:A7:0B:C0:AE:17:94:F3:55:D9:EC:70:AC:5B:B7:94:79:F0:D7:84:83:89:CB:A9:11:E0:08:D7:54:4D:33:85:89:D2:A8:DD:9D:15:F4:CC:95:DE:6A:E3:DF:6B:FA:8B:27:E3:DA:16:AF:0A", - "extra.signature_algorithm": "sha256WithRSAEncryption", - "extra.ssl_version": 2, - "extra.subject_common_name": "KABESRV.KABE.local", - "extra.tag": "rdp", - "protocol.application": "rdp", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 5678, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.178", - "source.port": 5678, - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T15:45:51+00:00" - }, - {'__type': 'Event', - 'feed.name': 'Accessible RDP', - "classification.identifier": "open-rdp", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.bluekeep_vulnerable": False, - "extra.cert_expiration_date": "2019-10-16 06:15:20", - "extra.cert_issue_date": "2019-04-16 06:15:20", - "extra.cert_length": 5678, - "extra.cert_serial_number": "3FF3EBC5CF154BA54D128A8548C8AAF5", - "extra.cve20190708_vulnerable": False, - "extra.issuer_common_name": "RAMBLA01.rambla.local", - "extra.key_algorithm": "rsaEncryption", - "extra.md5_fingerprint": "38:73:6A:B3:AA:41:69:C9:BA:E7:3D:D7:40:16:F8:AA", - "extra.naics": 517311, - "extra.rdp_protocol": "RDP", - "extra.sector": "Information Technology", - "extra.sha1_fingerprint": "7A:67:1F:F8:87:C6:B0:AC:A9:84:15:B7:40:EC:CB:19:AA:E3:19:52", - "extra.sha256_fingerprint": "8F:CD:7D:C4:80:2D:8D:9B:06:A0:40:18:9F:ED:73:7A:BA:83:55:BE:1B:56:83:A2:97:DF:BB:B4:06:57:CB:F1", - "extra.sha512_fingerprint": "E8:9B:9A:93:69:B4:58:01:D8:46:C2:DC:01:20:1E:DD:93:E1:EB:E3:9D:6B:65:A0:C5:00:6C:A4:44:08:FE:A4:A6:19:FF:55:79:F2:AA:61:68:C8:1C:B0:CE:78:EB:84:DD:29:9D:64:2F:4E:25:31:3A:6C:B8:02:C9:AF:F5:1F", - "extra.signature_algorithm": "sha1WithRSAEncryption", - "extra.ssl_version": 2, - "extra.subject_common_name": "RAMBLA01.rambla.local", - "extra.tag": "rdp", - "protocol.application": "rdp", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 5678, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.233", - "source.port": 5678, - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T15:45:51+00:00" - }, - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_rdpeudp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_rdpeudp.py deleted file mode 100644 index 54be35a26f..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_rdpeudp.py +++ /dev/null @@ -1,109 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Sebastian Waldbauer -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_rdpeudp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible MS RDPEUDP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_rdpeudp-test-geo.csv", - } - -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'accessible-msrdpeudp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 77.0, - 'extra.response_size': 1232, - 'extra.sessionid': '05b28c0c', - 'extra.tag': 'rdpeudp', - 'feed.name': 'Accessible MS RDPEUDP', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 3389, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-msrdpeudp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 77.0, - 'extra.response_size': 1232, - 'extra.sessionid': '053d355f', - 'extra.tag': 'rdpeudp', - 'feed.name': 'Accessible MS RDPEUDP', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 3389, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-msrdpeudp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 77.0, - 'extra.response_size': 1232, - 'extra.sessionid': '0567a8cb', - 'extra.tag': 'rdpeudp', - 'feed.name': 'Accessible MS RDPEUDP', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 3389, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_redis.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_redis.py deleted file mode 100644 index 04552e2ec0..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_redis.py +++ /dev/null @@ -1,107 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_redis.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open Redis', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_redis-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Open Redis', - "classification.identifier": "open-redis", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.build_id": "26069fb482f6334b", - "extra.connected_clients": "50", - "extra.gcc_version": "4.7.2", - "extra.git_sha1": "00000000", - "extra.mode": "standalone", - "extra.multiplexing_api": "epoll", - "extra.naics": 541512, - "extra.os.name": "Linux 3.2.0-4-amd64 x86_64", - "extra.process_id": "2127", - "extra.run_id": "d440b0b2fb3d1db655ad607e11e6f38011a0f599", - "extra.sic": 737999, - "extra.tag": "redis", - "extra.uptime": 27946314, - "extra.version": "2.8.19", - "protocol.application": "redis", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 201229, - "source.geolocation.cc": "DE", - "source.geolocation.city": "FRANKFURT AM MAIN", - "source.geolocation.region": "HESSEN", - "source.ip": "198.51.100.152", - "source.port": 6379, - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2016-07-24T00:42:33+00:00" - }, - {'__type': 'Event', - 'feed.name': 'Open Redis', - "classification.identifier": "open-redis", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.build_id": "e41bf84a0cecf09d", - "extra.connected_clients": "25376", - "extra.gcc_version": "4.8.4", - "extra.git_sha1": "00000000", - "extra.mode": "standalone", - "extra.multiplexing_api": "epoll", - "extra.os.name": "Linux 3.18.24-sirzion x86_64", - "extra.process_id": "343519", - "extra.run_id": "53d63f23511dc0080b49aaa8e8203d65619f1c8c", - "extra.tag": "redis", - "extra.uptime": 310556, - "extra.version": "3.0.6", - "protocol.application": "redis", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 12586, - "source.geolocation.cc": "DE", - "source.geolocation.city": "FRANKFURT AM MAIN", - "source.geolocation.region": "HESSEN", - "source.ip": "198.51.100.67", - "source.port": 6379, - "source.reverse_dns": "198-51-100-67.example.net", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2016-07-24T00:42:43+00:00" - }, - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_rsync.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_rsync.py deleted file mode 100644 index e2a961f710..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_rsync.py +++ /dev/null @@ -1,116 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_rsync.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible Rsync', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_rsync-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'accessible-rsync', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.has_password': 'N', - 'extra.module': 'system|Backup system;system_full|Backup full ' - 'system;mysql|Backup virtual mysql;netadmin|Backup virtual ' - 'netadmin;', - 'extra.tag': 'rsync', - 'feed.name': 'Accessible Rsync', - 'protocol.application': 'rsync', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 873, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-rsync', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.has_password': 'N', - 'extra.module': 'system|Backup system;system_full|Backup full ' - 'system;mysql|Backup virtual mysql;netadmin|Backup virtual ' - 'netadmin;', - 'extra.tag': 'rsync', - 'feed.name': 'Accessible Rsync', - 'protocol.application': 'rsync', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 873, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-rsync', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.has_password': 'N', - 'extra.module': 'system|Backup system;system_full|Backup full ' - 'system;mysql|Backup virtual mysql;netadmin|Backup virtual ' - 'netadmin;', - 'extra.tag': 'rsync', - 'feed.name': 'Accessible Rsync', - 'protocol.application': 'rsync', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 873, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_sip.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_sip.py deleted file mode 100644 index 6b972ec5d5..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_sip.py +++ /dev/null @@ -1,124 +0,0 @@ -# SPDX-FileCopyrightText: 2023 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_sip.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible-SIP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_sip-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-sip', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.sip_allow': 'INVITE,ACK,BYE,CANCEL,REGISTER', - 'extra.amplification': 15.57, - 'extra.content_length': 0, - 'extra.response_size': 109, - 'extra.sip': 'SIP/2.0', - 'extra.sip_code': '489', - 'extra.sip_reason': 'Event Package Not Supported', - 'extra.tag': 'sip', - 'feed.name': 'Accessible-SIP', - 'protocol.application': 'sip', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 5060, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-sip', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 62.57, - 'extra.content_length': 364, - 'extra.content_type': 'text/plain', - 'extra.response_size': 438, - 'extra.sip': 'SIP/2.0', - 'extra.sip_code': '400', - 'extra.sip_reason': 'Bad Request', - 'extra.tag': 'sip', - 'feed.name': 'Accessible-SIP', - 'protocol.application': 'sip', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 5060, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-sip', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 6.57, - 'extra.content_length': 0, - 'extra.response_size': 46, - 'extra.sip': 'SIP/2.0', - 'extra.sip_code': '400', - 'extra.sip_reason': 'Bad Request', - 'extra.tag': 'sip', - 'feed.name': 'Accessible-SIP', - 'protocol.application': 'sip', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 5060, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_slp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_slp.py deleted file mode 100644 index f05973cf5c..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_slp.py +++ /dev/null @@ -1,137 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_slp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible SLP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_slp-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-slp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.error_code': '5', - 'extra.error_code_text': 'Unsupported SLP SPI', - 'extra.flags': '0x0000', - 'extra.function': '2', - 'extra.function_text': 'Service reply', - 'extra.language_tag': 'en', - 'extra.language_tag_length': '2', - 'extra.raw_response': 'MDIwMjAwMDAxNDAwMDAwMDAwMDAwMDA1MDAwMjY1NmUwMDA1MDAwMA==', - 'extra.response_size': 40, - 'extra.tag': 'slp', - 'extra.version': '2', - 'extra.xid': '5', - 'feed.name': 'Accessible SLP', - 'protocol.application': 'slp', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 427, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-slp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.error_code': '5', - 'extra.error_code_text': 'Unsupported SLP SPI', - 'extra.flags': '0x0000', - 'extra.function': '2', - 'extra.function_text': 'Service reply', - 'extra.language_tag': 'en', - 'extra.language_tag_length': '2', - 'extra.raw_response': 'MDIwMjAwMDAxNDAwMDAwMDAwMDAwMDA1MDAwMjY1NmUwMDA1MDAwMA==', - 'extra.response_size': 40, - 'extra.tag': 'slp', - 'extra.version': '2', - 'extra.xid': '5', - 'feed.name': 'Accessible SLP', - 'protocol.application': 'slp', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 427, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-slp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.error_code': '5', - 'extra.error_code_text': 'Unsupported SLP SPI', - 'extra.flags': '0x0000', - 'extra.function': '2', - 'extra.function_text': 'Service reply', - 'extra.language_tag': 'en', - 'extra.language_tag_length': '2', - 'extra.raw_response': 'MDIwMjAwMDAxNDAwMDAwMDAwMDAwMDA1MDAwMjY1NmUwMDA1MDAwMA==', - 'extra.response_size': 40, - 'extra.tag': 'slp', - 'extra.version': '2', - 'extra.xid': '5', - 'feed.name': 'Accessible SLP', - 'protocol.application': 'slp', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 427, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_smb.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_smb.py deleted file mode 100644 index 921525122c..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_smb.py +++ /dev/null @@ -1,124 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_smb.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible SMB', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_smb-test-geo.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Accessible SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 445, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Accessible SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 445, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Accessible SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 445, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_smb_json.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_smb_json.py deleted file mode 100644 index cae83d2733..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_smb_json.py +++ /dev/null @@ -1,123 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Sebastian Waldbauer -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest -import json - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser_json import ShadowserverJSONParserBot -from intelmq.tests.bots.parsers.shadowserver.test_testdata import csvtojson - -EXAMPLE_FILE = csvtojson(os.path.join(os.path.dirname(__file__), 'testdata/scan_smb.csv')) - -EXAMPLE_REPORT = { - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_smb-test-geo.json", - } - -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Accessible-SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode(json.dumps([json.loads(EXAMPLE_FILE)[0]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 445, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Accessible-SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode(json.dumps([json.loads(EXAMPLE_FILE)[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 445, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Accessible-SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode(json.dumps([json.loads(EXAMPLE_FILE)[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 445, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverJSONParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverJSONParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverJSONParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_smtp_vulnerable.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_smtp_vulnerable.py deleted file mode 100644 index 4428420cfb..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_smtp_vulnerable.py +++ /dev/null @@ -1,92 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Mikk Margus Möll -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_smtp_vulnerable.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Vulnerable SMTP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2021-07-08T00:00:00+00:00", - "extra.file_name": "2021-07-08-scan_smtp_vulnerable-test-test.csv", - } - -EVENTS = [ - { - '__type' : 'Event', - 'classification.identifier' : 'vulnerable-smtp', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.banner' : '220 smtp-server.invalid ESMTP Exim 4.80 Wed, 11 Jun 2021 10:00:00 +0300|', - 'extra.tag' : 'smtp;21nails', - 'feed.name' : 'Vulnerable SMTP', - 'protocol.application' : 'smtp', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 12345, - 'source.geolocation.cc' : 'EE', - 'source.geolocation.city' : 'TALLINN', - 'source.geolocation.region' : 'HARJUMAA', - 'source.ip' : '1.2.3.4', - 'source.port' : 25, - 'source.reverse_dns' : 'smtp-server.invalid', - 'time.observation' : '2021-07-08T00:00:00+00:00', - 'time.source' : '2021-07-08T11:58:42+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'vulnerable-smtp', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.banner' : '220 smtp-out.invalid, ESMTP EXIM 4.86_2|', - 'extra.tag' : 'smtp;21nails', - 'feed.name' : 'Vulnerable SMTP', - 'protocol.application' : 'smtp', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 23456, - 'source.geolocation.cc' : 'EE', - 'source.geolocation.city' : 'TALLINN', - 'source.geolocation.region' : 'HARJUMAA', - 'source.ip' : '5.6.7.8', - 'source.port' : 25, - 'source.reverse_dns' : 'smtp-out.invalid', - 'time.observation' : '2021-07-08T00:00:00+00:00', - 'time.source' : '2021-07-08T11:58:44+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_snmp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_snmp.py deleted file mode 100644 index e6da5b34f9..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_snmp.py +++ /dev/null @@ -1,120 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_snmp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open SNMP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_snmp-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-snmp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 1.94, - 'extra.community': 'public', - 'extra.response_size': 165, - 'extra.sysdesc': 'Linux localhost 3.18.20 #1 SMP Mon Jul 9 14:11:21 CST 2018 ' - 'armv7l', - 'extra.tag': 'snmp', - 'extra.version': '2', - 'feed.name': 'Open SNMP', - 'protocol.application': 'snmp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 161, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-snmp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 1.35, - 'extra.community': 'public', - 'extra.device_sector': 'consumer', - 'extra.device_type': 'router', - 'extra.device_vendor': 'MikroTik', - 'extra.response_size': 115, - 'extra.sysdesc': 'RouterOS CCR1009-8G-1S-1S+', - 'extra.tag': 'snmp,iot', - 'extra.version': '2', - 'feed.name': 'Open SNMP', - 'protocol.application': 'snmp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 161, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-snmp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 1.0, - 'extra.community': 'public', - 'extra.response_size': 85, - 'extra.tag': 'snmp', - 'extra.version': '2', - 'feed.name': 'Open SNMP', - 'protocol.application': 'snmp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 161, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_socks.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_socks.py deleted file mode 100644 index 067602aa10..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_socks.py +++ /dev/null @@ -1,107 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_socks.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open SOCKS', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_socks-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-socks', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'feed.name' : 'Open SOCKS', - 'protocol.application' : 'socks4', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 1080, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-socks', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'feed.name' : 'Open SOCKS', - 'protocol.application' : 'socks5', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 1080, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-socks', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.source.sector' : 'Retail Trade', - 'feed.name' : 'Open SOCKS', - 'protocol.application' : 'socks4', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 1080, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssdp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ssdp.py deleted file mode 100644 index 0811f15eda..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssdp.py +++ /dev/null @@ -1,136 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ssdp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open SSDP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ssdp-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-ssdp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3.35, - 'extra.cache_control': 'max-age=100', - 'extra.header': 'HTTP/1.1 200 OK', - 'extra.host': 'node01.example.com', - 'extra.location': 'http://192.168.200.254:49152/description.xml', - 'extra.response_size': 325, - 'extra.search_target': 'upnp:rootdevice', - 'extra.sector': 'Government', - 'extra.server': 'Linux/2.6.26, UPnP/1.0, Portable SDK for UPnP devices/1.3.1', - 'extra.systime': 'Sun, 21 Aug 2022 09:51:13 GMT', - 'extra.tag': 'ssdp', - 'extra.unique_service_name': 'uuid:28802880-2880-1880-a880-001bc502f600::upnp:rootdevice', - 'feed.name': 'Open SSDP', - 'protocol.application': 'ssdp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 60194, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-ssdp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 2.71, - 'extra.cache_control': 'max-age = 1800', - 'extra.header': 'HTTP/1.1 200 OK', - 'extra.host': 'node02.example.com', - 'extra.location': 'http://95.160.216.14:52235/dmr/SamsungMRDesc.xml', - 'extra.response_size': 263, - 'extra.search_target': 'upnp:rootdevice', - 'extra.server': 'Linux/9.0 UPnP/1.0 PROTOTYPE/1.0', - 'extra.tag': 'ssdp', - 'extra.unique_service_name': 'uuid:f144ca92-6816-94b5-b95f-b58180834044::upnp:rootdevice', - 'feed.name': 'Open SSDP', - 'protocol.application': 'ssdp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 38732, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-ssdp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 4.79, - 'extra.cache_control': 'max-age=1800', - 'extra.header': 'HTTP/1.1 200 OK', - 'extra.host': 'node03.example.com', - 'extra.location': 'http://192.168.1.3:8008/ssdp/device-desc.xml', - 'extra.response_size': 465, - 'extra.search_target': 'upnp:rootdevice', - 'extra.sector': 'Government', - 'extra.server': 'Linux/3.10.79, UPnP/1.0, Portable SDK for UPnP ' - 'devices/1.6.18', - 'extra.systime': 'Sun, 03 Jan 2016 21:37:50 GMT', - 'extra.tag': 'ssdp', - 'extra.unique_service_name': 'uuid:62fa0fc8-079d-d00f-2e22-59b49fb488f9::upnp:rootdevice', - 'feed.name': 'Open SSDP', - 'protocol.application': 'ssdp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 57626, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssh.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ssh.py deleted file mode 100644 index a01383713b..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssh.py +++ /dev/null @@ -1,182 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ssh.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible SSH', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_ssh-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ssh', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.algorithm' : 'ecdsa-sha2-nistp256', - 'extra.available_ciphers' : 'chacha20-poly1305@openssh.com, aes128-ctr, aes192-ctr, aes256-ctr, aes128-gcm@openssh.com, aes256-gcm@openssh.com, aes128-cbc, aes192-cbc, aes256-cbc, blowfish-cbc, cast128-cbc, 3des-cbc', - 'extra.available_compression' : 'none, zlib@openssh.com', - 'extra.available_kex' : 'curve25519-sha256, curve25519-sha256@libssh.org, ecdh-sha2-nistp256, ecdh-sha2-nistp384, ecdh-sha2-nistp521, diffie-hellman-group-exchange-sha256, diffie-hellman-group16-sha512, diffie-hellman-group18-sha512, diffie-hellman-group-exchange-sha1, diffie-hellman-group14-sha256, diffie-hellman-group14-sha1, diffie-hellman-group1-sha1', - 'extra.available_mac' : 'umac-64-etm@openssh.com, umac-128-etm@openssh.com, hmac-sha2-256-etm@openssh.com, hmac-sha2-512-etm@openssh.com, hmac-sha1-etm@openssh.com, umac-64@openssh.com, umac-128@openssh.com, hmac-sha2-256, hmac-sha2-512, hmac-sha1', - 'extra.ecdsa_curve' : 'P-256', - 'extra.ecdsa_curve25519' : '1xx7ASut7BF4ED8b592bebZBMBKTCzOsmbH4cjwx/0U=', - 'extra.ecdsa_public_key_b' : 'WsY12Ko6k+ez671VdpiGvGUdBrDMU7D2O848PifSYEs=', - 'extra.ecdsa_public_key_gx' : 'axfR8uEsQkf4vOblY6RA8ncDfYEt6zOg9KE5RdiYwpY=', - 'extra.ecdsa_public_key_gy' : 'T+NC4v4af5uO5+tKfA+eFivOM1drMV7Oy7ZAaDe/UfU=', - 'extra.ecdsa_public_key_length' : '256', - 'extra.ecdsa_public_key_n' : '/////wAAAAD//////////7zm+q2nF56E87nKwvxjJVE=', - 'extra.ecdsa_public_key_p' : '/////wAAAAEAAAAAAAAAAAAAAAD///////////////8=', - 'extra.ecdsa_public_key_x' : 'NIQdotpy2HAHfSu0DjEmA3cb3NeQKabZaFX9OU0GsPY=', - 'extra.ecdsa_public_key_y' : '0fuNQAZX7XciX2YkqIHtK2dWLBYwVCCqvl//zoM42kI=', - 'extra.selected_cipher' : 'aes128-ctr', - 'extra.selected_compression' : 'none', - 'extra.selected_kex' : 'curve25519-sha256@libssh.org', - 'extra.selected_mac' : 'hmac-sha2-256', - 'extra.server_cookie' : 'bGjsifbPIDWT7tAu8BMjyg==', - 'extra.server_host_key' : 'AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDSEHaLacthwB30rtA4xJgN3G9zXkCmm2WhV/TlNBrD20fuNQAZX7XciX2YkqIHtK2dWLBYwVCCqvl//zoM42kI=', - 'extra.server_host_key_sha256' : 'a6e4e1c16ba25d51bcddc58a6e16797144575dd18d02d9dedf75093d2b15c557', - 'extra.server_signature_raw' : 'AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAABKAAAAIQCd+X/B/OEx+FrwJSlVecOvNMuS5w2vTRz0z4prM+5VBwAAACEArU60b9CHs/d5BgyaOd7vmFygTMK5SyL90bS8VIztX/4=', - 'extra.server_signature_value' : 'AAAAIQCd+X/B/OEx+FrwJSlVecOvNMuS5w2vTRz0z4prM+5VBwAAACEArU60b9CHs/d5BgyaOd7vmFygTMK5SyL90bS8VIztX/4=', - 'extra.serverid_raw' : 'SSH-2.0-OpenSSH_7.4', - 'extra.serverid_software' : 'OpenSSH_7.4', - 'extra.serverid_version' : '2.0', - 'extra.source.naics' : 454110, - 'extra.tag' : 'ssh', - 'extra.userauth_methods' : 'publickey', - 'feed.name' : 'Accessible SSH', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 16509, - 'source.geolocation.cc' : 'JP', - 'source.geolocation.city' : 'TOKYO', - 'source.geolocation.region' : 'TOKYO', - 'source.ip' : '18.179.0.0', - 'source.port' : 22, - 'source.reverse_dns' : 'ec2-18-179-0-0.ap-northeast-1.compute.amazonaws.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T02:20:37+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ssh', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.algorithm' : 'ssh-rsa', - 'extra.available_ciphers' : 'aes128-cbc, 3des-cbc, aes256-cbc, twofish256-cbc, twofish-cbc, twofish128-cbc, blowfish-cbc', - 'extra.available_compression' : 'none', - 'extra.available_kex' : 'diffie-hellman-group1-sha1', - 'extra.available_mac' : 'hmac-sha1-96, hmac-sha1, hmac-md5', - 'extra.device_vendor' : 'Arris', - 'extra.rsa_exponent' : '65537', - 'extra.rsa_length' : '1040', - 'extra.rsa_modulus' : 'g6tZBqqsHUFa8gjXKMg2waZy/6JLzNnfTZkdDo6a3E6Amdk2eVlGbdgRsDxNUCyWLJI4b+xsJX0OKoRKhw4N+cDK5Du4/S00Auylt8YrDi7lIpeBZjMT5KJerN/feOcwRqY8NJfwZiZ1A5brcWr5HXfOH6aRTMFzadr248VdUmFXPQ==', - 'extra.selected_cipher' : 'aes128-cbc', - 'extra.selected_compression' : 'none', - 'extra.selected_kex' : 'diffie-hellman-group1-sha1', - 'extra.selected_mac' : 'hmac-sha1', - 'extra.server_cookie' : 'Y4RQS9sdRgEFwNJKVP6bZg==', - 'extra.server_host_key' : 'AAAAB3NzaC1yc2EAAAADAQABAAAAgwCDq1kGqqwdQVryCNcoyDbBpnL/okvM2d9NmR0OjprcToCZ2TZ5WUZt2BGwPE1QLJYskjhv7GwlfQ4qhEqHDg35wMrkO7j9LTQC7KW3xisOLuUil4FmMxPkol6s39945zBGpjw0l/BmJnUDlutxavkdd84fppFMwXNp2vbjxV1SYVc9', - 'extra.server_host_key_sha256' : 'd53fedbfe92e631264629882b2e85bfd213ca4b07b824cd31f8de1fcb8d0ddcb', - 'extra.server_signature_raw' : 'AAAAB3NzaC1yc2EAAACCLQj+UTJEQqdb/p/c/19yVc63eo+rnedwXKjP6eNNxxijN2cFoOjVMeqT2QTBjyoN7yRWBU2EID+3y2jUYT8mCqmqfyUv1eEbiCfLVlUyQ0X/CY9I5DDb5l6yEjNkuH2xVNNV6R7GFRwyYKAsYzfy+i9o1OORlUh3tozkkPfA9z/NlA==', - 'extra.server_signature_value' : 'LQj+UTJEQqdb/p/c/19yVc63eo+rnedwXKjP6eNNxxijN2cFoOjVMeqT2QTBjyoN7yRWBU2EID+3y2jUYT8mCqmqfyUv1eEbiCfLVlUyQ0X/CY9I5DDb5l6yEjNkuH2xVNNV6R7GFRwyYKAsYzfy+i9o1OORlUh3tozkkPfA9z/NlA==', - 'extra.serverid_raw' : 'SSH-2.0-ARRIS_0.50', - 'extra.serverid_software' : 'ARRIS_0.50', - 'extra.serverid_version' : '2.0', - 'extra.tag' : 'ssh', - 'extra.userauth_methods' : 'publickey, password', - 'feed.name' : 'Accessible SSH', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 11976, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'MARSHALL', - 'source.geolocation.region' : 'TEXAS', - 'source.ip' : '170.10.0.0', - 'source.port' : 22, - 'source.reverse_dns' : '170-10-0-0.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T02:20:37+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ssh', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.algorithm' : 'ssh-rsa', - 'extra.available_ciphers' : 'aes128-cbc, 3des-cbc, aes192-cbc, aes256-cbc', - 'extra.available_compression' : 'none', - 'extra.available_kex' : 'diffie-hellman-group-exchange-sha1, diffie-hellman-group14-sha1, diffie-hellman-group1-sha1', - 'extra.available_mac' : 'hmac-sha1, hmac-sha1-96, hmac-md5, hmac-md5-96', - 'extra.device_sector' : 'enterprise', - 'extra.device_vendor' : 'Cisco', - 'extra.rsa_exponent' : '65537', - 'extra.rsa_length' : '4096', - 'extra.rsa_modulus' : 'yFVwcChoYt+YGm8BzWYugcZbNQRrQ1VWRYcL4U6SSkyoVeE9h5wxRu/hQaWHo3PdsB9Nuln/riRyKZypFUEZ5zlffMyl1uvE8/jp8E/GgUHSyPkGAwu8C8BkX/nDolxAJKTK6djiZnvhsEPe6AXHBMHbto/b3GABUNPngjzX8D63GYcFW9NJLf5qC1UsVkXbAzM0IjQ2X9s3pfhUCAJeXAn2i0gEGtUyF8vEjNdwdG655aXciKrpEEtM1L/zy/+gLH4YC13kAYI7NVyH+qi/mXbULLOQClA7iYK1g3Et58jWUIPwgLfF3SLC57bt2wp/lRgNTv4FBi0tWvRqBnf5UQK5ZjgzbW3bO+Ju4cWgH/4M4NCxSceh4cLm5lQs01xB5feSh2ByqA7wrVDoFJu81LoMVo4bCz30+lH2QsLwmNtUhlWLKBD4k09g4bgBa4jPj0/Nya3rBR4GQ6LG6ltFQotm8wCkgbv76YWqk20nQ6NMYZFvSQm981JFtoHv3vxq48VeHDV0QvV0P12BCFprRf4B0otIvSsHl+LDeUxJAf+Nbw78gzncjyfCbWtCPbwaJQ8CeqnTBzj5TluaFvN8goG5lCTWJGfjIrwAZXOokv9NOqmIiMJJx3s22OX6GHfJAzje2ALLDsAiXBub4iCOdGdTfVbBpFL+bGTK9qfa8vE=', - 'extra.selected_cipher' : 'aes128-cbc', - 'extra.selected_compression' : 'none', - 'extra.selected_kex' : 'diffie-hellman-group14-sha1', - 'extra.selected_mac' : 'hmac-sha1', - 'extra.server_cookie' : 'Z2fOfWsrLlh76Y0bOqa1cw==', - 'extra.server_host_key' : 'AAAAB3NzaC1yc2EAAAADAQABAAACAQDIVXBwKGhi35gabwHNZi6Bxls1BGtDVVZFhwvhTpJKTKhV4T2HnDFG7+FBpYejc92wH026Wf+uJHIpnKkVQRnnOV98zKXW68Tz+OnwT8aBQdLI+QYDC7wLwGRf+cOiXEAkpMrp2OJme+GwQ97oBccEwdu2j9vcYAFQ0+eCPNfwPrcZhwVb00kt/moLVSxWRdsDMzQiNDZf2zel+FQIAl5cCfaLSAQa1TIXy8SM13B0brnlpdyIqukQS0zUv/PL/6AsfhgLXeQBgjs1XIf6qL+ZdtQss5AKUDuJgrWDcS3nyNZQg/CAt8XdIsLntu3bCn+VGA1O/gUGLS1a9GoGd/lRArlmODNtbds74m7hxaAf/gzg0LFJx6HhwubmVCzTXEHl95KHYHKoDvCtUOgUm7zUugxWjhsLPfT6UfZCwvCY21SGVYsoEPiTT2DhuAFriM+PT83JresFHgZDosbqW0VCi2bzAKSBu/vphaqTbSdDo0xhkW9JCb3zUkW2ge/e/GrjxV4cNXRC9XQ/XYEIWmtF/gHSi0i9KweX4sN5TEkB/41vDvyDOdyPJ8Jta0I9vBolDwJ6qdMHOPlOW5oW83yCgbmUJNYkZ+MivABlc6iS/006qYiIwknHezbY5foYd8kDON7YAssOwCJcG5viII50Z1N9VsGkUv5sZMr2p9ry8Q==', - 'extra.server_host_key_sha256' : '06ff3cce443ed832927576d982b69d5a526d0e63334c72e87201deda61679406', - 'extra.server_signature_raw' : 'AAAAB3NzaC1yc2EAAAIAlrzL2DY9fVvwYg6CgB75uf2s8CLo+rL8Mp9tU1Ja3sDfBzj9QJjVDykupiy8s3usHfxMrHS2v3DhTiZjz/b5K6tVTgUBTXL94JfM4lwB+3EbLggPzKnlm1jQgnnU9c+tb7RX3IhBqU9Yj1gqxhErv9NFotgajQOOLgY0Ua5C0Ee+AIaMlLaNZe3LTejMsNUZMN5tl+sEmtutMHkGQsmjJxiJ3feF+Pys0I2+ojiiAfzqlMYar/5xOPl4Dj+HO+h91xVQ1/8nQRBc082fM7+ZJtDbRLtt4G8srlB5gew26jqfVASc/ui5gx4+BR9DG9VH8w+rJWBGfhOAaWqLFE2M3YuEWkjEmQMR1SQK1WFQ/oNiWJO2K5L3rk2LcAmyR6nQMtClVxYZ7CQOwa3uFL+JNXp9AhiiAtVaqhrEK81NJrJNh/+egTBl5STphxIShXd4KI9wyvkGlCIvNIMO94iXPVaWUXXbsGnU03+dsUkBzGf0eJ4DePInCk/RtunlSmOsjGld+rpS9g0VRxPrzbQRWuhpkgpV+CldyrI3C/rOxJRs2vSAKXocRsGwhqEKseAJzHXmiZ5ncsaGKoeB5lUkWLwcKjyok2tHVCDlzDUpE4aA/JHNEhT48det9RqtjC71yz8m0PeK2ySI/I+Qb7eBgevgduBmt+OUxgvfKi2UB6s=', - 'extra.server_signature_value' : 'lrzL2DY9fVvwYg6CgB75uf2s8CLo+rL8Mp9tU1Ja3sDfBzj9QJjVDykupiy8s3usHfxMrHS2v3DhTiZjz/b5K6tVTgUBTXL94JfM4lwB+3EbLggPzKnlm1jQgnnU9c+tb7RX3IhBqU9Yj1gqxhErv9NFotgajQOOLgY0Ua5C0Ee+AIaMlLaNZe3LTejMsNUZMN5tl+sEmtutMHkGQsmjJxiJ3feF+Pys0I2+ojiiAfzqlMYar/5xOPl4Dj+HO+h91xVQ1/8nQRBc082fM7+ZJtDbRLtt4G8srlB5gew26jqfVASc/ui5gx4+BR9DG9VH8w+rJWBGfhOAaWqLFE2M3YuEWkjEmQMR1SQK1WFQ/oNiWJO2K5L3rk2LcAmyR6nQMtClVxYZ7CQOwa3uFL+JNXp9AhiiAtVaqhrEK81NJrJNh/+egTBl5STphxIShXd4KI9wyvkGlCIvNIMO94iXPVaWUXXbsGnU03+dsUkBzGf0eJ4DePInCk/RtunlSmOsjGld+rpS9g0VRxPrzbQRWuhpkgpV+CldyrI3C/rOxJRs2vSAKXocRsGwhqEKseAJzHXmiZ5ncsaGKoeB5lUkWLwcKjyok2tHVCDlzDUpE4aA/JHNEhT48det9RqtjC71yz8m0PeK2ySI/I+Qb7eBgevgduBmt+OUxgvfKi2UB6s=', - 'extra.serverid_raw' : 'SSH-1.99-Cisco-1.25', - 'extra.serverid_software' : 'Cisco-1.25', - 'extra.serverid_version' : '1.99', - 'extra.source.naics' : 517311, - 'extra.tag' : 'ssh', - 'extra.userauth_methods' : 'publickey, keyboard-interactive, password', - 'feed.name' : 'Accessible SSH', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 33363, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'ORLANDO', - 'source.geolocation.region' : 'FLORIDA', - 'source.ip' : '72.17.0.0', - 'source.port' : 22, - 'source.reverse_dns' : '072-017-0-0.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T02:20:37+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl.py deleted file mode 100644 index f96c03e567..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl.py +++ /dev/null @@ -1,218 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ssl.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible SSL', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_ssl-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ssl', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.browser_error' : 'x509: unknown error', - 'extra.browser_trusted' : False, - 'extra.cert_expiration_date' : '2038-01-19 03:14:07', - 'extra.cert_expired' : False, - 'extra.cert_issue_date' : '2014-06-23 09:56:32', - 'extra.cert_length' : 1024, - 'extra.cert_serial_number' : '168CAE', - 'extra.cert_valid' : True, - 'extra.cipher_suite' : 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA', - 'extra.content_length' : 131, - 'extra.content_type' : 'text/html', - 'extra.freak_vulnerable' : False, - 'extra.handshake' : 'TLSv1.2', - 'extra.http_code' : 200, - 'extra.http_date' : '2022-01-10T00:01:44+00:00', - 'extra.http_reason' : 'OK', - 'extra.http_response_type' : 'HTTP/1.1', - 'extra.issuer_common_name' : 'support', - 'extra.issuer_country' : 'US', - 'extra.issuer_email_address' : 'support@fortinet.com', - 'extra.issuer_locality_name' : 'Sunnyvale', - 'extra.issuer_organization_name' : 'Fortinet', - 'extra.issuer_organization_unit_name' : 'Certificate Authority', - 'extra.issuer_state_or_province_name' : 'California', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : '99:45:1F:2E:AE:EB:88:91:27:43:33:79:FA:93:7D:CA', - 'extra.self_signed' : False, - 'extra.server_type' : 'xxxxxxxx-xxxxx', - 'extra.sha1_fingerprint' : '5A:3D:FF:06:F9:E9:25:37:57:F9:09:52:33:A4:85:15:24:2D:88:7F', - 'extra.sha256_fingerprint' : '35:AB:B6:76:2A:3D:17:B2:FB:40:45:1B:FC:0A:99:0A:6E:48:57:F7:30:0A:3B:B1:1A:E6:99:70:5B:7C:32:41', - 'extra.sha512_fingerprint' : '88:7B:16:DB:39:44:0C:47:0E:4A:8F:0B:C5:FB:4D:45:BC:93:5A:00:43:A1:D9:7F:05:1D:86:33:02:F8:FC:57:67:A6:1D:C0:FF:F7:D2:40:D8:9A:21:AE:4E:6D:DC:E7:FF:72:BF:13:CB:EE:A7:5F:CD:83:EA:8A:5E:FB:87:DD', - 'extra.signature_algorithm' : 'sha1WithRSAEncryption', - 'extra.source.naics' : 517311, - 'extra.ssl_poodle' : False, - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'FGT60D4614030700', - 'extra.subject_country' : 'US', - 'extra.subject_email_address' : 'support@fortinet.com', - 'extra.subject_locality_name' : 'Sunnyvale', - 'extra.subject_organization_name' : 'Fortinet', - 'extra.subject_organization_unit_name' : 'FortiGate', - 'extra.subject_state_or_province_name' : 'California', - 'extra.tag' : 'ssl,vpn', - 'feed.name' : 'Accessible SSL', - 'protocol.application': 'https', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 4181, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'MILWAUKEE', - 'source.geolocation.region' : 'WISCONSIN', - 'source.ip' : '96.60.0.0', - 'source.port' : 10443, - 'source.reverse_dns' : '96-60-0-0.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:42+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ssl', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.browser_error' : 'x509: unknown error', - 'extra.browser_trusted' : False, - 'extra.cert_expiration_date' : '2023-02-06 01:01:34', - 'extra.cert_expired' : False, - 'extra.cert_issue_date' : '2022-01-04 01:01:34', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : '36974C4C6B1B3785', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', - 'extra.content_type' : 'text/html; charset=UTF-8', - 'extra.freak_vulnerable' : False, - 'extra.handshake' : 'TLSv1.2', - 'extra.http_code' : 200, - 'extra.http_connection' : 'keep-alive', - 'extra.http_date' : '2022-01-10T00:01:44+00:00', - 'extra.http_reason' : 'OK', - 'extra.http_response_type' : 'HTTP/1.1', - 'extra.issuer_common_name' : '1078-btb-tbi-HungHa-61d39c6d5a7e2', - 'extra.issuer_organization_name' : 'pfSense webConfigurator Self-Signed Certificate', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : '16:93:9A:F4:35:7F:9A:85:45:71:91:C7:7C:80:88:00', - 'extra.self_signed' : True, - 'extra.server_type' : 'nginx', - 'extra.set_cookie' : 'PHPSESSID=e15bdfa5739c36877608eb4cf46cc388; path=/; secure; HttpO', - 'extra.sha1_fingerprint' : 'A9:00:BB:E1:54:4D:56:54:59:F1:B7:EA:F1:1A:D5:36:5C:63:90:8E', - 'extra.sha256_fingerprint' : '38:85:F0:44:1E:AD:84:B8:2F:43:68:BA:AC:EE:17:13:A4:BF:86:1D:48:75:7E:22:FA:08:4C:28:5F:AC:3E:5F', - 'extra.sha512_fingerprint' : 'AE:1B:4F:D1:E4:C0:35:9D:2A:4F:7A:37:B8:7B:11:9D:84:25:23:21:AB:EF:B2:0F:DC:C9:F2:A3:72:28:92:E1:74:72:FA:E1:09:6C:E1:F6:B6:E3:A7:61:1C:58:89:34:D7:06:5C:3D:0A:A7:F6:CC:8A:D6:24:D0:04:4C:03:02', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.source.naics' : 517311, - 'extra.ssl_poodle' : False, - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : '1078-btb-tbi-HungHa-61d39c6d5a7e2', - 'extra.subject_organization_name' : 'pfSense webConfigurator Self-Signed Certificate', - 'extra.tag' : 'ssl', - 'extra.transfer_encoding' : 'chunked', - 'feed.name' : 'Accessible SSL', - 'protocol.application': 'https', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 45899, - 'source.geolocation.cc' : 'VN', - 'source.geolocation.city' : 'THAI BINH', - 'source.geolocation.region' : 'THAI BINH', - 'source.ip' : '113.160.0.0', - 'source.port' : 10443, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:42+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ssl', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.browser_trusted' : True, - 'extra.cert_expiration_date' : '2022-11-06 15:30:28', - 'extra.cert_expired' : False, - 'extra.cert_issue_date' : '2021-10-07 15:30:28', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : '7B388364A24B88E77E5553B5C6748100', - 'extra.cert_valid' : True, - 'extra.cipher_suite' : 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA', - 'extra.content_length' : 131, - 'extra.content_type' : 'text/html', - 'extra.freak_vulnerable' : False, - 'extra.handshake' : 'TLSv1.2', - 'extra.http_code' : 200, - 'extra.http_date' : '2022-01-10T00:01:44+00:00', - 'extra.http_reason' : 'OK', - 'extra.http_response_type' : 'HTTP/1.1', - 'extra.issuer_common_name' : 'Entrust Certification Authority - L1K', - 'extra.issuer_country' : 'US', - 'extra.issuer_organization_name' : 'Entrust, Inc.', - 'extra.issuer_organization_unit_name' : '(c) 2012 Entrust, Inc. - for authorized use only', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'E7:34:BC:92:84:FA:39:DE:E1:46:6C:27:DA:5A:01:F4', - 'extra.self_signed' : False, - 'extra.server_type' : 'xxxxxxxx-xxxxx', - 'extra.sha1_fingerprint' : 'AD:19:B2:1C:CB:88:70:9B:DB:8E:7E:F5:65:50:13:D6:43:6C:BE:6E', - 'extra.sha256_fingerprint' : '9A:64:73:0B:8A:FA:DE:22:D4:6D:5A:C6:C4:6F:D4:A4:2A:28:FA:41:1E:FF:81:DC:D4:D9:00:FD:78:DF:C4:DD', - 'extra.sha512_fingerprint' : '9A:B7:BD:68:7D:F3:E7:C1:B7:D3:F4:2F:01:B6:C4:77:90:A3:2B:1E:C0:89:F5:08:EC:43:87:35:60:36:D4:87:61:AA:B8:A8:B3:8A:E9:F1:04:AA:5B:67:12:FF:63:D5:14:80:77:6E:8F:7D:C3:E2:3A:F3:13:DF:08:43:6C:B0', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.source.naics' : 454110, - 'extra.source.sector' : 'Retail Trade', - 'extra.ssl_poodle' : False, - 'extra.ssl_version' : 2, - 'extra.subject_country' : 'US', - 'extra.subject_locality_name' : 'Hanover', - 'extra.subject_organization_name' : 'Ciena Corporation', - 'extra.subject_state_or_province_name' : 'Maryland', - 'extra.tag' : 'ssl,vpn', - 'extra.validation_level' : 'OV', - 'feed.name' : 'Accessible SSL', - 'protocol.application': 'https', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 14618, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'ASHBURN', - 'source.geolocation.region' : 'VIRGINIA', - 'source.ip' : '34.224.0.0', - 'source.port' : 10443, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:42+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_freak.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_freak.py deleted file mode 100644 index 42221bda2b..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_freak.py +++ /dev/null @@ -1,136 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ssl_freak.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'SSL FREAK Vulnerable Servers', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ssl_freak-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'SSL FREAK Vulnerable Servers', - 'protocol.application': 'https', - "classification.identifier": "ssl-freak", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.browser_error": "x509: unknown error", - "extra.browser_trusted": False, - "extra.cert_expiration_date": "2032-05-05 00:01:19", - "extra.cert_expired": False, - "extra.cert_issue_date": "2012-05-10 00:01:19", - "extra.cert_length": 1024, - "extra.cert_serial_number": "4FAB054F", - "extra.cert_valid": True, - "extra.cipher_suite": "TLS_RSA_WITH_RC4_128_SHA", - "extra.content_type": "text/html", - "extra.freak_cipher_suite": "TLS_RSA_EXPORT_WITH_RC4_40_MD5", - "extra.freak_vulnerable": True, - "extra.handshake": "TLSv1.0", - "extra.http_code": 200, - "extra.http_date": "2018-04-23T13:25:26+00:00", - "extra.http_reason": "OK", - "extra.http_response_type": "HTTP/1.1", - "extra.issuer_common_name": "usg50_B0B2DC2FA69D", - "extra.key_algorithm": "rsaEncryption", - "extra.md5_fingerprint": "1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE", - "extra.self_signed": True, - "extra.sha1_fingerprint": "14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2", - "extra.sha256_fingerprint": "57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1", - "extra.sha512_fingerprint": "E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87", - "extra.signature_algorithm": "sha1WithRSAEncryption", - "extra.subject_common_name": "usg50_B0B2DC2FA69D", - "extra.tag": "ssl-freak", - "extra.transfer_encoding": "chunked", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 8447, - "source.geolocation.cc": "AT", - "source.geolocation.city": "VIENNA", - "source.geolocation.region": "WIEN", - "source.ip": "198.51.100.232", - "source.port": 443, - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2018-04-23T13:25:21+00:00" - }, - {'__type': 'Event', - 'feed.name': 'SSL FREAK Vulnerable Servers', - 'protocol.application': 'https', - "classification.identifier": "ssl-freak", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.browser_error": "x509: unknown error", - "extra.browser_trusted": False, - "extra.cert_expiration_date": "2029-12-27 00:00:53", - "extra.cert_expired": False, - "extra.cert_issue_date": "2010-01-01 00:00:53", - "extra.cert_length": 1024, - "extra.cert_serial_number": "4B3D3B35", - "extra.cert_valid": True, - "extra.cipher_suite": "TLS_RSA_WITH_RC4_128_SHA", - "extra.content_type": "text/html", - "extra.freak_cipher_suite": "TLS_RSA_EXPORT_WITH_RC4_40_MD5", - "extra.freak_vulnerable": True, - "extra.handshake": "TLSv1.0", - "extra.http_code": 200, - "extra.http_date": "2018-04-23T13:25:29+00:00", - "extra.http_reason": "OK", - "extra.http_response_type": "HTTP/1.1", - "extra.issuer_common_name": "usg20w_C86C870287EC", - "extra.key_algorithm": "rsaEncryption", - "extra.md5_fingerprint": "1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE", - "extra.self_signed": True, - "extra.sha1_fingerprint": "14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2", - "extra.sha256_fingerprint": "57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1", - "extra.sha512_fingerprint": "E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87", - "extra.signature_algorithm": "sha1WithRSAEncryption", - "extra.subject_common_name": "usg20w_C86C870287EC", - "extra.tag": "ssl-freak", - "extra.transfer_encoding": "chunked", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 12577, - "source.geolocation.cc": "AT", - "source.geolocation.city": "BADEN", - "source.geolocation.region": "NIEDEROSTERREICH", - "source.ip": "198.51.100.224", - "source.port": 443, - "source.reverse_dns": "198-51-100-224.example.net", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2018-04-23T13:25:26+00:00" - }] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_poodle.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_poodle.py deleted file mode 100644 index 41535e67a4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_poodle.py +++ /dev/null @@ -1,91 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_ssl_poodle.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'SSL POODLE Vulnerable Servers', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ssl_poodle-test-geo.csv", - } -EVENTS = [{'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'ssl-poodle', - 'extra.browser_error': 'x509: unknown error', - 'extra.browser_trusted': False, - 'extra.cert_expiration_date': '2034-06-20 00:00:42', - 'extra.cert_expired': False, - 'extra.cert_issue_date': '2014-06-25 00:00:42', - 'extra.cert_length': 1024, - 'extra.cert_serial_number': '53AA112A', - 'extra.cert_valid': True, - 'extra.cipher_suite': 'TLS_RSA_WITH_RC4_128_SHA', - 'extra.content_type': 'text/html', - 'extra.handshake': 'TLSv1.0', - 'extra.http_code': 200, - 'extra.http_date': '2018-08-08T00:51:44+00:00', - 'extra.http_reason': 'OK', - 'extra.http_response_type': 'HTTP/1.1', - 'extra.issuer_common_name': 'usg20_107BEF394BA5', - 'extra.key_algorithm': 'rsaEncryption', - 'extra.md5_fingerprint': '33:E3:61:3F:5D:AA:96:99:38:A5:D6:F1:11:C7:ED:FC', - 'extra.self_signed': True, - 'extra.sha1_fingerprint': '04:FA:DE:1D:BD:4A:05:25:61:FB:F3:D6:64:74:66:44:01:22:D7:C3', - 'extra.sha256_fingerprint': '16:25:9F:C7:A1:8D:64:1F:D9:25:42:BF:87:5C:4F:F3:63:14:97:21:EC:B6:67:10:F2:CA:52:37:C9:FE:49:2E', - 'extra.sha512_fingerprint': '0B:2D:48:8C:4B:55:8B:F3:AB:F8:45:ED:E0:A0:63:F4:84:2F:4C:19:DC:A8:6F:7D:6A:AF:61:D7:98:AA:58:0F:CB:CA:87:D2:C3:0B:C5:DF:49:A7:84:7C:47:58:89:7D:92:B6:7B:98:7D:B1:64:4B:DC:DD:BE:9D:11:2A:D1:AE', - 'extra.signature_algorithm': 'sha1WithRSAEncryption', - 'extra.ssl_poodle': True, - 'extra.ssl_version': 2, - 'extra.subject_common_name': 'usg20_107BEF394BA5', - 'extra.tag': 'ssl-poodle', - 'extra.transfer_encoding': 'chunked', - 'feed.name': 'SSL POODLE Vulnerable Servers', - 'protocol.application': 'https', - 'source.asn': 65540, - 'source.geolocation.cc': 'AT', - 'source.geolocation.city': 'VIENNA', - 'source.geolocation.region': 'WIEN', - 'source.ip': '203.0.113.85', - 'source.port': 8443, - 'source.reverse_dns': 'example.com', - 'time.source': '2018-08-08T00:51:42+00:00', - "time.observation": "2015-01-01T00:00:00+00:00", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - '__type': 'Event', - }, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_stun.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_stun.py deleted file mode 100644 index 7fd5f6ec21..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_stun.py +++ /dev/null @@ -1,146 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_stun.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible-Session-Traversal-Utilities-for-NAT', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_stun-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-stun', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'extra.amplification': 5.4, - 'extra.fingerprint': '0xfaedd06e', - 'extra.magic_cookie': '2112a442', - 'extra.mapped_address': '192.168.0.1', - 'extra.mapped_family': '01', - 'extra.mapped_port': 3243, - 'extra.message_length': 88, - 'extra.message_type': '0101', - 'extra.response_size': 108, - 'extra.software': "Coturn-4.5.1.1 'dan Eider'", - 'extra.tag': 'stun', - 'extra.transaction_id': '000000000000000000000000', - 'extra.xor_mapped_address': '192.168.0.1', - 'extra.xor_mapped_family': '01', - 'extra.xor_mapped_port': 3243, - 'feed.name': 'Accessible-Session-Traversal-Utilities-for-NAT', - 'protocol.application': 'session traversal utilities for nat', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 3478, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-stun', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'extra.amplification': 5.4, - 'extra.fingerprint': '0x21128641', - 'extra.magic_cookie': '2112a442', - 'extra.mapped_address': '51.77.39.195', - 'extra.mapped_family': '01', - 'extra.mapped_port': 45877, - 'extra.message_length': 88, - 'extra.message_type': '0101', - 'extra.response_size': 108, - 'extra.software': "Coturn-4.5.1.1 'dan Eider'", - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'stun', - 'extra.transaction_id': '000000000000000000000000', - 'extra.xor_mapped_address': '192.168.0.2', - 'extra.xor_mapped_family': '01', - 'extra.xor_mapped_port': 45877, - 'feed.name': 'Accessible-Session-Traversal-Utilities-for-NAT', - 'protocol.application': 'session traversal utilities for nat', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 3478, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-stun', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'extra.amplification': 4.8, - 'extra.magic_cookie': '2112a442', - 'extra.mapped_address': '192.168.0.3', - 'extra.mapped_family': '01', - 'extra.mapped_port': 16321, - 'extra.message_length': 76, - 'extra.message_type': '0101', - 'extra.response_size': 96, - 'extra.software': "ApolloProxy-1.20.1.28 'sunflower'", - 'extra.tag': 'stun', - 'extra.transaction_id': '000000000000000000000000', - 'extra.xor_mapped_address': '188.68.240.32', - 'extra.xor_mapped_family': '01', - 'extra.xor_mapped_port': 16321, - 'feed.name': 'Accessible-Session-Traversal-Utilities-for-NAT', - 'protocol.application': 'session traversal utilities for nat', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 3478, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_synfulknock.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_synfulknock.py deleted file mode 100644 index 9b7e1fd3d9..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_synfulknock.py +++ /dev/null @@ -1,117 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_synfulknock.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'SYNful Knock', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_synfulknock-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-synfulknock', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.ack_number' : 791102, - 'extra.raw_packet' : '3cfdfec601e4700f6a9a2000080045000034c3780000f706789442099555b869f7ee0050b20800000000000c123e8012200002aa0000020405b40101040201030305', - 'extra.tag' : 'synfulknock', - 'extra.tcp_flags' : '4608', - 'extra.window_size' : 8192, - 'feed.name' : 'SYNful Knock', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 18885, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'JERSEY CITY', - 'source.geolocation.region' : 'NEW JERSEY', - 'source.ip' : '66.9.0.0', - 'source.port' : 80, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T09:18:23+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-synfulknock', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.ack_number' : 791102, - 'extra.raw_packet' : '90e2baaf0b84700f6a9a200008004500003434100000f2064382d58337d2b8698b720050916200000000000c123e8012200059d50000020405b40101040201030305', - 'extra.tag' : 'synfulknock', - 'extra.tcp_flags' : '4608', - 'extra.window_size' : 8192, - 'feed.name' : 'SYNful Knock', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 35805, - 'source.geolocation.cc' : 'GE', - 'source.geolocation.city' : 'TBILISI', - 'source.geolocation.region' : 'TBILISI', - 'source.ip' : '213.131.0.0', - 'source.port' : 80, - 'source.reverse_dns' : 'host-213-131-55-210-customer.wanex.net', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T09:19:17+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-synfulknock', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.ack_number' : 791102, - 'extra.raw_packet' : '90e2bab9cfd4700f6a9a20000800450000340f1d0000ea068bdad5b2e6914a522f360050eb5200000000000c123e801220001b4a0000020405b40101040201030305', - 'extra.tag' : 'synfulknock', - 'extra.tcp_flags' : '4608', - 'extra.window_size' : 8192, - 'feed.name' : 'SYNful Knock', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 29256, - 'source.geolocation.cc' : 'SY', - 'source.geolocation.city' : 'DAMASCUS', - 'source.geolocation.region' : 'DIMASHQ', - 'source.ip' : '213.178.0.0', - 'source.port' : 80, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T09:27:39+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_telnet.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_telnet.py deleted file mode 100644 index 66408db4c5..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_telnet.py +++ /dev/null @@ -1,87 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_telnet.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible Telnet', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_telnet-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Accessible Telnet', - "classification.identifier": "open-telnet", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.banner": "|MikroTik v6.5|Login:", - "extra.tag": "telnet-alt", - "protocol.application": "telnet", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 20255, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.145", - "source.port": 5678, - "source.reverse_dns": "example.local", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T12:27:34+00:00" - }, - {'__type': 'Event', - 'feed.name': 'Accessible Telnet', - "classification.identifier": "open-telnet", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.banner": "|MikroTik v6.45.3 (stable)|Login:", - "extra.tag": "telnet-alt", - "protocol.application": "telnet", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 20255, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.145", - "source.port": 5678, - "source.reverse_dns": "example.local", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T12:27:40+00:00" - }] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_tftp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_tftp.py deleted file mode 100644 index 3cf3688f97..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_tftp.py +++ /dev/null @@ -1,121 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_tftp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open TFTP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2019-03-25T00:00:00+00:00", - "extra.file_name": "2019-03-25-scan_tftp-test-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-tftp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 1.57, - 'extra.error': 'Not defined', - 'extra.errormessage': 'Get not supported', - 'extra.opcode': '5', - 'extra.size': 22, - 'extra.tag': 'tftp', - 'feed.name': 'Open TFTP', - 'protocol.application': 'tftp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 35067, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-tftp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 1.36, - 'extra.error': 'File not found', - 'extra.errorcode': '1', - 'extra.errormessage': 'File not found', - 'extra.opcode': '5', - 'extra.size': 19, - 'extra.tag': 'tftp', - 'feed.name': 'Open TFTP', - 'protocol.application': 'tftp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 56709, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-tftp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 1.5, - 'extra.error': 'Access violation', - 'extra.errorcode': '2', - 'extra.errormessage': 'Access violation', - 'extra.opcode': '5', - 'extra.size': 21, - 'extra.tag': 'tftp', - 'feed.name': 'Open TFTP', - 'protocol.application': 'tftp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 32785, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ubiquiti.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ubiquiti.py deleted file mode 100644 index 396bff1e33..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ubiquiti.py +++ /dev/null @@ -1,124 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ubiquiti.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open Ubiquiti', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2018-03-04T00:00:00+00:00", - "extra.file_name": "2019-03-25-scan_ubiquiti-test-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'accessible-ubiquiti-discovery-service', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 37.0, - 'extra.essid': 'Kachine-Meta-Lidia-Tereixa', - 'extra.firmwarerev': 'XS5.ar2313.v3.5.4494.091109.1459', - 'extra.mac_address': '00156db98c3a', - 'extra.model': 'NS5', - 'extra.radio_name': 'kachine.meta.lidia.tereixa', - 'extra.response_size': 148, - 'extra.tag': 'ubiquiti,iot', - 'feed.name': 'Open Ubiquiti', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 10001, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-ubiquiti-discovery-service', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 39.0, - 'extra.essid': 'Adana-Mason-Lanikai-Ozaner', - 'extra.firmwarerev': 'XM.ar7240.v5.6.3.28591.151130.1749', - 'extra.mac_address': '00156d7c9188', - 'extra.model': 'LM5', - 'extra.model_full': 'NanoStation Loco M5', - 'extra.radio_name': 'adana.mason.lanikai.ozaner', - 'extra.response_size': 156, - 'extra.tag': 'ubiquiti,iot', - 'feed.name': 'Open Ubiquiti', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 10001, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-ubiquiti-discovery-service', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 36.25, - 'extra.essid': 'Tailynn-Kadija-Noreen-Dinkar', - 'extra.firmwarerev': 'XW.ar934x.v5.6.5.29033.160515.2108', - 'extra.mac_address': '0418d6000fd5', - 'extra.model': 'P2B-400', - 'extra.model_full': 'PowerBeam M2 400', - 'extra.radio_name': 'tailynn.kadija.noreen.dinkar', - 'extra.response_size': 145, - 'extra.tag': 'ubiquiti,iot', - 'feed.name': 'Open Ubiquiti', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 10001, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_vnc.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_vnc.py deleted file mode 100644 index 457ec4425a..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_vnc.py +++ /dev/null @@ -1,86 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_vnc.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible VNC', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_vnc-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Accessible VNC', - "classification.identifier": "open-vnc", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.banner": "RFB 003.889", - "extra.product": "Apple remote desktop vnc", - "protocol.application": "vnc", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 5678, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.53", - "source.port": 5678, - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T14:51:44+00:00" - }, - {'__type': 'Event', - 'feed.name': 'Accessible VNC', - "classification.identifier": "open-vnc", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.banner": "RFB 005.000", - "extra.naics": 517311, - "extra.product": "RealVNC Enterprise v5.3 or later", - "protocol.application": "vnc", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 5678, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.112", - "source.port": 5678, - "source.reverse_dns": "localhost.localdomain", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T14:51:44+00:00"}] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ws_discovery.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ws_discovery.py deleted file mode 100644 index 41ab55e584..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ws_discovery.py +++ /dev/null @@ -1,119 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ws_discovery.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible-WS-Discovery-Service', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_ws_discovery-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-ws-discovery', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 164.83, - 'extra.error': 'Validation constraint violation: SOAP message expected', - 'extra.raw_response': 'c2FtcGxlIHJlc3BvbnNlIGRhdGEK', - 'extra.response_size': 989, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'ws-discovery', - 'feed.name': 'Accessible-WS-Discovery-Service', - 'protocol.application': 'ws-discovery', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 3702, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-ws-discovery', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 183.6, - 'extra.error': 'Validation constraint violation: missing root element', - 'extra.raw_response': 'c2FtcGxlIHJlc3BvbnNlIGRhdGEK', - 'extra.response_size': 918, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'ws-discovery', - 'feed.name': 'Accessible-WS-Discovery-Service', - 'protocol.application': 'ws-discovery', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 3702, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-ws-discovery', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 197.8, - 'extra.error': 'Validation constraint violation: SOAP message expected', - 'extra.raw_response': 'c2FtcGxlIHJlc3BvbnNlIGRhdGEK', - 'extra.response_size': 989, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'ws-discovery', - 'feed.name': 'Accessible-WS-Discovery-Service', - 'protocol.application': 'ws-discovery', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 3702, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_xdmcp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_xdmcp.py deleted file mode 100644 index d17482e715..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_xdmcp.py +++ /dev/null @@ -1,117 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Sebastian Wagner -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_xdmcp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "ShadowServer XDMCP", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_xdmcp-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-xdmcp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 6.29, - 'extra.opcode': 'Willing', - 'extra.reported_hostname': 'node01.example.com', - 'extra.size': 44, - 'extra.status': 'Linux 3.0.101-100-default', - 'extra.tag': 'xdmcp', - 'feed.name': 'ShadowServer XDMCP', - 'protocol.application': 'xdmcp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 177, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-xdmcp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 6.86, - 'extra.opcode': 'Willing', - 'extra.reported_hostname': 'node02.example.com', - 'extra.size': 48, - 'extra.status': 'Linux 2.6.9-103.ELsmp', - 'extra.tag': 'xdmcp', - 'feed.name': 'ShadowServer XDMCP', - 'protocol.application': 'xdmcp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 47074, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-xdmcp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 6.57, - 'extra.opcode': 'Willing', - 'extra.reported_hostname': 'node03.example.com', - 'extra.size': 46, - 'extra.status': '1 user, load: 6,5, 6,6, 6,6', - 'extra.tag': 'xdmcp', - 'feed.name': 'ShadowServer XDMCP', - 'protocol.application': 'xdmcp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 177, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_special.py b/intelmq/tests/bots/parsers/shadowserver/test_special.py deleted file mode 100644 index abad86cacc..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_special.py +++ /dev/null @@ -1,106 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/special.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Special', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-special-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'special', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.status' : 'likely compromised', - 'feed.name' : 'Special', - 'malware.name' : 'cyclops-blink', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'special', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.status' : 'likely compromised', - 'feed.name' : 'Special', - 'malware.name' : 'cyclops-blink', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'special', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.source.sector' : 'Professional, Scientific, and Technical Services', - 'extra.status' : 'likely compromised', - 'feed.name' : 'Special', - 'malware.name' : 'cyclops-blink', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_testdata.py b/intelmq/tests/bots/parsers/shadowserver/test_testdata.py deleted file mode 100644 index 19cbdd7d77..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_testdata.py +++ /dev/null @@ -1,81 +0,0 @@ -# SPDX-FileCopyrightText: 2017 Sebastian Wagner -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import csv -import json -import os -import os.path -import unittest -import pathlib - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot -from intelmq.bots.parsers.shadowserver.parser_json import ShadowserverJSONParserBot - -def csvtojson(csvfile): - datalist = [] - - with open(csvfile) as fop: - reader = csv.DictReader(fop, restval="") - - for row in reader: - datalist.append(row) - - return json.dumps(datalist, indent=4) - -CSVREPORTS = {} -JSONREPORTS = {} -testdata = pathlib.Path(__file__).parent / 'testdata' -for filename in testdata.glob('*.csv'): - EXAMPLE_FILE = filename.read_text() - shortname = filename.stem - CSVREPORTS[shortname] = {"raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": f"2019-01-01-{shortname}-test-test.csv", - } - JSONREPORTS[shortname] = {"raw": utils.base64_encode(csvtojson(filename)), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": f"2019-01-01-{shortname}-test-test.json", - } - - -def generate_feed_function(feedname, reports): - def test_feed(self): - """ Test if no errors happen for feed %s. """ % feedname - self.input_message = reports[feedname] - self.run_bot() - return test_feed - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - -class TestShadowserverJSONParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverJSONParserBot - -for key in CSVREPORTS: - setattr(TestShadowserverParserBot, 'test_feed_%s' % key, generate_feed_function(key, CSVREPORTS)) -for key in JSONREPORTS: - setattr(TestShadowserverJSONParserBot, 'test_feed_%s' % key, generate_feed_function(key, JSONREPORTS)) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv deleted file mode 100644 index cfadcbb2d2..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","hostname","source","reason","asn","geo","region","city","naics","sic","sector","tag" -"2019-09-04 07:00:19","198.123.245.134",host.local,"Alien Vault","Malicious Host AA",5678,"XX","LOCATION","LOCATION",517311,0,0, -"2019-09-04 07:00:19","198.123.245.171",,"Alien Vault","Malicious Host AA",5678,"XX","LOCATION","LOCATION",517311,0,, -"2019-09-04 07:00:19","198.123.245.0/24",,"Alien Vault","Malicious Host AA",5678,"XX","LOCATION","LOCATION",517311,0,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv.license deleted file mode 100644 index 476908eebe..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2020 Thomas Hungenberg -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/botnet_drone.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/botnet_drone.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/botnet_drone.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/caida_ip_spoofer.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/caida_ip_spoofer.csv.license deleted file mode 100644 index 456b03316c..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/caida_ip_spoofer.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2020 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv deleted file mode 100644 index 117dd65607..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","port","hostname","tag","application","asn","geo","region","city","url","http_host","category","system","detected_since","server","redirect_target","naics","sic","sector","cc_url","family" -"2017-01-16 00:43:48","203.0.113.1",80,"example.com","hacked-webserver-stealrat-t1","http",64496,"AT","WIEN","VIENNA","/header.php","example.com","spam","WINNT","2015-05-09 05:51:12","Microsoft-IIS/7.5",,0,0,,, -"2018-04-09 15:43:41","203.0.113.1","80","","phishing","http","64496","AT","STEIERMARK","GRAZ","/","example.com","phishing","","","","","0","0","",, -"2022-02-07 21:52:29","66.249.0.0",,"66-249-0-0.example.com","magecart",,1234,"US","CALIFORNIA","MOUNTAIN VIEW",,,"stealer",,,,,519130,,"Communications, Service Provider, and Hosting Service","https://lolfree.pw/ads.txt", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv.license deleted file mode 100644 index f8f131c2ce..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/darknet.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/darknet.csv.license deleted file mode 100644 index f8f131c2ce..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/darknet.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/ddos_amplification.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/ddos_amplification.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/ddos_amplification.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv deleted file mode 100644 index 22cfdd69e6..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","device_vendor","device_type","device_model" -"2022-01-10 00:01:42","88.84.0.0","tcp",10443,,"ssl,vpn",2116,"NO","TROMS OG FINNMARK","TROMVIK",517311,,,"Fortinet","firewall","FortiGate" -"2022-01-10 00:01:42","170.231.0.0","tcp",10443,,"ssl,vpn",27843,"PE","METROPOLITANA DE LIMA","LIMA",,,,"Fortinet","firewall","FortiGate" -"2022-01-10 00:01:42","96.60.0.0","tcp",10443,"96-60-66-218.example.com","ssl,vpn",4181,"US","WISCONSIN","MILWAUKEE",517311,,,"Fortinet","firewall","FortiGate" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/drone_brute_force.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/drone_brute_force.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/drone_brute_force.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv deleted file mode 100644 index 3114c26b15..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","domain_source","public_source","infection","family","tag","application","version","event_id","dst_network","dst_netmask","attack","duration","attack_src_ip","attack_src_port","domain","domain_transaction_id","gcip","http_method","http_path","http_postdata","http_usessl","ip_header_ack","ip_header_acknum","ip_header_dont_fragment","ip_header_fin","ip_header_identity","ip_header_psh","ip_header_rst","ip_header_seqnum","ip_header_syn","ip_header_tos","ip_header_ttl","ip_header_urg","number_of_connections","packet_length","packet_randomized","http_agent" -"2010-02-10 00:00:00",tcp,192.168.0.1,38055,64512,ZZ,Region,City,node01.example.com,0,,,,,172.16.0.1,443,65534,ZZ,Region,City,node01.example.net,0,"",,,ddos-participant,,,https,,,,,,,,,www.example.com,,,GET,/??=GovpfOoaWYlk,,,,,,,,,,,,,,,,,, -"2010-02-10 00:00:01",udp,192.168.0.2,53,64512,ZZ,Region,City,node02.example.com,0,,,,,172.16.0.2,53,65534,ZZ,Region,City,node02.example.net,0,"",,,ddos-participant,,,dns,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -"2010-02-10 00:00:02",udp,192.168.0.3,53,64512,ZZ,Region,City,node03.example.com,0,,Microsoft,email,Exchange,172.16.0.3,53,65534,ZZ,Region,City,node03.example.net,0,"",,,ddos-participant,,,dns,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv.license deleted file mode 100644 index 9f58c89ef0..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2023 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv deleted file mode 100644 index 17ff15ee6c..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv +++ /dev/null @@ -1,7 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","service","start_time","end_time","client_version","username","password","payload_url","payload_md5" -"2021-03-27 00:00:00","tcp","141.98.1.2",30123,209588,"NL","NOORD-HOLLAND","AMSTERDAM",,,,,,,"162.250.1.2",22,26832,"CA","QUEBEC","MONTREAL",,,,"CAPRICA-EU","ssh-brute-force",,,"ssh",,,,"2021-03-27T00:00:00.521730Z","2021-03-27T00:00:01.710968Z","b'SSH-2.0-Go'",,,, -"2021-03-27 00:00:00","tcp","5.188.3.4",55690,57172,"NL","NOORD-HOLLAND","AMSTERDAM",,518210,,,,,"162.250.3.4",22,26832,"CA","QUEBEC","MONTREAL",,,,"CAPRICA-EU","ssh-brute-force",,,"ssh",,,,"2021-03-27T00:00:00.520927Z","2021-03-27T00:00:01.670993Z","b'SSH-2.0-Go'",,,, -"2021-03-27 00:00:00","tcp","45.14.5.6",38636,44220,"RO","BIHOR","ORADEA",,,,,,,"82.118.5.6",23,204957,"PL","MAZOWIECKIE","WARSAW",,,,"CAPRICA-EU","telnet-brute-force",,,"telnet",,,,"2021-03-27T00:00:00.781774Z","2021-03-27T00:00:00.857244Z",,,,, -"2021-03-27 00:00:00","tcp","5.188.6.7",56385,49453,"NL","NOORD-HOLLAND","AMSTERDAM",,518210,,,,,"102.16.6.7",22,37054,"MG","ANTANANARIVO","ANTANANARIVO",,,"Communications, Service Provider, and Hosting Service","CAPRICA-EU","ssh-brute-force",,,"ssh",,,,"2021-03-27T00:00:00.163870Z","2021-03-27T00:00:02.896640Z","b'SSH-2.0-Go'",,,, -"2021-03-27 00:00:00","tcp","45.14.7.8",35802,44220,"RO","BIHOR","ORADEA",,,,,,,"82.118.7.8",23,204957,"PL","MAZOWIECKIE","WARSAW",,,,"CAPRICA-EU","telnet-brute-force",,,"telnet",,,,"2021-03-27T00:00:00.781272Z","2021-03-27T00:00:00.856606Z",,,,, -"2021-03-27 00:00:00","tcp","5.188.9.10",33289,49453,"NL","NOORD-HOLLAND","AMSTERDAM",,518210,,,,,"60.234.9.10",22,9790,"NZ","WELLINGTON","LOWER HUTT",,,"Communications, Service Provider, and Hosting Service","CAPRICA-EU","ssh-brute-force",,,"ssh",,,,"2021-03-27T00:00:00.044871Z","2021-03-27T00:00:00.077322Z","b'SSH-2.0-Go'",,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv.license deleted file mode 100644 index 8b9580cf15..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv deleted file mode 100644 index dc78c1c1aa..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv +++ /dev/null @@ -1,9 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","count" -"2021-03-07 00:00:00","tcp","61.3.1.2",4717,9829,"IN","KERALA","CHENGANNUR",,518210,,,,,,23,,,,,,,,,"mirai",,"mirai",,,, -"2021-03-07 00:00:00","tcp","211.218.3.4",4405,4766,"KR","GANGWON-DO","PYEONGCHANG-EUP",,517311,,,,,,23,,,,,,,,,"mirai",,"mirai",,,, -"2021-03-07 00:00:00","tcp","45.225.5.6",59777,266915,"BR","BAHIA","VITORIA DA CONQUISTA","static-45-225-x-x.example.net",,,,,,,23,,,,,,,,,"mirai",,"mirai",,,, -"2021-03-07 00:00:00","tcp","125.122.7.8",8460,4134,"CN","ZHEJIANG SHENG","HANGZHOU",,517311,,,,,,23,,,,,,,,,"mirai",,"mirai",,,, -"2021-03-07 00:00:00","tcp","219.77.9.10",21867,4760,"HK","HONG KONG","HONG KONG","n219077092196.example.com",517311,,,,,,5555,,,,,,,,,"mirai",,"mirai",,,, -"2021-03-07 00:00:00","tcp","24.137.11.12",4680,14638,"PR","PUERTO RICO","SAN JUAN","dynamic.libertypr.net",,,,,,,5555,,,,,,,,,"mirai",,"mirai",,,, -"2021-03-07 00:00:00","tcp","119.182.13.14",13175,4837,"CN","SHANDONG SHENG","JINING",,517311,,,,,,23,,,,,,,,,"mirai",,"mirai",,,, -"2021-03-07 00:00:00","tcp","27.198.15.16",56133,4837,"CN","SHANDONG SHENG","JINAN",,517311,,,,,,23,,,,,,,,,"mirai",,"mirai",,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv.license deleted file mode 100644 index f4e16ec677..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Birger Schacht -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv deleted file mode 100644 index f41cb508f7..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","domain_source","public_source","infection","family","tag","application","version","event_id","dst_network","dst_netmask","attack","duration","attack_src_ip","attack_src_port","domain","domain_transaction_id","gcip","http_method","http_path","http_postdata","http_usessl","ip_header_ack","ip_header_acknum","ip_header_dont_fragment","ip_header_fin","ip_header_identity","ip_header_psh","ip_header_rst","ip_header_seqnum","ip_header_syn","ip_header_tos","ip_header_ttl","ip_header_urg","number_of_connections","packet_length","packet_randomized","http_agent" -"2010-02-10 00:00:00",,192.168.0.1,61234,64512,ZZ,Region,City,node01.example.com,0,"Communications, Service Provider, and Hosting Service",,,,172.16.0.1,88,65534,ZZ,Region,City,node01.example.net,0,,,,ddos,mirai,mirai,mirai,,,121.12.110.28/32,32,atk10,30,,,,,,,,,,,,,,,,,,,,,,,1440, -"2010-02-10 00:00:01",,192.168.0.2,61234,64512,ZZ,Region,City,node02.example.com,0,"Communications, Service Provider, and Hosting Service",,,,172.16.0.2,80,65534,ZZ,Region,City,node02.example.net,0,,,,ddos,mirai,mirai,mirai,,,180.97.183.94/32,32,atk10,30,,,,,,,,,,,,,,,,,,,,,,,1440, -"2010-02-10 00:00:02",,192.168.0.3,6379,64512,ZZ,Region,City,node03.example.com,0,"Communications, Service Provider, and Hosting Service",,,,172.16.0.3,,65534,ZZ,Region,City,node03.example.net,0,"Communications, Service Provider, and Hosting Service",,,ddos,mirai,mirai,mirai,,,104.237.138.135/32,32,atk7,10,,,,,,,,,,,,,,,,,,,,,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv deleted file mode 100644 index a7d0bc4f1d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv +++ /dev/null @@ -1,6 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","request","count","bytes","end_time","duration","avg_pps","max_pps" -"2021-03-28 00:00:02",,"107.141.1.2",,7018,"US","CALIFORNIA","VISALIA","107-141-x-x.lightspeed.frsnca.sbcglobal.net",517311,"Communications, Service Provider, and Hosting Service",,,,,389,,,,,,,,"CISPA","ddos-amplification",,,,,,,,,"2021-03-28 00:20:22",,, -"2021-03-28 00:00:02",,"74.59.3.4",,5769,"CA","QUEBEC","CHICOUTIMI","modemcablex-x-59-74.mc.videotron.ca",517311,"Communications, Service Provider, and Hosting Service",,,,,389,,,,,,,,"CISPA","ddos-amplification",,,,,,,,,"2021-03-28 00:13:50",,, -"2021-03-28 00:00:02",,"65.131.5.6",,209,"US","WYOMING","CASPER","65-131-x-x.chyn.qwest.net",517311,"Communications, Service Provider, and Hosting Service",,,,,389,,,,,,,,"CISPA","ddos-amplification",,,,,,,,,,,, -"2021-03-28 00:00:02",,"104.162.7.8",,12271,"US","NEW YORK","KINGSTON","cpe-104-162-x-x.hvc.res.rr.com",517311,"Communications, Service Provider, and Hosting Service",,,,,389,,,,,,,,"CISPA","ddos-amplification",,,,,,,,,,,, -"2021-03-28 00:00:02",,"37.120.178.9.10",,197540,"DE","NIEDERSACHSEN","GIFHORN","v22020111328131649.ultrasrv.de",,,,,,,389,,,,,,,,"CISPA","ddos-amplification",,,,,,,,,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv.license deleted file mode 100644 index 8b9580cf15..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv deleted file mode 100644 index 0e5b1e5e9c..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","protocol","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","device_vendor","device_type","device_model","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","domain_source","public_source","infection","family","tag","application","version","event_id","dst_network","dst_netmask","attack","duration","attack_src_ip","attack_src_port","domain","domain_transaction_id","gcip","http_method","http_path","http_postdata","http_usessl","ip_header_ack","ip_header_acknum","ip_header_dont_fragment","ip_header_fin","ip_header_identity","ip_header_psh","ip_header_rst","ip_header_seqnum","ip_header_syn","ip_header_tos","ip_header_ttl","ip_header_urg","number_of_connections","packet_length","packet_randomized" -"2010-02-10 00:00:00",,172.16.0.1,80,65534,ZZ,Region,City,node01.example.net,0,,,,,192.168.0.1,61234,64512,ZZ,Region,City,node01.example.com,0,"Communications, Service Provider, and Hosting Service",,,ddos,mirai,mirai,mirai,,,115.238.198.85/32,32,atk0,30,,,,,,,,,,,,,,,,,,,,,,,1440, -"2010-02-10 00:00:01",,172.16.0.2,43437,65534,ZZ,Region,City,node02.example.net,0,Information,,,,192.168.0.2,61234,64512,ZZ,Region,City,node02.example.com,0,"Communications, Service Provider, and Hosting Service",,,ddos,mirai,mirai,mirai,,,52.184.50.250/32,32,atk0,30,,,,,,,,,,,,,,,,,,,,,,,1440, -"2010-02-10 00:00:02",,172.16.0.3,80,65534,ZZ,Region,City,node03.example.net,0,,,,,192.168.0.3,61234,64512,ZZ,Region,City,node03.example.com,0,"Communications, Service Provider, and Hosting Service",,,ddos,mirai,mirai,mirai,,,211.99.102.216/32,32,atk10,30,,,,,,,,,,,,,,,,,,,,,,,1440, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv deleted file mode 100644 index d9448bd83d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","pattern","http_url","http_agent","http_request_method","url_scheme","session_tags","vulnerability_enum","vulnerability_id","vulnerability_class","vulnerability_score","vulnerability_severity","vulnerability_version","threat_framework","threat_tactic_id","threat_technique_id","target_vendor","target_product","target_class","file_md5","file_sha256","request_raw","body_raw" -"2021-08-01 00:24:08","tcp","191.23.45.67",36455,1234,"EE","HARJUMAA","TALLINN","191-23-45-67-host.example.com",518210,"Communications, Service Provider, and Hosting Service",,,,"109.87.65.43",80,5678,"UK","WINDSOR AND MAIDENHEAD","MAIDENHEAD",,518210,,"CAPRICA-EU","http-scan",,,,"3.1.3-dev",,"unknown","/js/ueditor/wwwroot/way-board.cgi",,,,,,,,,,,,,,,,,,,"GET /js/ueditor/wwwroot/way-board.cgi HTTP/1.0rnAccept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8rnAccept-Encoding: gzip, deflaternAccept-Language: en-US,en;q=0.5rnConnection: closernDnt: 1rnHost: 109.87.65.43rnOrigin: http://109.87.65.43rnReferer: http://109.87.65.43/rnUser-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.104 Safari/537.36 Core/1.53.3084.400 QQBrowser/9.6.11346.400", -"2021-08-01 05:21:59","tcp","45.67.89.123",58610,12345,"EE","HARJUMAA","TALLINN",,,,,,,"82.41.20.10",8080,23456,"UA","KHARKIVS'KA OBLAST'","KHARKIV",,,,"CAPRICA-EU","http-scan",,,,,,,"/","Mozilla/5.0 (Windows NT 5.1; rv:9.0.1) Gecko/20100101 Firefox/9.0.1","GET","http",,,,,,,,,,,,,,,,"R0VUIC8gSFRUUC8xLjENCkhvc3Q6IDgyLjQxLjIwLjEwOjgwODANCkFjY2VwdDogdGV4dC9odG1sLGFwcGxpY2F0aW9uL3hodG1sK3htbCxhcHBsaWNhdGlvbi94bWw7cT0wLjksKi8qO3E9MC44DQpBY2NlcHQtRW5jb2Rpbmc6IGRlZmxhdGUsIGd6aXAsIGlkZW50aXR5DQpBY2NlcHQtTGFuZ3VhZ2U6IGVuLVVTO3E9MC42LGVuO3E9MC40DQpVc2VyLUFnZW50OiBNb3ppbGxhLzUuMCAoV2luZG93cyBOVCA1LjE7IHJ2OjkuMC4xKSBHZWNrby8yMDEwMDEwMSBGaXJlZm94LzkuMC4xDQoNCg==", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv.license deleted file mode 100644 index c1900637ff..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Mikk Margus Möll -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv deleted file mode 100644 index 174360bbdc..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv +++ /dev/null @@ -1,7 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","network","routedspoof","session","nat" -"2021-03-28 00:42:59","tcp","98.191.250.0",,22898,"US","OKLAHOMA","OKLAHOMA CITY","ip-98.191.250.0.atlinkservices.com",517311,,,,,,,,,,,,,,"caida","ip-spoofer",,,,"ipv4",,"98.191.250.0/24","received",1112907,"True" -"2021-03-28 01:36:22","tcp","191.7.16.0",,262485,"BR","RIO DE JANEIRO","NOVA IGUACU",,,,,,,,,,,,,,,,"caida","ip-spoofer",,,,"ipv4",,"191.7.16.0/24","received",1112914,"False" -"2021-03-28 02:10:58","tcp","202.53.160.0",,23923,"BD","DHAKA","DHAKA",,,,,,,,,,,,,,,,"caida","ip-spoofer",,,,"ipv4",,"202.53.160.0/24","received",1112931,"True" -"2021-03-28 03:41:51","tcp","87.121.75.0",,134697,"AU","QUEENSLAND","BRISBANE",,,,,,,,,,,,,,,,"caida","ip-spoofer",,,,"ipv4",,"87.121.75.0/24","received",1112953,"True" -"2021-03-28 06:07:17","tcp","189.201.194.0",,262944,"MX","COAHUILA","SALTILLO","ip-189-201-194-0.slw.spectro.mx",,,,,,,,,,,,,,,"caida","ip-spoofer",,,,"ipv4",,"189.201.194.0/24","received",1113015,"True" -"2021-03-28 06:59:53","tcp","197.15.48.0",,37671,"TN","TUNIS","TUNIS",,517311,,,,,,,,,,,,,,"caida","ip-spoofer",,,,"ipv4",,"197.15.48.0/24","received",1113035,"True" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv.license deleted file mode 100644 index f4e16ec677..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Birger Schacht -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv deleted file mode 100644 index eb0cbbab95..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv +++ /dev/null @@ -1,7 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id" -"2021-06-07 00:00:00","tcp","190.229.1.2",52955,7303,"AR","BUENOS AIRES","CASEROS",,517311,,,,,"168.63.134.179",16464,8075,"HK","HONG KONG","HONG KONG",,334111,"Information","MSDCU","b68-zeroaccess-2-32bit","zeroaccess","b68-zeroaccess-2-32bit",,, -"2021-06-07 00:00:00","tcp","96.20.3.4",16464,5769,"CA","QUEBEC","LAVAL",,517311,"Communications, Service Provider, and Hosting Service",,,,"52.169.3.4",16464,8075,"IE","DUBLIN","DUBLIN",,334111,"Information","MSDCU","b68-zeroaccess-2-32bit","zeroaccess","b68-zeroaccess-2-32bit",,, -"2021-06-07 00:00:00","tcp","187.222.5.6",55049,8151,"MX","CIUDAD DE MEXICO","MEXICO CITY",,517311,,,,,"168.63.134.179",16464,8075,"HK","HONG KONG","HONG KONG",,334111,"Information","MSDCU","b68-zeroaccess-2-32bit","zeroaccess","b68-zeroaccess-2-32bit",,, -"2021-06-07 00:00:00","tcp","75.84.7.8",64190,20001,"US","CALIFORNIA","NORTH HOLLYWOOD",,517311,"Communications, Service Provider, and Hosting Service",,,,"52.169.7.8",16464,8075,"IE","DUBLIN","DUBLIN",,334111,"Information","MSDCU","b68-zeroaccess-2-32bit","zeroaccess","b68-zeroaccess-2-32bit",,, -"2021-06-07 00:00:00","tcp","24.15.9.10",60373,7922,"US","ILLINOIS","HOMER GLEN",,517311,"Communications, Service Provider, and Hosting Service",,,,"104.40.6.5",16464,8075,"US","CALIFORNIA","SAN FRANCISCO",,334111,"Information","MSDCU","b68-zeroaccess-2-32bit","zeroaccess","b68-zeroaccess-2-32bit",,, -"2021-06-07 00:00:00","tcp","124.101.11.12",50386,4713,"JP","FUKUOKA","FUKUOKA",,517311,"Communications, Service Provider, and Hosting Service",,,,"23.99.101.165",16465,8075,"HK","HONG KONG","HONG KONG",,334111,"Information","MSDCU","b68-zeroaccess-2-64bit","zeroaccess","b68-zeroaccess-2-64bit",, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv.license deleted file mode 100644 index f4e16ec677..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Birger Schacht -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv deleted file mode 100644 index c56d1f218b..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv +++ /dev/null @@ -1,6 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","http_url","http_host","http_agent","forwarded_by","ssl_cipher","http_referer" -"2021-06-07 00:00:00","tcp","31.206.1.2",49245,8386,"TR","ANTALYA","KEPEZ",,,,,,,"40.121.206.97",80,8075,"US","VIRGINIA","ASHBURN",,334111,"Information","MSDCU","necurs","necurs","necurs",,,,"/locator.php","40.121.206.97",,,, -"2021-06-07 00:00:00","tcp","177.140.3.4",35919,28573,"BR","SAO PAULO","SAO PAULO",,517312,,,,,"204.95.99.204",443,8075,"US","WASHINGTON","REDMOND",,334111,"Information","MSDCU","caphaw","caphaw","caphaw",,,,"/index.php","3fo8jrthz3y.rgk.cc","Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.0.6103)",,,"null" -"2021-06-07 00:00:01","tcp","180.190.5.6",49264,132199,"PH","CEBU","MANDAUE",,517311,,,,,"40.121.206.97",80,8075,"US","VIRGINIA","ASHBURN",,334111,"Information","MSDCU","necurs","necurs","necurs",,,,"/locator.php","40.121.206.97",,,, -"2021-06-07 00:00:01","tcp","197.157.7.8",55307,37129,"KE","NAIROBI CITY","NAIROBI",,,,,,,"40.121.206.97",80,8075,"US","VIRGINIA","ASHBURN",,334111,"Information","MSDCU","necurs","necurs","necurs",,,,"/news/stream.php","40.121.206.97",,,, -"2021-06-07 00:00:01","tcp","174.114.9.10",59000,812,"CA","ONTARIO","OTTAWA",,517311,"Communications, Service Provider, and Hosting Service",,,,"40.121.206.97",80,8075,"US","VIRGINIA","ASHBURN",,334111,"Information","MSDCU","necurs","necurs","necurs",,,,"/locator.php","40.121.206.97",,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv.license deleted file mode 100644 index f4e16ec677..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Birger Schacht -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv deleted file mode 100644 index c5126c843a..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id" -"2021-03-04 00:00:00","tcp","190.113.1.2",17409,12252,"PE","METROPOLITANA DE LIMA","LIMA",,,,,,,"178.162.1.2",4455,28753,"DE","HESSEN","FRANKFURT AM MAIN",,518210,"Communications, Service Provider, and Hosting Service","eset","victorygate.b","victorygate.b",,,, -"2021-03-04 00:00:00","tcp","35.205.9.10",44696,15169,"BE","BRUXELLES-CAPITALE","BRUSSELS","x.x.205.35.bc.googleusercontent.com",519130,"Communications, Service Provider, and Hosting Service",,,,"148.81.9.10",80,1887,"PL","MAZOWIECKIE","WARSAW",,,,,"virut","virut",,,, -"2021-03-04 00:00:00","tcp","35.197.11.12",36968,15169,"US","OREGON","THE DALLES","x.x.197.35.bc.googleusercontent.com",519130,"Communications, Service Provider, and Hosting Service",,,,"148.81.111.11.12",80,1887,"PL","MAZOWIECKIE","WARSAW",,,,,"virut","virut",,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv.license deleted file mode 100644 index f4e16ec677..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Birger Schacht -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv deleted file mode 100644 index 3e85690d85..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","infection","family","tag","query_type","query","count" -"2022-01-06 00:00:02","udp","217.110.0.0",29614,8220,"DE","HESSEN","FRANKFURT AM MAIN",,518210,"Communications, Service Provider, and Hosting Service",,,,"calypso","calypso","msexchange","A","YolkIsh.COM",1 -"2022-01-06 00:00:02","udp","209.66.0.0",46189,40934,"US","VIRGINIA","ASHBURN",,518210,,,,,"orcus","orcus","rat","A","verble.rocks",1 -"2022-01-06 00:00:02","udp","217.110.0.0",3590,8220,"DE","HESSEN","FRANKFURT AM MAIN",,518210,"Communications, Service Provider, and Hosting Service",,,,"calypso","calypso","msexchange","A","RAwFuNS.COM",1 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv.license deleted file mode 100644 index 662bb20b71..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv deleted file mode 100644 index 4514f248ed..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv +++ /dev/null @@ -1,6 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","http_url","http_host","http_agent","forwarded_by","ssl_cipher","http_referer" -"2021-03-04 00:00:00","tcp","103.196.1.2",60902,134707,"PH","NUEVA ECIJA","DEL PILAR",,,,,,,"184.105.1.2",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,,"andromeda","avalanche-andromeda",,,,,"differentia.ru",,,, -"2021-03-04 00:00:00","tcp","5.14.3.4",55002,8708,"RO","CONSTANTA","CONSTANTA",,517311,"Communications, Service Provider, and Hosting Service",,,,"184.105.3.4",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,,"andromeda","avalanche-andromeda",,,,,"differentia.ru",,,, -"2021-03-04 00:00:00","tcp","49.145.5.6",31350,9299,"PH","CEBU","CEBU",,517311,,,,,"184.105.5.6",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,,"andromeda","avalanche-andromeda",,,,,"disorderstatus.ru",,,, -"2021-03-04 00:00:00","tcp","200.44.7.8",28063,8048,"VE","CARABOBO","VALENCIA",,517311,,,,,"184.105.7.8",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,,"andromeda","avalanche-andromeda",,,,,"differentia.ru",,,, -"2021-03-04 00:00:00","tcp","187.189.9.10",45335,17072,"MX","CHIHUAHUA","JUAREZ",,,,,,,"184.105.9.10",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,,"andromeda","avalanche-andromeda",,,,,"differentia.ru",,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv.license deleted file mode 100644 index f4e16ec677..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Birger Schacht -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv deleted file mode 100644 index 23a3cb2b68..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv +++ /dev/null @@ -1,6 +0,0 @@ -"timestamp","protocol","http_referer_ip","http_referer_port","http_referer_asn","http_referer_geo","http_referer_region","http_referer_city","http_referer_hostname","http_referer_naics","http_referer_sector","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","http_url","http_host","http_referer" -"2021-03-04 00:00:02","tcp","178.162.203.211",80,28753,"DE","HESSEN","FRANKFURT AM MAIN","12106.mobapptrack.com",518210,"Communications, Service Provider, and Hosting Service","85.17.31.82",80,60781,"NL","NOORD-HOLLAND","AMSTERDAM",,518210,"Communications, Service Provider, and Hosting Service",,,"kovter","kovter",,,1614816002,"GET /favicon.ico HTTP/1.1","12106.mobapptrack.com","http://12106.mobapptrack.com/click/redirect?feed_id=12106&sub_id=7&q=8A5491983C8FBE7743E2D2C36E45EBC4-18307118D2626C9BD756B3F09D14BB910E381EE4" -"2021-03-04 00:00:11","tcp","59.106.1.2",80,9370,"JP","OSAKA","OSAKA","x.noizm.com",518210,"Communications, Service Provider, and Hosting Service","178.162.1.2",80,28753,"DE","HESSEN","FRANKFURT AM MAIN",,518210,"Communications, Service Provider, and Hosting Service",,,"sunburst","sunburst",,,1614816011,"GET /animalally.com HTTP/1.1","freescanonline.com","http://x.noizm.com/jump.php?u=http://freescanonline.com/animalally.com" -"2021-03-04 00:00:12","tcp","142.250.3.4",80,15169,"US","CALIFORNIA","MOUNTAIN VIEW","x.blogspot.com",519130,"Communications, Service Provider, and Hosting Service","178.162.1.2",80,28753,"DE","HESSEN","FRANKFURT AM MAIN",,518210,"Communications, Service Provider, and Hosting Service",,,"kovter","kovter",,,1614816012,"GET /getjs?r=0.6393021999392658 HTTP/1.1","rxrtb.bid","http://x.blogspot.com/" -"2021-03-04 00:00:13","tcp","34.232.5.6",80,14618,"US","VIRGINIA","ASHBURN","www.example.com",454110,"Retail Trade","5.79.71.225",80,60781,"NL","NOORD-HOLLAND","AMSTERDAM",,518210,"Communications, Service Provider, and Hosting Service",,,"sunburst","sunburst",,,1614816013,"GET /personalationmall.com HTTP/1.1","freescanonline.com","http://www.example.com/teams/default.asp?u=EKL&t=c&s=lacrosse&p=remote&url=http://freescanonline.com/personalationmall.com" -"2021-03-04 00:01:26","tcp","210.172.7.8",80,2516,"JP","HOKKAIDO","SAPPORO","x.communes.jp",517312,"Communications, Service Provider, and Hosting Service","5.79.1.2",80,60781,"NL","NOORD-HOLLAND","AMSTERDAM",,518210,"Communications, Service Provider, and Hosting Service",,,"sunburst","sunburst",,,1614816086,"GET /raftcomply.com HTTP/1.1","freescanonline.com","http://x.communes.jp/?url=http://freescanonline.com/raftcomply.com" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv.license deleted file mode 100644 index c1900637ff..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Mikk Margus Möll -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv deleted file mode 100644 index 016d2f912b..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","http_url","http_host","http_agent","forwarded_by","ssl_cipher","http_referer" -"2022-03-02 09:14:19","tcp","2001:448a:1082:4d9b:7491:bf9e:3d5f:a634",49431,7713,"ID","JAKARTA RAYA","JAKARTA",,,,,,,"2001:470:1:332::fe",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,"boaxxe","boaxxe","3ve",,,,"GET /QKMSvF2hl11j%2fbMkyPbF5EpHYhd6VWTG4u19K3Rt7JGU3lMYRqpq8wPYEuOGKKeidKW3pefVfKSjBnL0cXizZbmuWWu8AQNRqw5g9Ny5vZtiv638XKoWwCLuUOTISTV%2fLcpcS1%2f22NjWqgXkHGISAuyVtafqyCC%2f5cA0eYg9Me8VzAIFDdTArogQOdYhElf2xluhEFPsstGQ%2bwrM4VmKHJpzyjD7Y%2fN%2bQV3wnZNdVkEVk1k2iKBJkotYv3ajgYWr56xxCbY5vE1IpZBRNhhaUDNZo0kJgi%2b6knXZ4m7JHt%2fGtJeP%2bNTxHSUL2ELlTIiT3ENlPYD6FdH6ZBxT1OneW%2f0ih%2fcN7vctb5B5Qwa1ez7ZjN2QxgBYkFDDHHTs42ej5eF2BysWAQDSUr%2fcySyGxcfPveIpfQEdrynGKR6z3OYqkFnP%2bYRDQp2rt1qt0FwCB4L9cg05TQlSSTJVGfPDrtcqjvKY4c9hWwSHtE8jMRpeCYO4Es%2bWgwr5DjzMicmuZo%2f4Ycr16jpN7xlDJdJ8iCFZxbSGgVC7ksVlGE8wlfWPI4KTuX5U5s61eNWPTlAC%2fOGb8grtw%2ffzizoIX9D6ZUMvslGLQIp%2fvNmNQkZy8HhNoV6Lns%2figITP%2fpN0H8h9HjUTl9qn65xFOEVpc0motSy8alcTPtTRKq5Jvc4Ao0x3N%2fvCB1v4Epx7XC0UpFbw8TrYEvAczEfGsGM HTTP/1.1","devps.net","Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET4.0C; .NET4.0E)",,, -"2022-03-02 09:15:10","tcp","2001:448a:1082:4d9b:7491:bf9e:3d5f:a634",49460,7713,"ID","JAKARTA RAYA","JAKARTA",,,,,,,"2001:470:1:332::ef",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,"boaxxe","boaxxe","3ve",,,,"GET /QKMSvF2hl11j%2fbMkyPbF5EpHYhd6VWTG4u19K3Rt7JGU3lMYRqpq8wPYEuOGKKeidKW3pefVfKSjBnL0cXizZbmuWWu8AQNRqw5g9Ny5vZtiv638XKoWwCLuUOTISTV%2fLcpcS1%2f22NjWqgXkHGISAuyVtafqyCC%2f5cA0eYg9Me8VzAIFDdTArogQOdYhElf2xluhEFPsstGQ%2bwrM4VmKHJpzyjD7Y%2fN%2bQV3wnZNdVkEVk1k2iKBJkotYv3ajgYWr56xxCbY5vE1IpZBRNhhaUDNZo0kJgi%2b6knXZ4m7JHt%2fGtJeP%2bNTxHSUL2ELlTIiT3ENlPYD6FdH6ZBxT1OneW%2f0ih%2fcN7vctb5B5Qwa1ez7ZjN2QxgBYkFDDHHTs42ej5eF2BysWAQDSUr%2fcySyGxcfPveIpfQEdrynGKR6z3OYqkFnP%2bYRDQp2rt1qt0FwCB4L9cg05TQlSSTJVGfPDrtcqjvKY4c9hWwSHtE8jMRpeCYO4Es%2bWgwr5DjzMicmuZo%2f4Ycr16jpN7xlDJdJ8iCFZxbSGgVC7ksVlGE8wlfWPI4KTuX5U5s61eNWPTlAC%2fOGb8grtw%2ffzizoIX9D6ZUMvslGLQIp%2fvNmNQkZy8HhNoV6Lns%2figITP%2fpN0H8h9HjUTl9qn65xFOEVpc0motSy8alcTPtTRKq5Jvc4Ao0x3N%2fvCB1v4Epx7XC0UpFbw8TrYEvAczEfGsGM HTTP/1.1","devps.net","Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET4.0C; .NET4.0E)",,, -"2022-03-02 14:15:10","tcp","2603:8080:b20a:dc00:f06e:8304:71f6:27e2",62932,11427,"US","TEXAS","GARLAND",,517311,"Communications, Service Provider, and Hosting Service",,,,"2001:470:1:332::fe",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,"boaxxe","boaxxe","3ve",,,,"GET /WMoUNCvuKGzdqSCeQcadP1%2f0B%2f3bzpOmyKBU85Z25HVOhvDQUPFl%2fk8uOcLewS%2b1BsuHXalRAOIgGOYYs2igj6UX8FkdCAmDewWPvfDhPD45nwd2tx1lLf2IoIfuOtIpGR6bN5Q6hGpSBgfERqCa0ImHcwfcZ2EdO%2fWvg7R8H6SLcTiuUC0I4pzvlWt1CRLgLdIEU1hZ0nnFHIHhchb6D7ITEgBQ2chQDxy5TJMrGjm4Dac6dKl%2ft5uYhRhSjAHkLLtgrJjsqVtVbelTAkt5kdcqLlO09m1SH%2fvtAb%2fOvR2DbhBss7%2f64DG7g6cAnghNA6JrFn1uW7sw%2bnKH8koKQwzUjdSsbrQAvmg4r0KDDW8Diq64gfDzxFWkzCLOYifc%2fwlinXPCl7aJiNCoieDC1U98RNQg%2f5td4SZmJnDQ2%2f96CPbFeSpCez5WD1rCjrxLj1h2cqzIgkydEWACceWP9ztxc4QaObzEcgOGxbRckWC7H2aaLeT8jaYEYdKi1pwEKChSL3YdEt4ZIb2IFrWwzNaXEpQzFXf07f902OEdI9vVA1ZdEOBPG6rAIkzMdebfprfVyhKEWtrCd3Skg3COUFtRQks5jzG1nv4sVGijTfSgyn6xE9Taka668Nycik6nmHy8Huj3oC01j3tee%2f1Z3eI6tV7lgM5d3uFJ84slRGHUCwMfVozOGmZRwNo%2fz%2bA HTTP/1.1","devps.net","Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko",,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv.license deleted file mode 100644 index 662bb20b71..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/hp_http_scan.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/hp_http_scan.csv.license deleted file mode 100644 index f8f131c2ce..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/hp_http_scan.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/hp_ics_scan.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/hp_ics_scan.csv.license deleted file mode 100644 index f8f131c2ce..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/hp_ics_scan.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv deleted file mode 100644 index ccafbab3f1..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","url","host","ip","asn","geo","region","city","naics","sector","tag","source","sha256","application" -"2022-01-07 00:02:07","http://41.86.0.0:50008/Mozi.m","41.86.0.0","41.86.0.0",37203,"LR","MONTSERRADO","MONROVIA",,,"CVE-2016-10372",,"12013662c71da69de977c04cd7021f13a70cf7bed4ca6c82acbc100464d4b0ef","http" -"2022-01-07 00:03:14","http://42.225.0.0:38173/Mozi.m","42.225.0.0","42.225.0.0",4837,"CN","HENAN SHENG","ZHUMADIAN",517311,,"CVE-2018-10562",,,"http" -"2022-01-07 00:10:26","http://211.52.0.0:53029/Mozi.m","211.52.0.0","211.52.0.0",4766,"KR","CHUNGCHEONGNAM-DO","SAGOK-MYEON",517311,,"CVE-2018-10562",,,"http" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/outdated_dnssec_key.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/outdated_dnssec_key.csv.license deleted file mode 100644 index f8f131c2ce..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/outdated_dnssec_key.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv deleted file mode 100644 index 965d763a3c..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","url","host","ip","asn","geo","region","city","naics","sector","source" -"2022-02-01 08:00:07","https://priceless-pare.example.net/Postal-/acec6/","priceless-pare.example.net","172.245.0.0",64512,"US","NEW YORK","BUFFALO",518210,"Communications, Service Provider, and Hosting Service","openphish.com" -"2022-02-01 08:00:07","https://mailyahooattt.example.net/","mailyahooattt.example.net","199.34.0.0",64512,"US","CALIFORNIA","SAN FRANCISCO",,"Professional, Scientific, and Technical Services","openphish.com" -"2022-02-01 08:00:07","https://www.example.net/viewer/vbid-730ec2b1-omsttuer","www.example.net","216.58.0.0",64512,"US","UTAH","DRAPER",519130,"Communications, Service Provider, and Hosting Service","openphish.com" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv deleted file mode 100644 index d5baa730fe..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","http","http_code","http_reason","content_type","connection","proxy_authenticate","via","server","content_length","transfer_encoding","http_date" -"2010-02-10 00:00:00",192.168.0.1,tcp,3128,node01.example.com,http-connect-proxy-closed,64512,ZZ,Region,City,0,0,HTTP/1.1,407,"Proxy Authentication Required",text/html;charset=utf-8,keep-alive,"Basic realm=\"\"Squid proxy-caching web server\"\"",,squid/4.10,3741,,"Wed, 10 Feb 2010 00:00:00 GMT" -"2010-02-10 00:00:01",192.168.0.2,tcp,3128,node02.example.com,http-connect-proxy-closed,64512,ZZ,Region,City,0,0,HTTP/1.1,407,"Proxy Authentication Required",text/html;charset=utf-8,keep-alive,"Basic realm=\"\"00:23:24:43:1c:34\"\"",,,3833,,"Wed, 10 Feb 2010 00:00:01 GMT" -"2010-02-10 00:00:02",192.168.0.3,tcp,3128,node03.example.com,http-connect-proxy-closed,64512,ZZ,Region,City,0,0,HTTP/1.1,407,"Proxy Authentication Required",text/html;charset=utf-8,keep-alive,"Basic realm=\"\"Proxy\"\"",,,179,,"Wed, 10 Feb 2010 00:00:02 GMT" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv deleted file mode 100644 index 4710af9742..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","asn","geo","md5","protocol","port","host","bytes_in","bytes_out" -"2022-01-10 00:00:03","40.119.6.228",8075,"US","b575ce6dcce6502a8431db5610135c25","udp",123,"time.windows.com",0,0 -"2022-01-10 00:00:03","8.252.70.126",3356,"US","c0d947f9a8685b0d9f3efdba966389c2","tcp",80,,0,0 -"2022-01-10 00:00:03","52.109.8.22",8075,"US","c0d947f9a8685b0d9f3efdba966389c2","tcp",443,,0,0 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv deleted file mode 100644 index 697cb6209a..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","md5hash","request","type","response","family","tag","source" -"2022-01-10 00:00:02","b575ce6dcce6502a8431db5610135c25","time.windows.com","A","40.119.6.228",,, -"2022-01-10 00:00:08","807679198a39c80d3ca07e60fd51b581","time.windows.com","A","40.119.6.228",,, -"2022-01-10 00:00:20","d97e973b9bf073bd3a217425259cea26","client-office365-tas.msedge.net","A","13.107.5.88",,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv deleted file mode 100644 index bbfe596a24..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","asn","geo","md5","url","user_agent","host","method" -"2022-01-10 00:01:13","23.196.47.89",20940,"US","37514b54e679a5313334e830ad780ec7","http://www.msftncsi.com/ncsi.txt","Microsoft NCSI","www.msftncsi.com","GET" -"2022-01-10 00:01:28","72.21.81.240",15133,"US","37514b54e679a5313334e830ad780ec7","http://www.download.windowsupdate.com/msdownload/update/v3/static/trustedr/en/authrootstl.cab","Microsoft-CryptoAPI/6.1","www.download.windowsupdate.com","GET" -"2022-01-10 00:08:24","23.56.4.57",20940,"US","e97ea2820c0d79f3f3ca241d4dcd1060","http://crl.microsoft.com/pki/crl/products/MicTimStaPCA_2010-07-01.crl","Microsoft-CryptoAPI/6.1","crl.microsoft.com","GET" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv deleted file mode 100644 index c0ff0bdf1e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","name","model","device","features","device_vendor","device_type","device_model","device_version","device_sector" -"2018-07-26 02:07:16","36.239.124.210","tcp",5555,"36-239-124-210.dynamic-ip.hinet.net","adb",3462,"TW","TAOYUAN COUNTY","TAOYUAN CITY",518210,737415,"hlteuc","SAMSUNG-SM-N900A","hlteatt",,,,,, -"2018-07-26 02:07:16","36.236.108.107","tcp",5555,"36-236-108-107.dynamic-ip.hinet.net","adb",3462,"TW","TAIPEI CITY","TAIPEI",518210,737415,"marlin","Pixel XL","marlin","cmd,shell_v2",,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv deleted file mode 100644 index c5494d4582..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","machine_type","afp_versions","uams","flags","server_name","signature","directory_service","utf8_servername","network_address" -"2019-09-04 05:05:53","198.13.34.22","tcp",548,"host.local","afp",6057,"AA","LOCATION","LOCATION",517311,0,"TimeCapsule8,119","AFP3.3,AFP3.2,AFP3.1","DHCAST128,DHX2,SRP,Recon1","SupportsCopyFile,SupportsChgPwd,SupportsServerMessages,SupportsServerSignature,SupportsTCP/IP,SupportsSrvrNotifications,SupportsReconnect,SupportsOpenDirectory,SupportsUTF8Servername,SupportsUUIDs,SupportsSuperClient","airport-time-capsule-de-jack","4338364e37364442463948350069672d",,"AirPort Time Capsule de jack","198.33.24.165:548,10.0.1.1:548,fe80:0008:0000:0000:6e70:9fff:fed4::548,fe80:0009:0000:0000:6e70:9fff:fed4::548,179.24.24.165 (DNS address)," -"2019-09-04 05:05:56","198.40.27.212","tcp",548,"host.local","afp",6057,"AA","LOCATION","LOCATION",517311,0,"TimeCapsule8,119","AFP3.3,AFP3.2,AFP3.1","DHCAST128,DHX2,SRP,Recon1","SupportsCopyFile,SupportsChgPwd,SupportsServerMessages,SupportsServerSignature,SupportsTCP/IP,SupportsSrvrNotifications,SupportsReconnect,SupportsOpenDirectory,SupportsUTF8Servername,SupportsUUIDs,SupportsSuperClient","time-capsule-del-jack","433836544b303147463948360069672d",,"Time Capsule del Jack","0.0.0.1:548,10.0.1.1:548,198.33.42.1:548,fe80:000b:0000:0000:dea4:caff:feba::548,fe80:000c:0000:0000:dea4:caff:feba::548,fe80:000d:0000:0000:4c7d:ffff:fec7::548,0.0.0.1 (DNS address)," diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv deleted file mode 100644 index 92f078af7b..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","channel","message_length","class","method","version_major","version_minor","capabilities","cluster_name","platform","product","product_version","mechanisms","locales","sector" -"2022-01-10 04:32:13","47.103.0.0","tcp",5672,,"amqp",37963,"CN","SHANGHAI SHI","SHANGHAI",518210,,0,509,10,10,0,9,"publisher_confirms,exchange_exchange_bindings,basic.nack,consumer_cancel_notify,connection.blocked,consumer_priorities,authentication_failure_close,per_consumer_qos","rabbit@iZuf63m0nnq9bwf7lhjxrkZ","Erlang/OTP","RabbitMQ","3.3.5","PLAIN AMQPLAIN","en_US", -"2022-01-10 04:32:13","141.95.0.0","tcp",5672,,"amqp",16276,"DE","SAARLAND","SAARBRUCKEN",518210,,0,509,10,10,0,9,"publisher_confirms,exchange_exchange_bindings,basic.nack,consumer_cancel_notify,connection.blocked,consumer_priorities,authentication_failure_close,per_consumer_qos,direct_reply_to","rabbit@mtk-breizh","Erlang/OTP 24.0.3","RabbitMQ","3.8.19","AMQPLAIN PLAIN","en_US", -"2022-01-10 04:32:13","54.234.0.0","tcp",5672,"ec2-54.234.0.0.compute-1.amazonaws.com","amqp",14618,"US","VIRGINIA","ASHBURN",454110,,0,509,10,10,0,9,"publisher_confirms,exchange_exchange_bindings,basic.nack,consumer_cancel_notify,connection.blocked,consumer_priorities,authentication_failure_close,per_consumer_qos,direct_reply_to","rabbit@1397a0e9629b","Erlang/OTP 24.2","RabbitMQ","3.9.11","PLAIN AMQPLAIN","en_US", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv deleted file mode 100644 index 9c43f8598b..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","machine_name","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,3283,node01.example.com,ard,64512,ZZ,Region,City,0,0,"Macmini (radio)",1006,201.20 -"2010-02-10 00:00:01",192.168.0.2,udp,3283,node02.example.com,ard,64512,ZZ,Region,City,0,0,biuro-rip-org-pl,1006,201.20 -"2010-02-10 00:00:02",192.168.0.3,udp,3283,node03.example.com,ard,64512,ZZ,Region,City,0,0,127.0.0.1,1006,201.20 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv deleted file mode 100644 index 7bd2b20e03..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","size","asn","geo","region","city","naics","sic","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,19,node01.example.com,chargen,,64512,ZZ,Region,City,0,0,,74,74.00 -"2010-02-10 00:00:01",192.168.0.2,udp,19,node02.example.com,chargen,,64512,ZZ,Region,City,0,0,,74,74.00 -"2010-02-10 00:00:02",192.168.0.3,udp,19,node03.example.com,chargen,,64512,ZZ,Region,City,0,0,Government,74,74.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv deleted file mode 100644 index 5182817c11..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic" -"2017-11-18 08:42:45","198.51.100.103","tcp",4786,"198-51-100-103.example.net","cisco-smart-install",8559,"AT","WIEN","VIENNA",0,0 -"2017-11-18 08:47:54","198.51.100.218","tcp",4786,,"cisco-smart-install",35609,"AT","WIEN","VIENNA",0,0 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv deleted file mode 100644 index 6d72dac539..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","response","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,5683,node01.example.com,coap,2,64512,ZZ,Region,City,0,0,",,",43,2.05 -"2010-02-10 00:00:01",192.168.0.2,udp,5683,node02.example.com,coap,2,64512,ZZ,Region,City,0,0,",,,,,,,,,",113,5.38 -"2010-02-10 00:00:02",192.168.0.3,udp,5683,node03.example.com,coap,1,64512,ZZ,Region,City,0,0,"`EsjAy************************************************************|CoAP RFC 7252 |************************************************************|This server is using the Eclipse Californium (Cf) CoAP framework|published under EPL+EDL: http://www.eclipse.org/californium/||(c) 2014, 2015, 2016 Institute for Pervasive Computing, ETH Zurich and others|************************************************************",454,113.50 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv deleted file mode 100644 index f4074f3ed9..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","server_version","couchdb_message","couchdb_version","git_sha","features","vendor","visible_databases","error","error_reason" -"2010-02-10 00:00:00",192.168.0.1,tcp,5984,node01.example.com,couchdb,64512,ZZ,Region,City,0,0,,"CouchDB/1.6.1 (Erlang OTP/18)",Welcome,1.6.1,,,"Ubuntu 16.04",_replicator;_users;test;shops;god,, -"2010-02-10 00:00:01",192.168.0.2,tcp,5984,node02.example.com,couchdb,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service","CouchDB/3.2.1 (Erlang OTP/23)",Welcome,3.2.1,244d428af,"access-ready,partitioned,pluggable-storage-engines,reshard,scheduler","The Apache Software Foundation",,, -"2010-02-10 00:00:02",192.168.0.3,tcp,5984,node03.example.com,couchdb,64512,ZZ,Region,City,0,0,"Retail Trade","CouchDB/3.2.1 (Erlang OTP/20)",Welcome,3.2.1,244d428af,"access-ready,partitioned,pluggable-storage-engines,reshard,scheduler","The Apache Software Foundation",,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv deleted file mode 100644 index 5aebed0500..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","http","http_code","http_reason","content_type","connection","www_authenticate","set_cookie","server","content_length","transfer_encoding","date","sector" -"2019-09-04 10:44:55","198.123.245.142","tcp",30005,,"cwmp",5678,"AA","LOCATION","LOCATION",517311,0,"HTTP/1.1",200,"OK","text/html","keep-alive",,,"DNVRS-Webs",5678,,"Wed, 04 Sep 2019 07:42:37 GMT", -"2019-09-04 11:06:50","198.123.245.162","tcp",5678,"localhost.localdomain","cwmp",5678,"AA","LOCATION","LOCATION",517311,0,"HTTP/1.1",404,"Not Found","text/html",,,,"RomPager/4.07 UPnP/1.0",,"chunked",, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv deleted file mode 100644 index c4bb32e573..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","db2_hostname","servername","size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,523,node01.example.com,db2,64512,ZZ,Region,City,0,0,NOWAK_SERWER,node01.example.com,298,14.90 -"2010-02-10 00:00:01",192.168.0.2,udp,523,node02.example.com,db2,64512,ZZ,Region,City,0,0,SPZOZ-DZIEWIN,node02.example.com,298,14.90 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv deleted file mode 100644 index 25e6f11d0e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","source_port","bytes","amplification","method" -"2010-02-10 00:00:00",192.168.0.1,tcp,80,node01.example.com,ddos-middlebox,64512,ZZ,Region,City,0,0,,49002,99,2,SYN+ACK:PSH -"2010-02-10 00:00:01",192.168.0.2,tcp,80,node02.example.com,ddos-middlebox,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",41200,99,2,SYN+ACK:PSH -"2010-02-10 00:00:02",192.168.0.3,tcp,80,node03.example.com,ddos-middlebox,64512,ZZ,Region,City,0,0,,47492,99,2,SYN+ACK:PSH diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv deleted file mode 100644 index 05b8078835..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv +++ /dev/null @@ -1,101 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","dns_version","asn","geo","region","city","min_amplification","p0f_genre","p0f_detail","naics","sic","sector" -"2018-04-14 00:14:34","198.51.100.179","udp",53,"198-51-100-189.example.net","openresolver","dnsmasq-2.66",25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:14:36","198.51.100.8","udp",53,"198-51-100-111.example.net","openresolver","dnsmasq-2.51",25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:14:34","198.51.100.179","udp",53,"198-51-100-189.example.net","openresolver","dnsmasq-2.66",25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:14:36","198.51.100.8","udp",53,"198-51-100-111.example.net","openresolver","dnsmasq-2.51",25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:14:36","198.51.100.158","udp",53,,"openresolver",,8447,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:14:37","198.51.100.7","udp",53,"198-51-100-7.example.net","openresolver","9.9.4-rpz2.13269.14-P2",13292,"AT","STEIERMARK","EISENERZ","4.6190",,,0,0, -"2018-04-14 00:14:38","198.51.100.167","udp",53,"198-51-100-167.example.net","openresolver","ZyWALL DNS",6830,"AT","KARNTEN","VILLACH","4.6667",,,0,0, -"2018-04-14 00:14:40","198.51.100.10","udp",53,"198-51-100-10.example.net","openresolver",,6830,"AT","WIEN","VIENNA","1.3810",,,518210,737415, -"2018-04-14 00:14:41","198.51.100.191","udp",53,"198-51-100-63.example.net","openresolver",,25255,"AT","TIROL","LIENZ","4.6190",,,0,0, -"2018-04-14 00:14:43","198.51.100.25","udp",53,"198-51-100-187.example.net","openresolver","p.4.0",25255,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:14:54","198.51.100.174","udp",53,"198-51-100-174.example.net","openresolver",,8445,"AT","SALZBURG","SALZBURG","6.4048",,,0,0, -"2018-04-14 00:14:54","198.51.100.181","udp",53,"198-51-100-181.example.net","openresolver","PowerDNS Recursor tele2/sil",8437,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:14:54","198.51.100.80","udp",53,"198-51-100-80.example.net","openresolver",,1901,"AT","WIEN","VIENNA","1.3810",,,518210,737415, -"2018-04-14 00:14:57","198.51.100.43","udp",53,"198-51-100-43.example.net","openresolver","vi2zcnsat10, Customer DNS",6830,"AT","STEIERMARK","GRAZ","1.3810",,,0,0, -"2018-04-14 00:14:58","198.51.100.124","udp",53,"198-51-100-124.example.net","openresolver","dnsmasq-2.47",28919,"AT","TIROL","EIBERG","3.8095",,,0,0, -"2018-04-14 00:15:00","198.51.100.60","udp",53,"198-51-100-60.example.net","openresolver",,24992,"AT","VORARLBERG","DORNBIRN","3.4762",,,0,0, -"2018-04-14 00:15:00","198.51.100.201","udp",53,"198-51-100-201.example.net","openresolver",,1853,"AT","STEIERMARK","GRAZ","4.6190",,,0,0, -"2018-04-14 00:15:01","198.51.100.82","udp",53,"198-51-100-82.example.net","openresolver","9.6-ESV-R7-P2",20811,"AT","TIROL","INNSBRUCK","4.6190",,,0,0, -"2018-04-14 00:15:01","198.51.100.105","udp",53,"198-51-100-105.example.net","openresolver",,6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:15:02","198.51.100.173","udp",53,"198-51-100-173.example.net","openresolver",,8445,"AT","NIEDEROSTERREICH","WALD","1.3810",,,0,0, -"2018-04-14 00:15:03","198.51.100.82","udp",53,"198-51-100-82.example.net","openresolver","PowerDNS Recursor tele2/sil",1257,"AT","NIEDEROSTERREICH","MODLING","1.3810",,,518111,737401, -"2018-04-14 00:15:05","198.51.100.39","udp",53,,"openresolver",,8437,"AT","VORARLBERG","LUSTENAU","1.3810",,,0,0, -"2018-04-14 00:15:09","198.51.100.33","udp",53,,"openresolver","dnsmasq-2.55",8447,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:15:09","198.51.100.248","udp",53,"198-51-100-248.example.net","openresolver",,39912,"AT","NIEDEROSTERREICH","HOLLABRUNN","3.8095",,,0,0, -"2018-04-14 00:15:10","198.51.100.119","udp",53,"198-51-100-172.example.net","openresolver","dnsmasq-2.62",25255,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:15:12","198.51.100.135","udp",53,"198-51-100-135.example.net","openresolver","no access.",43848,"AT","NIEDEROSTERREICH","WIESELBURG","3.8095",,,0,0, -"2018-04-14 00:15:15","198.51.100.64","udp",53,"198-51-100-64.example.net","openresolver",,6830,"AT","VORARLBERG","UBERSAXEN","1.3810",,,0,0, -"2018-04-14 00:15:17","198.51.100.80","udp",53,"198-51-100-80.example.net","openresolver",,42473,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:15:18","198.51.100.60","udp",53,"198-51-100-60.example.net","openresolver","198-51-100-60.example.net",35369,"AT","OBEROSTERREICH","LINZ","1.3810",,,0,0, -"2018-04-14 00:15:21","198.51.100.50","udp",53,,"openresolver","ZyWALL DNS",8447,"AT","STEIERMARK","TAUPLITZ","4.6667",,,0,0, -"2018-04-14 00:15:23","198.51.100.93","udp",53,,"openresolver","Microsoft DNS 6.1.7601 (1DB15D39)",8447,"AT","NIEDEROSTERREICH","SCHWADORF","1.3810",,,0,0, -"2018-04-14 00:15:24","198.51.100.33","udp",53,,"openresolver",,8447,"AT","STEIERMARK","FURSTENFELD","4.6190",,,0,0, -"2018-04-14 00:15:31","198.51.100.45","udp",53,,"openresolver","dnsmasq-2.52",8245,"AT","BURGENLAND","EISENSTADT","1.3810",,,0,0, -"2018-04-14 00:15:34","198.51.100.13","udp",53,"198-51-100-13.example.net","openresolver",,8447,"AT","WIEN","VIENNA","6.4048",,,518210,737415, -"2018-04-14 00:15:36","198.51.100.190","udp",53,,"openresolver",,8447,"AT","BURGENLAND","PINKAFELD","1.3810",,,0,0, -"2018-04-14 00:15:41","198.51.100.104","udp",53,,"openresolver",,21013,"AT","OBEROSTERREICH","LEONDING","2.6667",,,0,0, -"2018-04-14 00:15:42","198.51.100.101","udp",53,"198-51-100-101.example.net","openresolver",,8447,"AT","STEIERMARK","KAINACH BEI VOITSBERG","1.3810",,,0,0, -"2018-04-14 00:15:44","198.51.100.62","udp",53,"198-51-100-62.example.net","openresolver",,1901,"AT","OBEROSTERREICH","GMUNDEN","1.3810",,,518210,737415, -"2018-04-14 00:15:46","198.51.100.186","udp",53,"198-51-100-186.example.net","openresolver",,31239,"AT","WIEN","VIENNA","6.4048",,,0,0, -"2018-04-14 00:15:46","198.51.100.197","udp",53,,"openresolver",,8447,"AT","OBEROSTERREICH","KIRCHDORF AN DER KREMS","4.6190",,,0,0, -"2018-04-14 00:15:49","198.51.100.16","udp",53,,"openresolver",,8447,"AT","OBEROSTERREICH","LAAKIRCHEN","4.6190",,,0,0, -"2018-04-14 00:15:50","198.51.100.62","udp",53,"198-51-100-62.example.net","openresolver",,6830,"AT","NIEDEROSTERREICH","WIENER NEUSTADT","4.6190",,,0,0, -"2018-04-14 00:15:53","198.51.100.7","udp",53,"198-51-100-7.example.net","openresolver",,198950,"AT","TIROL","REUTTE","4.6190",,,518210,737415, -"2018-04-14 00:15:53","198.51.100.177","udp",53,"198-51-100-177.example.net","openresolver","Microsoft DNS 6.1.7601 (1DB1446A)",12605,"AT","OBEROSTERREICH","LINZ","1.3810",,,0,0, -"2018-04-14 00:15:57","198.51.100.47","udp",53,,"openresolver",,8447,"AT","NIEDEROSTERREICH","KOTTINGBRUNN","1.3810",,,0,0, -"2018-04-14 00:15:59","198.51.100.95","udp",53,"198-51-100-67.example.net","openresolver","GNS DNS Version 3",57169,"AT","WIEN","VIENNA","1.3810",,,0,0,"Information Technology" -"2018-04-14 00:16:02","198.51.100.104","udp",53,"198-51-100-104.example.net","openresolver",,6830,"AT","OBEROSTERREICH","BAD WIMSBACH-NEYDHARTING","1.3810",,,0,0, -"2018-04-14 00:16:04","198.51.100.106","udp",53,,"openresolver",,8447,"AT","STEIERMARK","GRAZ","1.3810",,,0,0, -"2018-04-14 00:16:05","198.51.100.204","udp",53,"198-51-100-204.example.net","openresolver",,12605,"AT","OBEROSTERREICH","LINZ","4.6190",,,0,0, -"2018-04-14 00:16:05","198.51.100.111","udp",53,"198-51-100-111.example.net","openresolver",,8447,"AT","OBEROSTERREICH","LINZ","1.3810",,,518210,737415, -"2018-04-14 00:16:06","198.51.100.131","udp",53,"198-51-100-139.example.net","openresolver","p.4.0",25255,"AT","OBEROSTERREICH","TRAUN","1.3810",,,0,0, -"2018-04-14 00:16:10","198.51.100.240","udp",53,"198-51-100-240.example.net","openresolver",,6830,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:16:13","198.51.100.9","udp",53,"198-51-100-42.example.net","openresolver",,13026,"AT","STEIERMARK","LEIBNITZ","6.4048",,,0,0, -"2018-04-14 00:16:15","198.51.100.231","udp",53,"198-51-100-74.example.net","openresolver",,25255,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:16:17","198.51.100.228","udp",53,"198-51-100-227.example.net","openresolver","u.1.0",25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:16:19","198.51.100.152","udp",53,"198-51-100-152.example.net","openresolver",,34694,"AT","TIROL","WORGL","4.6190",,,0,0, -"2018-04-14 00:16:21","198.51.100.88","udp",53,,"openresolver",,8447,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:16:22","198.51.100.97","udp",53,"198-51-100-97.example.net","openresolver",,8447,"AT","TIROL","INNSBRUCK","1.3810",,,518210,737415, -"2018-04-14 00:16:23","198.51.100.208","udp",53,"198-51-100-208.example.net","openresolver","dnsmasq-2.62",8447,"AT","TIROL","OTZTAL-BAHNHOF","1.3810",,,0,0, -"2018-04-14 00:16:33","198.51.100.113","udp",53,"198-51-100-121.example.net","openresolver","dnsmasq-2.62",25255,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:16:35","198.51.100.34","udp",53,"198-51-100-44.example.net","openresolver",,25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:16:37","198.51.100.236","udp",53,,"openresolver",,8447,"AT","NIEDEROSTERREICH","ST. ANDRAE-WOERDERN","4.6190",,,0,0, -"2018-04-14 00:16:40","198.51.100.46","udp",53,"198-51-100-46.example.net","openresolver",,21013,"AT","OBEROSTERREICH","LEONDING","2.6667",,,0,0, -"2018-04-14 00:16:45","198.51.100.72","udp",53,"198-51-100-5.example.net","openresolver",,25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:16:50","198.51.100.179","udp",53,"198-51-100-179.example.net","openresolver",,31125,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:16:50","198.51.100.107","udp",53,"198-51-100-107.example.net","openresolver","dnsmasq-2.66",18845,"AT","WIEN","VIENNA","1.3810",,,0,0,"Information Technology" -"2018-04-14 00:16:51","198.51.100.188","udp",53,,"openresolver","9.9.4-RedHat-9.9.4-51.el7_4.2",49322,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:16:54","198.51.100.232","udp",53,"198-51-100-232.example.net","openresolver",,6830,"AT","SALZBURG","SALZBURG","1.3810",,,0,0, -"2018-04-14 00:16:55","198.51.100.102","udp",53,"198-51-100-102.example.net","openresolver","ZyWALL DNS",6830,"AT","KARNTEN","WERNBERG","3.4762",,,0,0, -"2018-04-14 00:16:59","198.51.100.162","udp",53,"198-51-100-162.example.net","openresolver",,8445,"AT","SALZBURG","SALZBURG","1.3810",,,0,0, -"2018-04-14 00:17:00","198.51.100.110","udp",53,"198-51-100-110.example.net","openresolver",,31543,"AT","TIROL","SOLDEN","4.6190",,,0,0, -"2018-04-14 00:17:02","198.51.100.193","udp",53,"198-51-100-193.example.net","openresolver",,8447,"AT","STEIERMARK","FOHNSDORF","1.3810",,,0,0, -"2018-04-14 00:17:06","198.51.100.45","udp",53,"198-51-100-45.example.net","openresolver",,61201,"AT","KARNTEN","KLAGENFURT AM WORTHERSEE","1.3810",,,0,0, -"2018-04-14 00:17:06","198.51.100.219","udp",53,"198-51-100-219.example.net","openresolver",,8437,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:10","198.51.100.47","udp",53,"198-51-100-47.example.net","openresolver","unsupported query",8412,"AT","NIEDEROSTERREICH","WIENER NEUSTADT","3.8095",,,0,0, -"2018-04-14 00:17:13","198.51.100.87","udp",53,"198-51-100-87.example.net","openresolver","dnsmasq-2.40",6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:16","198.51.100.121","udp",53,"198-51-100-121.example.net","openresolver",,8447,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:20","198.51.100.115","udp",53,,"openresolver",,8447,"AT","TIROL","WAIDRING","1.3810",,,0,0, -"2018-04-14 00:17:22","198.51.100.235","udp",53,,"openresolver",,8447,"AT","OBEROSTERREICH","GRIESKIRCHEN","1.3810",,,0,0, -"2018-04-14 00:17:33","198.51.100.154","udp",53,,"openresolver",,8447,"AT","STEIERMARK","GRAZ","4.6190",,,0,0, -"2018-04-14 00:17:36","198.51.100.36","udp",53,"198-51-100-36.example.net","openresolver","BIND",12605,"AT","OBEROSTERREICH","LINZ","4.6190",,,0,0, -"2018-04-14 00:17:38","198.51.100.100","udp",53,"198-51-100-100.example.net","openresolver","dnsmasq-2.40",6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:41","198.51.100.181","udp",53,"198-51-100-181.example.net","openresolver","Microsoft DNS 6.0.6002 (17724D35)",6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:41","198.51.100.242","udp",53,"198-51-100-242.example.net","openresolver","Microsoft DNS 6.0.6002 (17724D35)",34767,"AT","NIEDEROSTERREICH","WIENER NEUSTADT","3.2857",,,0,0, -"2018-04-14 00:17:42","198.51.100.38","udp",53,,"openresolver","PowerDNS Recursor tele2/sil",8437,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:43","198.51.100.132","udp",53,"198-51-100-132.example.net","openresolver","PowerDNS Recursor tele2/sil",8437,"AT","STEIERMARK","GRAZ","1.3810",,,0,0, -"2018-04-14 00:17:49","198.51.100.166","udp",53,"198-51-100-166.example.net","openresolver","9.8.4-rpz2+rl005.12-P1",13292,"AT","STEIERMARK","KINDBERG","4.6190",,,0,0, -"2018-04-14 00:17:49","198.51.100.212","udp",53,"198-51-100-212.example.net","openresolver","dnsmasq-2.40",6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:51","198.51.100.225","udp",53,,"openresolver",,8220,"AT","WIEN","VIENNA","1.3810",,,518210,737415, -"2018-04-14 00:17:53","198.51.100.161","udp",53,"198-51-100-161.example.net","openresolver",,6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:54","198.51.100.12","udp",53,,"openresolver",,8447,"AT","NIEDEROSTERREICH","LANGENLOIS","1.3810",,,0,0, -"2018-04-14 00:17:55","198.51.100.113","udp",53,"198-51-100-113.example.net","openresolver",,6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:57","198.51.100.175","udp",53,"198-51-100-175.example.net","openresolver","PowerDNS Recursor tele2/sil",1257,"AT","NIEDEROSTERREICH","MODLING","1.3810",,,518111,737401, -"2018-04-14 00:17:59","198.51.100.107","udp",53,"198-51-100-107.example.net","openresolver",,50719,"AT","STEIERMARK","TIESCHEN","3.8095",,,0,0, -"2018-04-14 00:17:59","198.51.100.51","udp",53,"198-51-100-68.example.net","openresolver","dnsmasq-2.66",25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:18:04","198.51.100.131","udp",53,,"openresolver","ZyWALL DNS",8447,"AT","TIROL","OBERPERFUSS","3.4762",,,0,0, -"2018-04-14 00:18:05","198.51.100.138","udp",53,"198-51-100-138.example.net","openresolver","unsupported query",8412,"AT","NIEDEROSTERREICH","WIENER NEUSTADT","3.8095",,,0,0, -"2018-04-14 00:18:06","198.51.100.62","udp",53,"198-51-100-62.example.net","openresolver","viezcnsat13, Customer DNS",6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:18:07","198.51.100.109","udp",53,"198-51-100-109.example.net","openresolver",,1901,"AT","OBEROSTERREICH","LINZ","6.9524",,,518210,737415, -"2018-04-14 00:18:10","198.51.100.205","udp",53,"198-51-100-205.example.net","openresolver",,8437,"AT","WIEN","VIENNA","1.3810",,,0,0, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv deleted file mode 100644 index 535dc4ea8e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","sector","http","http_code","http_reason","content_type","server","date","experimental","api_version","arch","go_version","os","kernel_version","git_commit","min_api_version","build_time","pkg_version" -"2010-02-10 00:00:00",192.168.0.1,tcp,2375,node01.example.com,docker,18.05.0-ce,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",HTTP/1.1,200,OK,"application/json; charset=UTF-8","Docker/18.05.0-ce (linux)","Fri, 06 May 2022 14:06:30 GMT",false,1.37,amd64,go1.9.5,linux,3.10.0-514.26.2.el7.x86_64,f150324,1.12,2018-05-09T22:18:36.000000000+00:00, -"2010-02-10 00:00:01",192.168.0.2,tcp,2375,node02.example.com,docker,1.13.1,64512,ZZ,Region,City,0,0,,HTTP/1.1,200,OK,application/json,"Docker/1.13.1 (linux)","Fri, 06 May 2022 14:08:07 GMT",false,1.26,amd64,go1.10.3,linux,3.10.0-693.2.2.el7.x86_64,7d71120/1.13.1,1.12,2022-03-02T15:25:43.414574467+00:00,docker-1.13.1-209.git7d71120.el7.centos.x86_64 -"2010-02-10 00:00:02",192.168.0.3,tcp,2375,node03.example.com,docker,18.05.0-ce,64512,ZZ,Region,City,0,0,,HTTP/1.1,200,OK,"application/json; charset=UTF-8","Docker/18.05.0-ce (linux)","Fri, 06 May 2022 14:08:06 GMT",false,1.37,amd64,go1.9.5,linux,3.10.0-514.26.2.el7.x86_64,f150324,1.12,2018-05-09T22:18:36.000000000+00:00, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv deleted file mode 100644 index 60c7119733..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","device_vendor","device_type","device_model","device_version","device_id","device_serial","machine_name","manufacturer","method","http_port","internal_port","video_input_channels","alarm_input_channels","video_output_channels","alarm_output_channels","remote_video_input_channels","mac_address","ipv4_address","ipv4_gateway","ipv4_subnet_mask","ipv4_dhcp_enable","ipv6_address","ipv6_link_local","ipv6_gateway","ipv6_dhcp_enable","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,37810,node01.example.com,dvrdhcpdiscover,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",General,IPC,BCS-TIP3401IR-E-V,2.800.106F004.0.R,,6J0E022PAG35073,6J0E022PAG35073,General,client.notifyDevInfo,80,37777,1,0,0,0,0,38:c4:e8:03:b3:e2,192.168.0.1,192.168.0.240,255.255.255.0,0,fd09:4ab5:dae9:b078::1,fe80::3ac4:e8ff:fe03:b3e2/64,fd09:4ab5:dae9:b078::ff,0,794,794.00 -"2010-02-10 00:00:01",192.168.0.2,udp,37810,node02.example.com,dvrdhcpdiscover,64512,ZZ,Region,City,0,0,,Private,HCVR,HCVR,3.210.1.4,,2K0488CPAGS0ND6,HCVR,Private,client.notifyDevInfo,80,37777,3,0,0,0,9,3c:ef:8c:18:a5:07,192.168.0.2,192.168.0.240,255.255.255.0,0,fd09:4ab5:dae9:b078::2,fe80::3eef:8cff:fe18:a507/64,fd09:4ab5:dae9:b078::ff,,761,761.00 -"2010-02-10 00:00:02",192.168.0.3,udp,37810,node03.example.com,dvrdhcpdiscover,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",General,HCVR,BCS-XVR0401-IV,4.000.0000002.11,,5L034FAPAZA0E30,XVR,General,client.notifyDevInfo,80,37777,4,0,0,0,0,38:c4:e8:02:74:da,192.168.0.3,192.168.0.240,255.255.255.0,0,fd09:4ab5:dae9:b078::3,fe80::3ac4:e8ff:fe02:74da/64,fd09:4ab5:dae9:b078::ff,,711,711.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv deleted file mode 100644 index c681a8595d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","ok","name","cluster_name","http_code","build_hash","build_timestamp","build_snapshot","lucene_version","tagline","sector" -"2010-02-10 00:00:00",192.168.0.1,tcp,9200,node01.example.com,elasticsearch,2.3.5,64512,ZZ,Region,City,0,0,,"Red Skull",elasticsearch,,90f439ff60a3c0f497f91663701e64ccd01edbb4,2016-07-27T10:36:52Z,false,5.5.0,"You Know, for Search","Communications, Service Provider, and Hosting Service" -"2010-02-10 00:00:01",192.168.0.2,tcp,9200,node02.example.com,elasticsearch,7.17.0,64512,ZZ,Region,City,0,0,,allinonepod,docker-cluster,,bee86328705acaa9a6daede7140defd4d9ec56bd,,false,8.11.1,"You Know, for Search","Communications, Service Provider, and Hosting Service" -"2010-02-10 00:00:02",192.168.0.3,tcp,9200,node03.example.com,elasticsearch,7.15.0,64512,ZZ,Region,City,0,0,,f547c2952610,docker-cluster,,79d65f6e357953a5b3cbcc5e2c7c21073d89aa29,,false,8.9.0,"You Know, for Search","Communications, Service Provider, and Hosting Service" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv deleted file mode 100644 index 4e375a9b42..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv +++ /dev/null @@ -1,8 +0,0 @@ -"timestamp","ip","port","hostname","tag","asn","geo","region","city","naics","sic","sector","version","servername","url" -"2021-05-14 00:11:30","12.237.1.2",443,"afs-exch-cas2.xxx.com","exchange;cve-2021-26855",7018,"US","CALIFORNIA","TURLOCK",517311,,"Communications, Service Provider, and Hosting Service","15.2.721","AFS-EXCH2019", -"2021-05-14 00:11:37","98.153.3.4",443,"rrcs-98-153-x-x.west.biz.rr.com","exchange;webshell",20001,"US","CALIFORNIA","LOS ANGELES",517311,,"Communications, Service Provider, and Hosting Service","15.0.847","SSAMAIL", -"2021-05-14 00:11:38","206.210.5.6",443,"webmail.xxx.com","exchange;webshell",17054,"US","PENNSYLVANIA","PITTSBURGH",518210,,,"15.0.1178","OMNYXEXCH02", -"2021-05-14 00:11:38","12.33.7.8",443,"mail.xxx.org","exchange;cve-2021-26855",7018,"US","ARKANSAS","LITTLE ROCK",921120,,"Communications, Service Provider, and Hosting Service","15.1.2176","MHASVR02", -"2021-05-14 00:11:38","41.204.9.10",443,"mail.xxx.mg","exchange;cve-2021-26855",21042,"MG","ANTANANARIVO","ANTANANARIVO",,,,,"SABMHQE0232", -"2021-05-14 00:11:38","62.33.11.12",443,,"exchange;cve-2021-26855",20485,"RU","ALTAYSKIY KRAY","BARNAUL",,,,"15.2.659","PV-SRV04", -"2021-05-14 00:11:43","199.33.13.14",443,"mail.xxx.tv","exchange;cve-2021-26855",26481,"US","CALIFORNIA","LOS ANGELES",,,,"15.1.1779","MAIL", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv.license deleted file mode 100644 index f4e16ec677..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Birger Schacht -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv deleted file mode 100644 index 912e73d841..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","banner","handshake","cipher_suite","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serial_number","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","cert_valid","self_signed","cert_expired","validation_level","auth_tls_response","auth_ssl_response","tlsv13_support","tlsv13_cipher","jarm","device_vendor","device_type","device_model","device_version","device_sector" -"2019-03-06 06:37:00","61.126.3.70","tcp",21,"arcus-net.co.jp","ftp",4713,"JP","TOKYO","TOKYO",517311,737401,"220 FTP Server ready.|","TLSv1.2","TLS_RSA_WITH_AES_128_CBC_SHA",2048,"*.bizmw.com","GlobalSign Organization Validation CA - SHA256 - G2","Jan 14 08:04:50 2015 GMT","Jan 14 08:04:50 2020 GMT","D9:98:3F:2E:F9:D1:BE:9A:10:1E:DE:51:2C:C1:DF:01:18:0A:20:65","1121DC7421AB7924C3B1D396AEA3707E9E29",2,"sha256WithRSAEncryption","rsaEncryption","NTT Communications Corporation",,"JP","Tokyo","Minato-ku",,,,,,,,"GlobalSign nv-sa",,"BE",,,,,,,,,,"27:4A:8A:3A:A7:DF:82:D0:43:03:0E:6F:48:30:30:C9:24:77:11:1A:08:EF:F7:B9:74:0C:CE:40:87:03:D2:51","E5:93:8B:72:84:0F:35:52:8E:7A:6C:E3:EF:36:90:4C:F2:86:A7:4D:B2:DD:C0:C6:23:83:18:EF:DD:86:34:92:91:57:22:29:75:45:71:8B:3A:CD:F1:27:A9:CA:5F:70:5E:AC:15:A5:E6:63:FD:6F:BB:C5:E2:45:99:73:E9:E6","D1:A7:BC:96:78:1D:16:D0:24:A8:62:7C:3A:95:5A:4A","N","N","N","OV","234 AUTH TLS successful",,,,,,,,, -"2019-03-06 06:37:00","62.48.156.65","tcp",21,"dial-62-48-156-65.ptprime.net","ftp",15525,"PT","LISBOA","FRIELAS",0,0,"220-================================================================| PT Empresas| Acesso Reservado| Acesso nao autorizado punido por lei: 109/91; 67/98| ----------------------------------------------------------------| HENNES & MAURITZ LDA - 149093| SITE: PT303 - Cascais Shopping| MORADA: | NIR: EWS1822940| ================================================================|220 FTP server ready, 1 active clients of 4 simultaneous clients allowed.|",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"500 Syntax error, command unrecognized.","500 Syntax error, command unrecognized.",,,,,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv deleted file mode 100644 index 26f8ccbcf0..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","port","hostname","version","asn","geo","region","city","naics","sic","server_type","clusterid","total_disk","used_disk","free_disk","livenodes","namenodeaddress","volumeinfo" -"2017-09-13 02:06:05","199.116.235.200",50070,,"2.7.3, rbaa91f7c6bc9cb92be5982de4719c1c8af91ccff",15296,"CA","ALBERTA","CALGARY",0,0,"namenode","CID-64471a53-60cb-4302-9832-92f321f111fe",41567956992,53248,25160089600,"edmonton:50010",, -"2017-09-13 02:07:48","104.43.235.92",50075,,"2.7.1.2.4.0.0-169",8075,"US","IOWA","DES MOINES",334111,357101,"datanode","CID-771bae52-9e4f-4ec4-bc1a-c867585751f0",,,,,"sandbox.hortonworks.com","/hadoop/hdfs/data/current" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv.license deleted file mode 100644 index f8f131c2ce..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv deleted file mode 100644 index a7e3eb7074..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","http","http_code","http_reason","content_type","connection","www_authenticate","set_cookie","server","content_length","transfer_encoding","http_date" -"2018-04-19 00:02:26","75.74.78.113","tcp",8080,"c-75-74-78-113.hsd1.fl.comcast.net","http",7922,"US","FLORIDA","MIAMI",518111,737401,"HTTP/1.1",200,"OK","text/html",,,,"lighttpd",,"chunked","Thu, 19 Apr 2018 00:02:28 GMT" -"2018-04-19 00:02:26","88.162.174.130","tcp",8080,"sto95-3-88-162-174-130.fbx.proxad.net","http",12322,"FR",,"SAINT-OUEN-LAUMONE",518210,737415,"HTTP/1.1",200,"OK","text/html",,,,,17729,,"Thu, 19 Apr 2018 02:02:28 GMT" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv deleted file mode 100644 index b1f2330f1f..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","http","http_code","http_reason","content_type","connection","proxy_authenticate","via","server","content_length","transfer_encoding","http_date" -"2010-02-10 00:00:00",192.168.0.1,tcp,3128,node01.example.com,http-connect-proxy,64512,ZZ,Region,City,0,0,HTTP/1.1,200,"Connection established",,,,,,,,"Wed, 10 Feb 2010 00:00:00 GMT" -"2010-02-10 00:00:01",192.168.0.2,tcp,3128,node02.example.com,http-connect-proxy,64512,ZZ,Region,City,0,0,HTTP/1.1,200,"Connection established",,,,"HTTP/1.1 s_proxy_den1",,,,"Wed, 10 Feb 2010 00:00:01 GMT" -"2010-02-10 00:00:02",192.168.0.3,tcp,3128,node03.example.com,http-connect-proxy,64512,ZZ,Region,City,0,0,HTTP/1.1,200,"Connection established",,,,"HTTP/1.1 s_proxy_yvr",,,,"Wed, 10 Feb 2010 00:00:02 GMT" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv deleted file mode 100644 index 195342533e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","http","http_code","http_reason","content_type","connection","www_authenticate","set_cookie","server","content_length","transfer_encoding","http_date","version","build_date","detail" -"2010-02-10 00:00:00",192.168.0.1,tcp,8080,node01.example.com,"basic-auth,http",64512,ZZ,Region,City,0,0,HTTP/1.1,401,Unauthorized,"text/html; charset=utf-8",,"Basic realm=\"\"OpenWebif\"\"",TWISTED_SESSION=5473ad3faa3de66685fb3a53bffb390b4fcec2039893009a06caf38e1bec8aa8,TwistedWeb/19.7.0,149,,"Wed, 10 Feb 2010 00:00:00 GMT",,, -"2010-02-10 00:00:01",192.168.0.2,tcp,80,node02.example.com,"basic-auth,http",64512,ZZ,Region,City,0,0,HTTP/1.1,401,Unauthorized,"text/html; charset=utf-8",,"Basic realm=\"\"OpenWebif\"\"",TWISTED_SESSION=d2460d37b7fdbdd6c27dd74423ead5704e553d4f2c230672313edc5602059e33,TwistedWeb/19.7.0,149,,"Wed, 10 Feb 2010 00:00:01 GMT",,, -"2010-02-10 00:00:02",192.168.0.3,tcp,443,node03.example.com,git-config-file,64512,ZZ,Region,City,0,0,,,,,,,,,,,"Wed, 10 Feb 2010 00:00:02 GMT",,,"repositoryformatversion = 0;filemode = false;bare = false;logallrefupdates = true;symlinks = false;ignorecase = true" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv deleted file mode 100644 index d327f1f3ba..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","device_vendor","device_type","device_model","device_version","device_id","response_size","raw_response" -2022-03-02 00:34:22,192.168.0.1,tcp,502,host1.example.net,modbus,64512,ZZ,REGION,CITY,0,0,Sector,Vendor 1,device_type,device_model,device_version,0,5,dGVzdDE= -2022-03-02 00:34:22,192.168.0.2,tcp,502,host2.example.net,modbus,64513,ZZ,REGION,CITY,0,0,Sector,Vendor 2,device_type,device_model,device_version,0,5,dGVzdDI= -2022-03-02 00:34:22,192.168.0.3,tcp,502,host3.example.net,modbus,64514,ZZ,REGION,CITY,0,0,Sector,Vendor 3,device_type,device_model,device_version,0,5,dGVzdDM= diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv deleted file mode 100644 index 87a98157ff..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv +++ /dev/null @@ -1,96 +0,0 @@ -"timestamp","ip","port","hostname","tag","ipmi_version","asn","geo","region","city","none_auth","md2_auth","md5_auth","passkey_auth","oem_auth","defaultkg","permessage_auth","userlevel_auth","usernames","nulluser","anon_login","error","deviceid","devicerev","firmwarerev","version","manufacturerid","manufacturername","productid","productname","naics","sic","sector" -"2016-07-24 00:09:42","198.51.100.4",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:43","198.51.100.182",623,,"ipmi","2.0",28753,"DE","HESSEN","FRANKFURT AM MAIN","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:42","198.51.100.4",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:43","198.51.100.182",623,,"ipmi","2.0",28753,"DE","HESSEN","FRANKFURT AM MAIN","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:43","198.51.100.176",623,,"ipmi","1.5",2914,"DE","BAYERN","MUNICH","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:43","198.51.100.221",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:44","198.51.100.176",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:45","198.51.100.174",623,,"ipmi","1.5",2914,"DE","HESSEN","FRANKFURT AM MAIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:45","198.51.100.167",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:46","198.51.100.60",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:09:47","198.51.100.7",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:48","198.51.100.24",623,,"ipmi","2.0",24940,"DE","BAYERN","GUNZENHAUSEN","no","yes","yes","yes","yes","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:49","198.51.100.86",623,,"ipmi","1.5",2914,"DE","HESSEN","FRANKFURT AM MAIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:49","198.51.100.231",623,,"ipmi","1.5",20686,"DE","BAYERN","HAPPURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:49","198.51.100.197",623,,"ipmi","2.0",3320,"DE","BERLIN","BERLIN","no","no","yes","yes","yes","default","enabled","enabled","yes","no","yes",,,,,,,,,,541690,874899, -"2016-07-24 00:09:49","198.51.100.87",623,,"ipmi","1.5",3209,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:09:49","198.51.100.6",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:51","198.51.100.193",623,,"ipmi","2.0",15598,"DE","BAYERN","NUREMBERG","no","yes","yes","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:51","198.51.100.63",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:52","198.51.100.179",623,,"ipmi","2.0",3320,"DE","BAYERN","DENKLINGEN","no","no","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,518210,737415, -"2016-07-24 00:09:53","198.51.100.112",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:53","198.51.100.189",623,,"ipmi","2.0",30134,"DE","HESSEN","FRANKFURT AM MAIN","no","no","no","no","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0,"Communications" -"2016-07-24 00:09:54","198.51.100.44",623,"198-51-100-44.example.net","ipmi","2.0",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE","no","yes","yes","no","no","default","disabled","disabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:54","198.51.100.215",623,,"ipmi","1.5",2914,"DE","BAYERN","MUNICH","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:55","198.51.100.231",623,"198-51-100-231.example.net","ipmi","2.0",6805,"DE","HAMBURG","HAMBURG","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:55","198.51.100.234",623,,"ipmi","2.0",31103,"DE","THURINGEN","ERFURT","no","no","yes","no","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:55","198.51.100.165",623,,"ipmi","2.0",31400,"DE","RHEINLAND-PFALZ","FREINSHEIM","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:56","198.51.100.170",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:09:56","198.51.100.66",623,,"ipmi","2.0",41412,"DE","BAYERN","REGENSBURG","no","yes","yes","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:56","198.51.100.150",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:57","198.51.100.222",623,,"ipmi","2.0",34309,"DE","BERLIN","BERLIN","no","yes","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:57","198.51.100.19",623,,"ipmi","2.0",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:58","198.51.100.83",623,,"ipmi","1.5",3209,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:10:00","198.51.100.61",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:00","198.51.100.94",623,,"ipmi","1.5",2914,"DE","HESSEN","FRANKFURT AM MAIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:01","198.51.100.242",623,,"ipmi","2.0",28753,"DE","HESSEN","FRANKFURT AM MAIN","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:03","198.51.100.251",623,,"ipmi","2.0",553,"DE","BADEN-WURTTEMBERG","HEIDELBERG","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:10:03","198.51.100.41",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:04","198.51.100.160",623,"198-51-100-160.example.net","ipmi","1.5",2914,"DE","BAYERN","MUNICH","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:04","198.51.100.243",623,,"ipmi","1.5",2914,"DE","BAYERN","MUNICH","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:05","198.51.100.190",623,,"ipmi","1.5",2914,"DE","HESSEN","FRANKFURT AM MAIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:05","198.51.100.29",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:05","198.51.100.224",623,,"ipmi","2.0",13301,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:06","198.51.100.143",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","HEMER","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:07","198.51.100.120",623,,"ipmi","2.0",13003,"DE","SACHSEN","LEIPZIG","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:07","198.51.100.196",623,,"ipmi","1.5",20686,"DE","BAYERN","HAPPURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:07","198.51.100.123",623,,"ipmi","1.5",2914,"DE","HESSEN","FRANKFURT AM MAIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:08","198.51.100.122",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:08","198.51.100.192",623,,"ipmi","2.0",34171,"DE","BERLIN","BERLIN","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,518210,737415, -"2016-07-24 00:10:08","198.51.100.146",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:08","198.51.100.127",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:09","198.51.100.112",623,,"ipmi","2.0",24940,"DE","BAYERN","GUNZENHAUSEN","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:10:09","198.51.100.45",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:09","198.51.100.46",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","NEUSS","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:10","198.51.100.202",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:11","198.51.100.6",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:11","198.51.100.34",623,,"ipmi","2.0",3320,"DE","HESSEN","LEUN","no","yes","yes","no","yes","default","enabled","enabled","yes","no","no",,,,,,,,,,518210,737415, -"2016-07-24 00:10:12","198.51.100.210",623,,"ipmi","2.0",3320,"DE","BADEN-WURTTEMBERG","AALEN","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,541690,874899, -"2016-07-24 00:10:12","198.51.100.97",623,,"ipmi","2.0",42730,"DE","BERLIN","BERLIN","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:12","198.51.100.172",623,,"ipmi","1.5",2914,"DE","HESSEN","FRANKFURT AM MAIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:13","198.51.100.20",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:13","198.51.100.181",623,,"ipmi","2.0",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE","no","yes","yes","no","no","default","disabled","disabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:13","198.51.100.244",623,,"ipmi","2.0",28753,"DE","HESSEN","FRANKFURT AM MAIN","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:13","198.51.100.85",623,,"ipmi","2.0",34309,"DE","HESSEN","FRANKFURT AM MAIN","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:13","198.51.100.150",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:13","198.51.100.154",623,,"ipmi","2.0",196763,"DE","SAARLAND","ST. INGBERT","no","yes","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:13","198.51.100.83",623,,"ipmi","2.0",31342,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:14","198.51.100.6",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:14","198.51.100.228",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:14","198.51.100.150",623,,"ipmi","2.0",44066,"DE","BAYERN","MUNICH","no","yes","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:15","198.51.100.71",623,,"ipmi","2.0",44066,"DE","BAYERN","MUNICH","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:15","198.51.100.239",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:10:17","198.51.100.46",623,"198-51-100-53.example.net","ipmi","2.0",29083,"DE","BRANDENBURG","MAHLOW","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:17","198.51.100.78",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:18","198.51.100.164",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,812990,489999, -"2016-07-24 00:10:18","198.51.100.142",623,,"ipmi","2.0",34568,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:18","198.51.100.85",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:19","198.51.100.173",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:19","198.51.100.180",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:19","198.51.100.119",623,,"ipmi","2.0",12843,"DE","RHEINLAND-PFALZ","SPEYER","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:19","198.51.100.183",623,,"ipmi","1.5",12348,"DE","BAYERN","NUREMBERG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:20","198.51.100.108",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:20","198.51.100.221",623,"198-51-100-156.example.net","ipmi","2.0",24940,"DE","BAYERN","GUNZENHAUSEN","no","yes","yes","yes","yes","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:10:21","198.51.100.200",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:21","198.51.100.162",623,,"ipmi","1.5",30766,"DE","HESSEN","BENSHEIM","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:21","198.51.100.140",623,,"ipmi","2.0",31400,"DE","RHEINLAND-PFALZ","FREINSHEIM","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:21","198.51.100.121",623,,"ipmi","2.0",34549,"DE","HESSEN","FRANKFURT AM MAIN","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:22","198.51.100.33",623,,"ipmi","2.0",47215,"DE","NORDRHEIN-WESTFALEN","GUTERSLOH","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:22","198.51.100.203",623,,"ipmi","2.0",201011,"DE","BAYERN","NUREMBERG","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:23","198.51.100.16",623,,"ipmi","2.0",28753,"DE","HESSEN","FRANKFURT AM MAIN","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:24","198.51.100.166",623,,"ipmi","2.0",24940,"DE","BAYERN","GUNZENHAUSEN","no","no","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:25","198.51.100.135",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:25","198.51.100.154",623,"198-51-100-154.example.net","ipmi","1.5",2914,"DE","HESSEN","FRANKFURT AM MAIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:25","198.51.100.237",623,,"ipmi","2.0",12586,"DE","HESSEN","FRANKFURT AM MAIN","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:25","198.51.100.45",623,,"ipmi","2.0",13301,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv deleted file mode 100644 index a585db6eb6..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv +++ /dev/null @@ -1,2 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","ipp_version","cups_version","printer_uris","printer_name","printer_info","printer_more_info","printer_make_and_model","printer_firmware_name","printer_firmware_string_version","printer_firmware_version","printer_organization","printer_organization_unit","printer_uuid","printer_wifi_ssid","device_vendor","device_type","device_model","device_version","device_sector" -"2020-06-08 11:30:14","123.45.67.89","tcp",631,"some.host.com","ipp",12345,"AA","REGION","CITY",517311,0,"IPP/2.1","CUPS/2.0","ipp://123.45.67.89:631/ipp/print","NPI3F0D22","HP Color LaserJet MFP M277dw","http://123.45.67.89:631/hp/device/info_config_AirPrint.html?tab=Networking&menu=AirPrintStatus","HP Color LaserJet MFP M277dw",20191203,20191203,20191203,"org","unit","urn:uuid:456e4238-4a44-4643-4c42-10e1813f0a18","wifissid",,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv.license deleted file mode 100644 index 476908eebe..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2020 Thomas Hungenberg -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv deleted file mode 100644 index cef6b027c6..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","initiator_spi","responder_spi","next_payload","exchange_type","flags","message_id","next_payload2","domain_of_interpretation","protocol_id","spi_size","notify_message_type" -"2019-09-04 00:17:25","198.123.245.42","udp",500,"example.local","isakmp-vulnerable",5678,"AA","LOCATION","LOCATION",517311,0,"3e35c70729dfedef","253acab7cbfda607",11,05,00,00000000,00,00,,0,14 -"2019-09-04 00:17:28","198.123.245.67","udp",500,"example.local","isakmp-vulnerable",20255,"AA","LOCATION","LOCATION",0,0,"3e35c70729dfedef","b274460e7adc1bf0",11,05,00,00000000,00,00,,0,14 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv deleted file mode 100644 index ab71b9a15d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","sector","http","http_code","http_reason","content_type","server","date","major","minor","git_version","git_commit","git_tree_state","build_date","go_version","compiler","platform","handshake","cipher_suite","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serial_number","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","cert_valid","self_signed","cert_expired","validation_level","browser_trusted","browser_error","raw_cert","raw_cert_chain" -"2010-02-10 00:00:00",192.168.0.1,tcp,6443,node01.example.com,kubernetes,,64512,ZZ,Region,City,0,0,,HTTP/1.1,200,OK,application/json,,"Tue, 10 May 2022 14:24:13 GMT",1,20,v1.20.13,2444b3347a2c45eb965b182fb836e1f51dc61b70,clean,2021-11-17T13:00:29Z,go1.15.15,gc,linux/amd64,TLSv1.2,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,2,sha256WithRSAEncryption,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,N,Y,unknown,N,"x509: failed to load system roots and no roots provided",, -"2010-02-10 00:00:01",192.168.0.2,tcp,6443,node02.example.com,kubernetes,,64512,ZZ,Region,City,0,0,"Retail Trade",HTTP/1.1,200,OK,application/json,,"Tue, 10 May 2022 14:24:12 GMT",1,23,v1.23.3+e419edf,6f5a5295923a614a4202a7ad274b38b69f9ca8c0,clean,2022-02-25T06:26:46Z,go1.17.5,gc,linux/amd64,TLSv1.2,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,2,sha256WithRSAEncryption,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,N,Y,unknown,N,"x509: failed to load system roots and no roots provided",, -"2010-02-10 00:00:02",192.168.0.3,tcp,6443,node03.example.com,kubernetes,,64512,ZZ,Region,City,0,0,,HTTP/1.1,200,OK,application/json,,"Tue, 10 May 2022 14:24:12 GMT",1,16+,v1.16.9-aliyun.1,4f7ea78,,2020-05-08T07:29:59Z,go1.13.9,gc,linux/amd64,TLSv1.2,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,2,sha256WithRSAEncryption,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,N,Y,unknown,N,"x509: failed to load system roots and no roots provided",, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv deleted file mode 100644 index 54121fd3b7..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","size","configuration_naming_context","current_time","default_naming_context","dns_host_name","domain_controller_functionality","domain_functionality","ds_service_name","forest_functionality","highest_committed_usn","is_global_catalog_ready","is_synchronized","ldap_service_name","naming_contexts","root_domain_naming_context","schema_naming_context","server_name","subschema_subentry","supported_capabilities","supported_control","supported_ldap_policies","supported_ldap_version","supported_sasl_mechanisms","amplification" -"2010-02-10 00:00:00",192.168.0.1,tcp,389,node01.example.com,ldap-tcp,64512,ZZ,Region,City,0,0,0,"CN=Configuration,DC=ad,DC=example,DC=com",,"DC=ad,DC=example,DC=com",node01.example.com,7,,"CN=Configuration,DC=ad,DC=example,DC=com",2,,,,node01.example.com,"DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com","DC=example,DC=com","CN=Schema,CN=Configuration,DC=example,DC=com","CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com","CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com",,1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|,MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent,3|2,GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5, -"2010-02-10 00:00:01",192.168.0.2,tcp,389,node02.example.com,ldap-tcp,64512,ZZ,Region,City,0,0,0,"CN=Configuration,DC=ad,DC=example,DC=com",20220821124435.0Z,"DC=ad,DC=example,DC=com",node02.example.com,7,7,"CN=Configuration,DC=ad,DC=example,DC=com",7,25029662,TRUE,TRUE,node02.example.com,"DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com","DC=example,DC=com","CN=Schema,CN=Configuration,DC=example,DC=com","CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com","CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com",1.2.840.113556.1.4.800|1.2.840.113556.1.4.1670|1.2.840.113556.1.4.1791|1.2.840.113556.1.4.1935|1.2.840.113556.1.4.2080|1.2.840.113556.1.4.2237,1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|1.2.840.113556.1.4.2206|1.2.840.113556.1.4.2211|1.2.840.113556.1.4.2239|1.2.840.113556.1.4.2255|1.2.840.113556.1.4.2256|1.2.840.113556.1.4.2309,MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent,3|2,GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5, -"2010-02-10 00:00:02",192.168.0.3,tcp,389,node03.example.com,ldap-tcp,64512,ZZ,Region,City,0,0,0,"CN=Configuration,DC=ad,DC=example,DC=com",20220821124539.0Z,"DC=ad,DC=example,DC=com",node03.example.com,,,"CN=Configuration,DC=ad,DC=example,DC=com",,,,,node03.example.com,"DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com","DC=example,DC=com","CN=Schema,CN=Configuration,DC=example,DC=com","CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com","CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com",,1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv deleted file mode 100644 index 3cd5021c54..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","size","configuration_naming_context","current_time","default_naming_context","dns_host_name","domain_controller_functionality","domain_functionality","ds_service_name","forest_functionality","highest_committed_usn","is_global_catalog_ready","is_synchronized","ldap_service_name","naming_contexts","root_domain_naming_context","schema_naming_context","server_name","subschema_subentry","supported_capabilities","supported_control","supported_ldap_policies","supported_ldap_version","supported_sasl_mechanisms","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,389,node01.example.com,ldap-udp,64512,ZZ,Region,City,0,0,3038,"CN=Configuration,DC=ad,DC=example,DC=com",20220821044533.0Z,"DC=ad,DC=example,DC=com",node01.example.com,7,7,"CN=Configuration,DC=ad,DC=example,DC=com",7,222537,TRUE,TRUE,node01.example.com,"DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com","DC=example,DC=com","CN=Schema,CN=Configuration,DC=example,DC=com","CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com","CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com",1.2.840.113556.1.4.800|1.2.840.113556.1.4.1670|1.2.840.113556.1.4.1791|1.2.840.113556.1.4.1935|1.2.840.113556.1.4.2080|1.2.840.113556.1.4.2237,1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|1.2.840.113556.1.4.2206|1.2.840.113556.1.4.2211|1.2.840.113556.1.4.2239|1.2.840.113556.1.4.2255|1.2.840.113556.1.4.2256|1.2.840.113556.1.4.2309|1.2.840.113556.1.4.2330|1.2.840.113556.1.4.2354,MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent,3|2,GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5,58.42 -"2010-02-10 00:00:01",192.168.0.2,udp,389,node02.example.com,ldap-udp,64512,ZZ,Region,City,0,0,3062,"CN=Configuration,DC=ad,DC=example,DC=com",20220821044948.0Z,"DC=ad,DC=example,DC=com",node02.example.com,7,7,"CN=Configuration,DC=ad,DC=example,DC=com",7,1478714,TRUE,TRUE,node02.example.com,"DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com","DC=example,DC=com","CN=Schema,CN=Configuration,DC=example,DC=com","CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com","CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com",1.2.840.113556.1.4.800|1.2.840.113556.1.4.1670|1.2.840.113556.1.4.1791|1.2.840.113556.1.4.1935|1.2.840.113556.1.4.2080|1.2.840.113556.1.4.2237,1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|1.2.840.113556.1.4.2206|1.2.840.113556.1.4.2211|1.2.840.113556.1.4.2239|1.2.840.113556.1.4.2255|1.2.840.113556.1.4.2256|1.2.840.113556.1.4.2309|1.2.840.113556.1.4.2330|1.2.840.113556.1.4.2354,MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent,3|2,GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5,58.88 -"2010-02-10 00:00:02",192.168.0.3,udp,389,node03.example.com,ldap-udp,64512,ZZ,Region,City,0,0,36,"CN=Configuration,DC=ad,DC=example,DC=com",,"DC=ad,DC=example,DC=com",node03.example.com,,,"CN=Configuration,DC=ad,DC=example,DC=com",,,,,node03.example.com,"DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com","DC=example,DC=com","CN=Schema,CN=Configuration,DC=example,DC=com","CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com","CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com",,,,,,0.69 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv deleted file mode 100644 index 4a97121e75..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","mdns_name","mdns_ipv4","mdns_ipv6","services","workstation_name","workstation_ipv4","workstation_ipv6","workstation_info","http_name","http_ipv4","http_ipv6","http_ptr","http_info","http_target","http_port","spotify_name","spotify_ipv4","spotify_ipv6","opc_ua_discovery" -"2010-02-10 00:00:00",192.168.0.1,udp,5353,node01.example.com,mdns,64512,ZZ,Region,City,0,0,,,,"_smb._tcp.local.; _device-info._tcp.local.; _http._tcp.local.; _dacp._tcp.local.;",,192.168.0.1,fd09:4ab5:dae9:b078::1,,,192.168.0.1,fd09:4ab5:dae9:b078::1,,,,,,,, -"2010-02-10 00:00:01",192.168.0.2,udp,5353,node02.example.com,mdns,64512,ZZ,Region,City,0,0,,,,_home-assistant._tcp.local.;,,192.168.0.2,fd09:4ab5:dae9:b078::2,,,192.168.0.2,fd09:4ab5:dae9:b078::2,,,,,,,, -"2010-02-10 00:00:02",192.168.0.3,udp,5353,node03.example.com,"mdns,iot",64512,ZZ,Region,City,0,0,,,,"_webdav._tcp.local.; _adisk._tcp.local.; _smb._tcp.local.; _http._tcp.local.; _dacp._tcp.local.; _afpovertcp._tcp.local.; _device-info._tcp.local.;",,192.168.0.3,fd09:4ab5:dae9:b078::3,,snmeijer.local.,192.168.0.3,fd09:4ab5:dae9:b078::3,snmeijer._http._tcp.local.,"\"\"vendor=Synology\"\" \"\"model=DS218+\"\" \"\"serial=17A0PCN482002\"\" \"\"version_major=6\"\" \"\"version_minor=2\"\" \"\"version_build=25556\"\" \"\"admin_port=5000\"\" \"\"secure_admin_port=5001\"\" \"\"mac_address=00:11:32:80:fd:b5\"\"",snmeijer.local.,5000,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv deleted file mode 100644 index 6a1d445e7a..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","pid","pointer_size","uptime","time","curr_connections","total_connections","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,50260,node01.example.com,memcached,1.4.15,64512,ZZ,Region,City,0,0,1010,64,32908114,"2022-08-21 10:34:06",243,6106,"Communications, Service Provider, and Hosting Service",1144,81.71 -"2010-02-10 00:00:01",192.168.0.2,udp,11211,node02.example.com,memcached,1.4.13,64512,ZZ,Region,City,0,0,5316,64,9618498,"2022-08-21 10:39:21",9,2962,"Communications, Service Provider, and Hosting Service",1053,75.21 -"2010-02-10 00:00:02",192.168.0.3,udp,11211,node03.example.com,memcached,1.2.6,64512,ZZ,Region,City,0,0,1460,32,1375159,"2022-08-21 10:39:39",2,534,,442,31.57 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv deleted file mode 100644 index 1228dcfc60..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv +++ /dev/null @@ -1,11 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","gitversion","sysinfo","opensslversion","allocator","javascriptengine","bits","maxbsonobjectsize","ok","visible_databases","sector" -"2016-07-24 00:40:07","198.51.100.203","tcp",27017,"198-51-100-203.example.net","mongodb","2.4.5",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,"a2ddc68ba7c9cee17bfe69ed840383ec3506602b","Linux ip-198-51-100-100 198.51.100.103-2.ec2.v1.2.fc8xen #1 SMP Fri Nov 20 17:48:28 EST 2009 x86_64 BOOST_LIB_VERSION=1_49",,"tcmalloc","V8",64,16777216,1,"local | countly | admin", -"2016-07-24 00:40:07","198.51.100.42","tcp",27017,"198-51-100-208.example.net","mongodb","2.6.12",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,"d73c92b1c85703828b55c2916a5dd4ad46535f6a","Linux build5.ny.cbi.10gen.cc 2.6.32-431.3.1.el6.x86_64 #1 SMP Fri Jan 3 21:39:27 UTC 2014 x86_64 BOOST_LIB_VERSION=1_49",,"tcmalloc","V8",64,16777216,1,"none visible","Information Technology" -"2016-07-24 00:40:07","198.51.100.225","tcp",27017,"198-51-100-225.example.net","mongodb","3.0.6",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,"1ef45a23a4c5e3480ac919b28afcba3c615488f2","Linux ip-198-51-100-100 3.4.43-43.43.amzn1.x86_64 #1 SMP Mon May 6 18:04:41 UTC 2013 x86_64 BOOST_LIB_VERSION=1_49","OpenSSL 1.0.0-fips 29 Mar 2010","tcmalloc","V8",64,16777216,1,"bluu | local","Communications" -"2016-07-24 00:40:07","198.51.100.144","tcp",27017,"198-51-100-144.example.net","mongodb","2.2.2",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,"d1b43b61a5308c4ad0679d34b262c5af9d664267","Linux ip-198-51-100-100 198.51.100.252-2.ec2.v1.2.fc8xen #1 SMP Fri Nov 20 17:48:28 EST 2009 x86_64 BOOST_LIB_VERSION=1_49",,,,64,16777216,1,"errbit_production | DELETED_BECAUSE_YOU_DIDNT_PASSWORD_PROTECT_YOUR_MONGODB | admin | local", -"2016-07-24 00:40:07","198.51.100.68","tcp",27017,,"mongodb","3.2.6",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,"05552b562c7a0b3143a729aaa0838e558dc49b25","deprecated",,"tcmalloc","mozjs",64,16777216,1,"none visible", -"2016-07-24 00:40:07","198.51.100.101","tcp",27017,,"mongodb","3.0.9",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,"20d60d3491908f1ae252fe452300de3978a040c7","Linux ip-198-51-100-100 3.13.0-24-generic #46-Ubuntu SMP Thu Apr 10 19:11:08 UTC 2014 x86_64 BOOST_LIB_VERSION=1_49","OpenSSL 1.0.1f 6 Jan 2014","tcmalloc","V8",64,16777216,1,"none visible", -"2016-07-24 00:40:07","198.51.100.53","tcp",27017,"198-51-100-162.example.net","mongodb","3.2.6",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,"05552b562c7a0b3143a729aaa0838e558dc49b25","deprecated",,"tcmalloc","mozjs",64,16777216,1,"none visible", -"2016-07-24 00:40:07","198.51.100.206","tcp",27017,"198-51-100-206.example.net","mongodb","2.4.10",34011,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,"e3d78955d181e475345ebd60053a4738a4c5268a","Linux bs-linux32.10gen.cc 198.51.100.34-2.fc8xen #1 SMP Fri Feb 15 12:39:36 EST 2008 i686 BOOST_LIB_VERSION=1_49",,"system","V8",32,16777216,1,"sharelatex | test1 | local | tmp | lococms_production", -"2016-07-24 00:40:10","198.51.100.157","tcp",27017,"198-51-100-157.example.net","mongodb","2.0.6",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,"nogitversion","Linux biber 2.6.32-5-amd64 #1 SMP Mon Feb 25 00:26:11 UTC 2013 i686 BOOST_LIB_VERSION=1_49",,,,32,16777216,1,"none visible", -"2016-07-24 00:40:10","198.51.100.173","tcp",27017,"198-51-100-173.example.net","mongodb","2.6.12",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,"nogitversion","FreeBSD 101amd64-default-job-24 10.1-RELEASE-p33 FreeBSD 10.1-RELEASE-p33 amd64 BOOST_LIB_VERSION=1_49","OpenSSL 1.0.1l-freebsd 15 Jan 2015","system","V8",64,16777216,1,"none visible", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv deleted file mode 100644 index cfe4f00614..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv +++ /dev/null @@ -1,2 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","anonymous_access","raw_response","hex_code","code","cipher_suite","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serialNumber" -"2022-02-07 12:56:53","18.220.0.0","tcp",8883,"18-220-0-0.example.com","mqtt",12345,"US","OHIO","COLUMBUS",454110,,"N",20020005,05,"Connection Refused, not authorized","TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",2048,"*.tracesafe.io","Sectigo RSA Domain Validation Secure Server CA","2020-08-12 00:00:00","2022-11-14 00:00:00","70:84:F1:6D:28:DA:B6:E6:27:60:13:8B:2C:93:52:B6:7B:4B:13:7B","D2:D7:54:52:EB:86:4E:2D:34:4D:FC:CE:CD:CF:39:41:E1:06:5C:8B:B8:54:E6:0C:DF:FD:6E:E3:F1:B5:41:00","17:57:FB:88:9D:BE:A7:F0:29:A5:31:FC:79:DF:F7:8A:1C:D6:4A:DF:1B:4A:DC:BF:05:E7:E8:2F:79:9A:FA:FE:F7:E8:66:22:CB:B9:4C:72:F7:FB:6C:1D:59:8C:54:63:70:05:DE:7F:3C:2F:BA:B8:37:18:CE:29:6F:11:E8:AB","DE:2C:98:30:27:2E:7D:C9:ED:A3:9D:AF:9E:CE:14:CC","085699743A23114C9B6B8DC975A8AF42",2,"sha256WithRSAEncryption","rsaEncryption",,,,,,,,,,,,,"Sectigo Limited",,"GB","Greater Manchester","Salford",,,,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv.license deleted file mode 100644 index 476908eebe..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2020 Thomas Hungenberg -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv deleted file mode 100644 index e0ab4b9298..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","raw_response","hex_code","code","cipher_suite","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serialNumber" -"2022-01-10 00:59:34","47.106.0.0","tcp",8883,,"mqtt,mqtt-anon",37963,"CN","GUANGDONG SHENG","SHENZHEN",518210,,20020000,00,"Connection Accepted","TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",2048,"Server","RootCA","2020-05-08 08:07:05","2030-05-06 08:07:05","70:1A:1E:1F:EC:5F:7E:A9:12:32:B2:C9:8A:C9:EE:91:8E:0B:82:45","85:26:A2:F2:A2:50:CD:96:33:19:A6:2D:12:2E:97:6B:D3:06:3C:11:EA:01:B4:B7:25:2A:B7:4F:0A:8F:45:40","72:50:07:30:9A:6F:CB:FD:E2:80:69:02:65:62:77:16:C3:B4:0C:98:44:4E:D4:2C:AC:6B:AF:F8:9E:AB:51:C2:FA:A8:72:A3:45:DF:81:09:50:08:18:EB:03:34:FC:92:33:A7:12:46:FE:90:20:91:86:C5:4D:89:48:86:4C:CD","AB:A8:E0:2C:EF:AE:BF:9D:DD:FA:70:BA:2F:F2:CA:5C",02,2,"sha256WithRSAEncryption","rsaEncryption","EMQ",,"CN","hangzhou",,,,,,,,,"EMQ",,"CN","hangzhou",,,,,,,, -"2022-01-10 00:59:34","144.76.0.0","tcp",8883,,"mqtt,mqtt-anon",24940,"DE","SACHSEN-ANHALT","WERNIGERODE",518210,,20020000,00,"Connection Accepted","TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",2048,"example.com","R3","2021-12-06 13:48:04","2022-03-06 13:48:03","20:0E:AC:E7:AF:07:8D:D3:16:7C:63:D1:B9:12:AD:1D:2C:F0:46:86","DD:7A:4C:8A:1D:66:1D:7C:F5:17:04:5B:A0:B4:C4:E0:80:58:44:B4:DB:A7:5E:61:AE:43:9D:85:4C:9E:DC:83","55:B6:3D:56:A4:39:6E:99:B6:AF:72:AF:4D:3C:7C:C5:A8:C5:4F:A1:79:92:D0:46:8A:A2:9B:2A:48:0D:00:68:39:F0:B8:67:B4:E0:88:51:2A:D7:55:46:83:BD:ED:1E:09:6E:DB:3D:21:E2:AA:DB:42:6A:33:45:1A:2A:DB:4C","23:99:39:C6:77:D8:9F:55:90:FC:A5:FB:BA:72:8B:42","06B25BEAD1F43266ABCFCDDE408D3544D04B",2,"sha256WithRSAEncryption","rsaEncryption",,,,,,,,,,,,,"Lets Encrypt",,"US",,,,,,,,, -"2022-01-10 00:59:34","173.0.0.0","tcp",8883,"example.com","mqtt,mqtt-anon",5555,"US","CALIFORNIA","BURBANK",,,20020000,00,"Connection Accepted","TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA",2048,"foo.example.com","ClearView2Dev","2020-08-07 16:51:57","2030-08-05 16:51:57","32:4B:66:98:FA:5B:D2:D1:F2:53:83:21:19:11:5A:A9:BE:85:56:16","AE:0D:65:34:2F:51:F7:32:1E:DF:B1:DA:12:C7:6A:DE:42:B5:4B:FF:80:2C:E5:EF:99:F6:CC:01:4B:C9:77:68","44:C4:B8:19:FA:39:55:51:EC:E4:6D:C4:6D:0F:A5:46:BB:D5:F9:FD:A6:8D:DF:F3:2D:D2:92:6C:0B:D5:D3:25:CB:19:50:9D:A6:A4:D4:D3:2E:53:10:F5:8D:77:F7:90:F8:65:A7:79:AB:14:62:72:01:F3:EA:38:E2:68:C7:25","43:0D:A7:89:9E:76:8D:6E:D5:AD:95:CC:F2:91:87:56","A71541EFAE529B03",0,"sha256WithRSAEncryption","rsaEncryption","Sohonet",,,,"<",,,,,,,,"Sohonet","ClearView2Dev",,,,,,,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv deleted file mode 100644 index c12a6063eb..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","server_name","instance_name","tcp_port","named_pipe","response_size","amplification","sector" -"2010-02-10 00:00:00",192.168.0.1,udp,1434,node01.example.com,mssql,13.2.5026.0,64512,ZZ,Region,City,0,0,ERPOPTIMA,OPTIMA,49729,"\\\\ERPOPTIMA\\pipe\\MSSQL$OPTIMA\\sql\\query",310,310.00, -"2010-02-10 00:00:01",192.168.0.2,udp,1434,node02.example.com,mssql,13.0.1601.5,64512,ZZ,Region,City,0,0,SERWER,MSSQLSERVER,1433,,226,226.00,"Communications, Service Provider, and Hosting Service" -"2010-02-10 00:00:02",192.168.0.3,udp,1434,node03.example.com,mssql,10.50.2500.0,64512,ZZ,Region,City,0,0,ILONY,INSERTGT,49358,"\\\\ILONY\\pipe\\MSSQL$INSERTGT\\sql\\query",304,304.00, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv deleted file mode 100644 index 25fed2166b..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","sector","mysql_protocol_version","server_version","error_code","error_id","error_message","client_can_handle_expired_passwords","client_compress","client_connect_attrs","client_connect_with_db","client_deprecated_eof","client_found_rows","client_ignore_sigpipe","client_ignore_space","client_interactive","client_local_files","client_long_flag","client_long_password","client_multi_results","client_multi_statements","client_no_schema","client_odbc","client_plugin_auth","client_plugin_auth_len_enc_client_data","client_protocol_41","client_ps_multi_results","client_reserved","client_secure_connection","client_session_track","client_ssl","client_transactions","handshake","cipher_suite","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serial_number","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","cert_valid","self_signed","cert_expired","validation_level","browser_trusted","browser_error","raw_cert","raw_cert_chain" -"2010-02-10 00:00:00",192.168.0.1,tcp,3306,node01.example.com,mysql,,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",10,5.7.37-0ubuntu0.18.04.1,1,1,1,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,N,N,N,TLSv1.2,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,2,sha256WithRSAEncryption,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,,Y,unknown,,"x509: failed to load system roots and no roots provided",, -"2010-02-10 00:00:01",192.168.0.2,tcp,3306,node02.example.com,mysql,,64512,ZZ,Region,City,0,0,,10,5.7.30-0ubuntu0.18.04.1-log,1,1,1,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,N,N,N,TLSv1.2,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,2,sha256WithRSAEncryption,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,,Y,unknown,,"x509: failed to load system roots and no roots provided",, -"2010-02-10 00:00:02",192.168.0.3,tcp,3306,node03.example.com,mysql,,64512,ZZ,Region,City,0,0,"Retail Trade",10,8.0.23,1,1,1,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,N,N,N,TLSv1.2,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,2,sha256WithRSAEncryption,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,,Y,unknown,,"x509: failed to load system roots and no roots provided",, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv deleted file mode 100644 index e8a1108d5a..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","opcode","uptime","external_ip","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,5351,node01.example.com,nat-pmp,0,64512,ZZ,Region,City,0,0,128,291278940,192.168.0.1,,12,6.00 -"2010-02-10 00:00:01",192.168.0.2,udp,5351,node02.example.com,nat-pmp,0,64512,ZZ,Region,City,0,0,128,768416,192.168.0.2,,12,6.00 -"2010-02-10 00:00:02",192.168.0.3,udp,5351,node03.example.com,nat-pmp,0,64512,ZZ,Region,City,0,0,128,19629454,192.168.0.3,,12,6.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv deleted file mode 100644 index 932225b0b0..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","mac_address","asn","geo","region","city","workgroup","machine_name","username","naics","sic","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,137,node01.example.com,netbios,00-00-00-00-00-00,64512,ZZ,Region,City,,NBG6503,NBG6503,0,0,,229,4.58 -"2010-02-10 00:00:01",192.168.0.2,udp,137,node02.example.com,netbios,00-00-00-00-00-00,64512,ZZ,Region,City,PRACOWNIAELN.,NAS-OLD,NAS-OLD,0,0,,193,3.86 -"2010-02-10 00:00:02",192.168.0.3,udp,137,node03.example.com,netbios,00-25-90-F0-64-64,64512,ZZ,Region,City,HRSIGMA,HR-SRV01,,0,0,Government,157,3.14 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv deleted file mode 100644 index 4e91593565..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","port","hostname","tag","response","asn","geo","region","city","naics","sic","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,53413,node01.example.com,netis_vulnerability,Login:,64512,ZZ,Region,City,0,0,,18,18.00 -"2010-02-10 00:00:01",192.168.0.2,53413,node02.example.com,netis_vulnerability,Login:,64512,ZZ,Region,City,0,0,,18,18.00 -"2010-02-10 00:00:02",192.168.0.3,53413,node03.example.com,netis_vulnerability,Login:,64512,ZZ,Region,City,0,0,,18,18.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv deleted file mode 100644 index cc3cf6fc2f..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","version","clk_wander","clock","error","frequency","jitter","leap","mintc","noise","offset","peer","phase","poll","precision","processor","refid","reftime","rootdelay","rootdispersion","stability","state","stratum","system","tai","tc","naics","sic","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,123,node01.example.com,ntpversion,64512,ZZ,Region,City,4,,0xe6ac3809.363028e7,,2.018,0.977,0,,0.984,0.557,18986,,10,-10,unknown,81.15.252.130,0xe6ac35ba.2d2e8f2b,17.685,61.254,0.027,4,4,UNIX,,,0,0,,324,27.00 -"2010-02-10 00:00:01",192.168.0.2,udp,123,node02.example.com,ntpversion,64512,ZZ,Region,City,4,0.007,0xE6AC3806.7DF3B7A0,,-20.407,8.776,0,3,,-14.502,19244,,,-10,unknown,10.48.21.21,0xE6AC3431.B3B64790,32.25,105.778,,,8,UNIX,,10,0,0,"Transportation and Warehousing",328,27.33 -"2010-02-10 00:00:02",192.168.0.3,udp,123,node03.example.com,ntpversion,64512,ZZ,Region,City,4,0.001,0xE6AC380A.5A1CAD00,,-24.01,2.343,0,3,,0.49,51892,,,-10,unknown,172.28.0.1,0xE6AC3020.0C49BA80,7.749,81.612,,,4,UNIX,,10,0,0,,324,27.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv deleted file mode 100644 index dca5386d9e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","packets","size","asn","geo","region","city","naics","sic","sector","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,123,node01.example.com,2,664,64512,ZZ,Region,City,0,0,,55.33 -"2010-02-10 00:00:01",192.168.0.2,udp,123,node02.example.com,100,44000,64512,ZZ,Region,City,0,0,,3666.67 -"2010-02-10 00:00:02",192.168.0.3,udp,123,node03.example.com,100,44000,64512,ZZ,Region,City,0,0,,3666.67 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv deleted file mode 100644 index c32bc3d4d0..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","programs","mountd_port","exports","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,111,node01.example.com,portmapper,64512,ZZ,Region,City,0,0,"100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp; 100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp;",,"/mnt/export 192.168.0.0","Communications, Service Provider, and Hosting Service",148,3.70 -"2010-02-10 00:00:01",192.168.0.2,udp,111,node02.example.com,portmapper,64512,ZZ,Region,City,0,0,"100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp; 100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp;",,"/mnt/export 192.168.0.0","Communications, Service Provider, and Hosting Service",148,3.70 -"2010-02-10 00:00:02",192.168.0.3,udp,111,node03.example.com,portmapper,64512,ZZ,Region,City,0,0,"100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp; 100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp;",,"/mnt/export 192.168.0.0",Government,148,3.70 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv deleted file mode 100644 index 8c1d6f725a..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","sector","supported_protocols","protocol_error_code","protocol_error_file","protocol_error_line","protocol_error_message","protocol_error_routine","protocol_error_severity","protocol_error_severity_v","startup_error_code","startup_error_file","startup_error_line","startup_error_message","startup_error_routine","startup_error_severity","startup_error_severity_v","client_ssl","handshake","cipher_suite","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serial_number","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","cert_valid","self_signed","cert_expired","validation_level","browser_trusted","browser_error","raw_cert","raw_cert_chain" -"2010-02-10 00:00:00",192.168.0.1,tcp,5432,node01.example.com,postgres,,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",1.0-3.0,0A000,postmaster.c,1798,"unsupported frontend protocol 255.255: server supports 1.0 to 3.0",ProcessStartupPacket,FATAL,,28000,postmaster.c,1893,"no PostgreSQL user name specified in startup packet",ProcessStartupPacket,FATAL,,N,,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,,,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,,Y,,,,, -"2010-02-10 00:00:01",192.168.0.2,tcp,5432,node02.example.com,postgres,,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",1.0-3.0,0A000,postmaster.c,1798,"unsupported frontend protocol 255.255: server supports 1.0 to 3.0",ProcessStartupPacket,FATAL,,28000,postmaster.c,1893,"no PostgreSQL user name specified in startup packet",ProcessStartupPacket,FATAL,,N,,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,,,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,,Y,,,,, -"2010-02-10 00:00:02",192.168.0.3,tcp,5432,node03.example.com,postgres,,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",1.0-3.0,0A000,postmaster.c,1798,"unsupported frontend protocol 255.255: server supports 1.0 to 3.0",ProcessStartupPacket,FATAL,,28000,postmaster.c,1893,"no PostgreSQL user name specified in startup packet",ProcessStartupPacket,FATAL,,N,,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,,,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,,Y,,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv deleted file mode 100644 index 857699376e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","quote","asn","geo","region","city","naics","sic","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,17,node01.example.com,qotd,"_The secret of being miserable is to have leisure to bother about whether?? you are happy or not. The cure for it is occupation._?? George Bernard Shaw (1856-1950)?",64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",166,166.00 -"2010-02-10 00:00:01",192.168.0.2,udp,17,node02.example.com,qotd,"_Oh the nerves, the nerves; the mysteries of this machine called man!?? Oh the little that unhinges it, poor creatures that we are!_?? Charles Dickens (1812-70)?",64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",162,162.00 -"2010-02-10 00:00:02",192.168.0.3,udp,17,node03.example.com,qotd,"_Oh the nerves, the nerves; the mysteries of this machine called man!?? Oh the little that unhinges it, poor creatures that we are!_?? Charles Dickens (1812-70)?",64512,ZZ,Region,City,0,0,,162,162.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv deleted file mode 100644 index c9fb18896e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","version_field_1","version_field_2","version_field_3","version_field_4" -"2022-01-10 14:31:17","176.255.0.0","udp",443,"test1.example.com","quic",5607,"UK","LONDON","LONDON",517311,,"Q050",,"Q046","Q043" -"2022-01-10 14:31:17","24.244.0.0","udp",443,,"quic",6327,"CA","SASKATCHEWAN","MEACHAM",517311,,"Q050","Q046",,"Q043" -"2022-01-10 14:31:17","23.60.0.0","udp",443,"test3.example.com","quic",20940,"JP","OSAKA","OSAKA",517919,,,"Q050","Q046","Q043" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv deleted file mode 100644 index 76b388acaa..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv +++ /dev/null @@ -1,10 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic" -"2020-07-06 13:55:26","74.101.218.75","tcp",4899,"static-74-101-218-75.nycmny.fios.verizon.net","radmin","Radmin (Details Unknown)",701,"US","NEW YORK","BROOKLYN",517312, -"2020-07-06 13:55:27","192.162.189.171","tcp",4899,"rubin.an.ru","radmin","Radmin v3.X Radmin Authentication",56618,"RU","MURMANSKAYA OBLAST","MURMANSK",0, -"2020-07-06 13:55:27","111.197.143.69","tcp",4899,,"radmin","Radmin (Details Unknown)",4808,"CN","BEIJING SHI","BEIJING",517311, -"2020-07-06 13:55:27","121.147.215.220","tcp",4899,,"radmin","Radmin v3.X Radmin Authentication",4766,"KR","GWANGJU-GWANGYEOKSI","DAEIN-DONG",517311, -"2020-07-06 13:55:27","121.147.215.178","tcp",4899,,"radmin","Radmin v3.X Radmin Authentication",4766,"KR","GWANGJU-GWANGYEOKSI","DAEIN-DONG",517311, -"2020-07-06 13:55:27","183.230.5.219","tcp",4899,,"radmin","Radmin v3.X Radmin Authentication",9808,"CN","CHONGQING SHI","CHONGQING",517312, -"2020-07-06 13:55:27","85.93.154.74","tcp",4899,,"radmin","Radmin v3.X Radmin Authentication",34300,"RU","MOSKVA","MOSCOW",0, -"2020-07-06 13:55:27","81.246.135.247","tcp",4899,"247.135-246-81.adsl-dyn.isp.belgacom.be","radmin","Radmin v3.X Radmin Authentication",5432,"BE","ANTWERPEN","BRASSCHAAT",517311, -"2020-07-06 13:55:27","46.27.146.22","tcp",4899,"static-22-146-27-46.ipcom.comunitel.net","radmin","Radmin v3.X Radmin Authentication",12430,"ES","LAS PALMAS","LAS PALMAS DE GRAN CANARIA",517312, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv.license deleted file mode 100644 index 833024a759..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2020 sinus-x -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv deleted file mode 100644 index 4bac90f199..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","port","hostname","tag","handshake","asn","geo","region","city","rdp_protocol","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","naics","sic","sector","tlsv13_support","tlsv13_cipher","cve20190708_vulnerable","bluekeep_vulnerable","jarm" -"2019-09-04 15:45:51","198.123.245.178",5678,,"rdp",,5678,"AA","LOCATION","LOCATION","RDP",5678,"KABESRV.KABE.local","KABESRV.KABE.local","2019-04-29 02:22:06","2019-10-29 02:22:06","EC:BB:4D:DB:9F:0C:D3:FF:5B:49:EA:B1:56:62:B6:A7:5D:60:54:42","1EF2B37AF850C9BF4E88F18177001D6B",2,"sha256WithRSAEncryption","rsaEncryption","B7:C9:F4:07:D5:C0:75:1D:EA:0C:40:E7:26:39:C2:30:C6:13:83:7E:18:46:D8:E9:4C:45:3F:88:1B:0B:70:76","08:AC:75:FA:EB:A3:2B:44:15:DE:6D:A7:0B:C0:AE:17:94:F3:55:D9:EC:70:AC:5B:B7:94:79:F0:D7:84:83:89:CB:A9:11:E0:08:D7:54:4D:33:85:89:D2:A8:DD:9D:15:F4:CC:95:DE:6A:E3:DF:6B:FA:8B:27:E3:DA:16:AF:0A","BC:6E:C3:E2:98:22:EC:BA:5B:30:E2:53:FD:4A:9D:FF",517311,0,,,,"N","N" -"2019-09-04 15:45:51","198.123.245.233",5678,,"rdp",,5678,"AA","LOCATION","LOCATION","RDP",5678,"RAMBLA01.rambla.local","RAMBLA01.rambla.local","2019-04-16 06:15:20","2019-10-16 06:15:20","7A:67:1F:F8:87:C6:B0:AC:A9:84:15:B7:40:EC:CB:19:AA:E3:19:52","3FF3EBC5CF154BA54D128A8548C8AAF5",2,"sha1WithRSAEncryption","rsaEncryption","8F:CD:7D:C4:80:2D:8D:9B:06:A0:40:18:9F:ED:73:7A:BA:83:55:BE:1B:56:83:A2:97:DF:BB:B4:06:57:CB:F1","E8:9B:9A:93:69:B4:58:01:D8:46:C2:DC:01:20:1E:DD:93:E1:EB:E3:9D:6B:65:A0:C5:00:6C:A4:44:08:FE:A4:A6:19:FF:55:79:F2:AA:61:68:C8:1C:B0:CE:78:EB:84:DD:29:9D:64:2F:4E:25:31:3A:6C:B8:02:C9:AF:F5:1F","38:73:6A:B3:AA:41:69:C9:BA:E7:3D:D7:40:16:F8:AA",517311,0,"Information Technology",,,"N","N" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv deleted file mode 100644 index 73d0d55efd..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sessionid","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,3389,node01.example.com,rdpeudp,64512,ZZ,Region,City,0,0,05b28c0c,1232,77.00 -"2010-02-10 00:00:01",192.168.0.2,udp,3389,node02.example.com,rdpeudp,64512,ZZ,Region,City,0,0,053d355f,1232,77.00 -"2010-02-10 00:00:02",192.168.0.3,udp,3389,node03.example.com,rdpeudp,64512,ZZ,Region,City,0,0,0567a8cb,1232,77.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv deleted file mode 100644 index dc9760cf2d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv +++ /dev/null @@ -1,94 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","git_sha1","git_dirty_flag","build_id","mode","os","architecture","multiplexing_api","gcc_version","process_id","run_id","uptime","connected_clients","sector" -"2016-07-24 00:42:33","198.51.100.152","tcp",6379,,"redis","2.8.19",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"26069fb482f6334b","standalone","Linux 3.2.0-4-amd64 x86_64",,"epoll","4.7.2",2127,"d440b0b2fb3d1db655ad607e11e6f38011a0f599",27946314,50, -"2016-07-24 00:42:43","198.51.100.67","tcp",6379,"198-51-100-67.example.net","redis","3.0.6",12586,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",310556,25376, -"2016-07-24 00:42:43","198.51.100.125","tcp",6379,"198-51-100-125.example.net","redis","2.8.17",8972,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"5937320cdd80c1e4","standalone","Linux 2.6.32-5-amd64 x86_64",,"epoll","4.9.2",11573,"0d58143df099738a7ce9330ee5ec2367d11b1187",25888041,4, -"2016-07-24 00:42:43","198.51.100.203","tcp",6379,"198-51-100-203.example.net","redis","2.8.4",31103,"DE","THURINGEN","ERFURT",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.13.0-83-generic x86_64",,"epoll","4.8.2",3847,"4f7765dee91d8c4b1b24604cc5f0c29fca1a4f32",3068554,38, -"2016-07-24 00:42:43","198.51.100.240","tcp",6379,"198-51-100-30.example.net","redis","3.0.7",20473,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"869e89100d5ea8c2","standalone","Linux 3.13.0-87-generic x86_64",,"epoll","4.8.4",1011,"864c8d7df1e72c662a4edd77b6df6cd30161af6e",2476542,2,"Information Technology" -"2016-07-24 00:42:49","198.51.100.69","tcp",6379,"198-51-100-69.example.net","redis","3.0.6",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"315c8c74805fca88","standalone","Linux 3.2.0-98-generic x86_64",,"epoll","4.6.3",28961,"bc705102c854ea1818213e4740a3c6fd9b9f1716",4633191,1, -"2016-07-24 00:42:53","198.51.100.50","tcp",6379,"198-51-100-50.example.net","redis","3.0.7",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"6afb1e1f0d80abd0","standalone","Linux 2.6.32-5-amd64 x86_64",,"epoll","4.4.5",1717,"f729595b3642b48f3ac9e098bcccab1d6ef82e3e",6345372,3, -"2016-07-24 00:43:49","198.51.100.113","tcp",6379,,"redis","3.0.6",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",310623,24628, -"2016-07-24 00:43:49","198.51.100.228","tcp",6379,"198-51-100-131.example.net","redis","2.8.210",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"83ad777ec89a946b","standalone","Windows",,"winsock_IOCP",,1948,"f5d6ad26e423039636afaf3918ee7e6a7e0b5b68",2214134,4,"Information Technology" -"2016-07-24 00:43:59","198.51.100.155","tcp",6379,,"redis","3.0.7",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"f09a0843cc9876c3","standalone","Linux 3.13.0-79-generic x86_64",,"epoll","4.9.2",1,"5f4f5b7158f928cc96e3ae6af6092a163ace15eb",2897902,24, -"2016-07-24 00:43:59","198.51.100.171","tcp",6379,,"redis","3.0.6",12586,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",310633,25031, -"2016-07-24 00:44:09","198.51.100.230","tcp",6379,"198-51-100-230.example.net","redis","2.8.11",8560,"DE","BADEN-WURTTEMBERG","KARLSRUHE",0,0,00000000,0,"f26bfdf4b8265fc","standalone","Linux 3.8.0-29-generic x86_64",,"epoll","4.6.3",14551,"0d001175cf26cee88486d814b4f0c972a5aa89b9",21038337,9, -"2016-07-24 00:44:09","198.51.100.182","tcp",6379,"198-51-100-182.example.net","redis","3.0.7",197540,"DE","BADEN-WURTTEMBERG","KARLSRUHE",0,0,00000000,0,"fd24f54fec00684b","standalone","Linux 3.13.0-85-generic x86_64",,"epoll","4.8.4",949,"b11fdf2b95251b8e6c3e9e782409ef82fc8b89aa",8643389,11, -"2016-07-24 00:44:10","198.51.100.23","tcp",6379,"198-51-100-116.example.net","redis","2.8.4",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 4.2.0-27-generic x86_64",,"epoll","4.8.2",335,"90079d58e970a1ae94aa91bc0ea0236a0e55269c",4930922,2,"Information Technology" -"2016-07-24 00:44:19","198.51.100.51","tcp",6379,"198-51-100-51.example.net","redis","3.0.6",12586,"DE","BERLIN","BERLIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",310652,26257, -"2016-07-24 00:44:22","198.51.100.88","tcp",6379,,"redis","3.0.6",13301,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",310656,26371, -"2016-07-24 00:44:22","198.51.100.107","tcp",6379,"octopus-dev","redis","2.8.14",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"78be6d5e32e34139","standalone","Linux 2.6.32-042stab108.2 x86_64",,"epoll","4.8.2",21205,"b98a41b6ea690c207527587f60bff1f1d24236b4",9364864,4, -"2016-07-24 00:44:22","198.51.100.75","tcp",6379,,"redis","3.0.0",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"2b5201a6bfd5f75e","standalone","Linux 3.11.0-19-generic x86_64",,"epoll","4.8.2",832,"2bdcda8b3b59cef244785b58935d68daf48645be",6745479,5, -"2016-07-24 00:44:25","198.51.100.12","tcp",6379,,"redis","3.0.6",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.13.0-79-generic x86_64",,"epoll","4.8.4",899,"94550e510bf770aa315cc3983ce9958853c77cfe",7816856,9, -"2016-07-24 00:44:27","198.51.100.13","tcp",6379,"198-51-100-13.example.net","redis","3.0.7",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"6f8b503a2787e3a6","standalone","Linux 4.4.5-15.26.amzn1.x86_64 x86_64",,"epoll","4.9.2",1,"e050f40e755a739ffecdb2468e1333f371e2abca",7124048,6,"Communications" -"2016-07-24 00:44:29","198.51.100.12","tcp",6379,"198-51-100-12.example.net","redis","2.8.3",8972,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"992c97be25a6b6d2","standalone","Linux 2.6.32-042stab111.12 x86_64",,"epoll","4.4.5",12340,"d7cda18212cf4bcdfd7c42fff33e506a4e9a2614",16874891,8, -"2016-07-24 00:44:38","198.51.100.66","tcp",6379,"198-51-100-66.example.net","redis","3.2.1",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"ed627d97d5dc311e","standalone","Linux 4.2.0-19-generic x86_64",,"epoll","4.9.2",1,"4a6beb721ddbaa411f53e5268e6112127903cae3",2029470,3,"Chemical" -"2016-07-24 00:44:38","198.51.100.170","tcp",6379,,"redis","3.0.6",8881,"DE","SACHSEN","RADEBEUL",0,0,00000000,0,"1b14d17ce6fea422","standalone","Linux 4.2.6-1-pve x86_64",,"epoll","4.9.2",728,"c423ba856285690a2fae350b03514cec80db9d5e",1679635,1, -"2016-07-24 00:44:38","198.51.100.67","tcp",6379,"198-51-100-67.example.net","redis","2.8.23",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"8e819a849ea2d7f8","standalone","Linux 4.2.0-23-generic x86_64",,"epoll","4.9.2",1,"7ee1dc403540ff4d1fc0a80d9f0b2910857b6c1b",9451832,68,"Information Technology" -"2016-07-24 00:44:44","198.51.100.238","tcp",6379,,"redis","2.8.4",44066,"DE","BAYERN","MUNICH",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 2.6.32-19-pve x86_64",,"epoll","4.8.2",2207,"6a079396cc44c1aca745edab13f4014c394da3ab",10338949,3, -"2016-07-24 00:44:44","198.51.100.84","tcp",6379,"198-51-100-84.example.net","redis","3.0.2",51862,"DE","BADEN-WURTTEMBERG","KARLSRUHE",0,0,00000000,0,"4795df119e2d77fe","standalone","Linux 3.13.0-91-generic x86_64",,"epoll","4.7.2",1,"c120481a551c232b8e1a9cff20d9e0968a402dd9",1040551,7, -"2016-07-24 00:44:44","198.51.100.23","tcp",6379,"198-51-100-23.example.net","redis","3.0.6",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"98c227055d7fa7b6","standalone","Linux 3.10.0-327.10.1.el7.x86_64 x86_64",,"epoll","4.8.5",35198,"424b15e04ce09f26299ff19b252a920916d4e4be",8875355,2, -"2016-07-24 00:44:47","198.51.100.160","tcp",6379,"198-51-100-160.example.net","redis","2.8.210",6724,"DE","BERLIN","BERLIN",0,0,00000000,0,"83ad777ec89a946b","standalone","Windows",,"winsock_IOCP",,2284,"9bde76afda6f81acfb241ea5ee3a9e878ad53881",742778,2, -"2016-07-24 00:44:47","198.51.100.111","tcp",6379,"198-51-100-98.example.net","redis","3.2.1",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"e19bb8c3d1c28291","standalone","Linux 3.10.0-327.22.2.el7.x86_64 x86_64",,"epoll","5.3.0",1,"c951371f430c1d94299bfc93759f6940d8bfce78",208557,2, -"2016-07-24 00:44:48","198.51.100.227","tcp",6379,"198-51-100-227.example.net","redis","3.0.6",12586,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",310681,26496, -"2016-07-24 00:44:54","198.51.100.18","tcp",6379,"198-51-100-18.example.net","redis","2.8.9",8972,"DE","NORDRHEIN-WESTFALEN","COLOGNE",0,0,00000000,0,"52c7b9284559eb20","standalone","Linux 2.6.32-5-amd64 x86_64",,"epoll","4.4.5",31887,"e5b1da35862482c4df8d4fce635ec89a36476a4d",14393072,6, -"2016-07-24 00:44:54","198.51.100.248","tcp",6379,"198-51-100-248.example.net","redis","3.0.6",12586,"DE","BERLIN","BERLIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",310687,26112, -"2016-07-24 00:44:57","198.51.100.228","tcp",6379,"198-51-100-228.example.net","redis","3.0.7",8972,"DE","NORDRHEIN-WESTFALEN","COLOGNE",0,0,00000000,0,"5e03212a543f54f8","standalone","Linux 3.13.0-042stab116.1 x86_64",,"epoll","4.8.4",719,"537e3e824a45414c3199ef20201b4362b752eeb5",1263367,2, -"2016-07-24 00:45:04","198.51.100.227","tcp",6379,"198-51-100-227.example.net","redis","2.8.12",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"ff040dde4a39b4ff","standalone","Windows",,"winsock_IOCP","0.0.0",1872,"c78751c65793a9a72f6fb0318efa532eb4fc87de",277953,18,"Chemical" -"2016-07-24 00:45:07","198.51.100.132","tcp",6379,,"redis","3.0.5",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"30405cba8f6c2d55","standalone","Linux 3.13.0-57-generic x86_64",,"epoll","4.8.4",2500,"10b4084b930d5a77e5f09e89cf0b21702027bd60",10028956,695, -"2016-07-24 00:46:10","198.51.100.47","tcp",6379,"198-51-100-185.example.net","redis","3.0.7",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"6a943c0b5bf37fa1","standalone","Linux 4.4.0-24-generic x86_64",,"epoll","5.3.1",1023,"de9c9c0da3d971f689bd7366c1edc93a00fd1506",2791106,1, -"2016-07-24 01:23:27","198.51.100.246","tcp",6379,"198-51-100-190.example.net","redis","2.8.19",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"665519ce00ddac9b","standalone","Linux 2.6.32-43-pve x86_64",,"epoll","4.9.2",2310,"94595838457eddb30a60184a9db66212268e6f82",9481199,4, -"2016-07-24 01:23:29","198.51.100.187","tcp",6379,"198-51-100-63.example.net","redis","2.8.19",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"c0359e7aa3798aa2","standalone","Linux 3.10.0-229.7.2.el7.x86_64 x86_64",,"epoll","4.8.3",14050,"e67a19de4bd2dc485b98ca353eb6fdc65e8fed4a",14051444,10, -"2016-07-24 01:23:29","198.51.100.228","tcp",6379,"198-51-100-228.example.net","redis","2.8.4",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.13.0-32-generic x86_64",,"epoll","4.8.2",22837,"daf5dba760d3db12716c6dc1d0bfe6d5e7b33749",10916038,8, -"2016-07-24 01:23:43","198.51.100.180","tcp",6379,"198-51-100-180.example.net","redis","3.2.1",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"ed627d97d5dc311e","standalone","Linux 4.2.0-19-generic x86_64",,"epoll","4.9.2",1,"569881874d8d5e1508d584a3fd9dff0ac3515839",1677711,1,"Chemical" -"2016-07-24 01:23:56","198.51.100.5","tcp",6379,"198-51-100-207.example.net","redis","3.0.7",20473,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"869e89100d5ea8c2","standalone","Linux 3.13.0-87-generic x86_64",,"epoll","4.8.4",1011,"864c8d7df1e72c662a4edd77b6df6cd30161af6e",2479015,2,"Information Technology" -"2016-07-24 01:24:03","198.51.100.226","tcp",6379,"198-51-100-226.example.net","redis","3.0.5",8972,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"b33bc3e2f8ad13f6","standalone","Linux 2.6.32-573.12.1.el6.x86_64 x86_64",,"epoll","4.4.7",1801,"7f4bb7ed008cdbd665672e88d57fc55616b6dbf2",13189200,9, -"2016-07-24 01:24:14","198.51.100.253","tcp",6379,"198-51-100-136.example.net","redis","2.8.4",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.19.0-39-generic x86_64",,"epoll","4.8.2",28272,"13a889aa846c6302dc8f5453e35e051a6f359e9a",14046610,185, -"2016-07-24 01:24:28","198.51.100.206","tcp",6379,,"redis","3.0.6",13301,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",313061,26695, -"2016-07-24 01:24:35","198.51.100.73","tcp",6379,,"redis","3.0.2",28753,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"6be7fc9e6b88f79","standalone","Linux 2.6.32-71.29.1.el6.x86_64 x86_64",,"epoll","4.4.7",811,"bbd4dce247ab51d029a64243810aeb900e00d1d6",10082205,15, -"2016-07-24 01:24:35","198.51.100.83","tcp",6379,"198-51-100-174.example.net","redis","3.2.1",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"7e7b61a55b95e8e7","standalone","Linux 4.2.0-41-generic x86_64",,"epoll","4.8.4",1076,"48f5f780ca53553fc4c0bbdbb32a5cb06a0551cd",814255,88,"Information Technology" -"2016-07-24 01:25:30","198.51.100.182","tcp",6379,,"redis","3.0.7",31400,"DE","RHEINLAND-PFALZ","FREINSHEIM",0,0,00000000,0,"d9ceac045f7983a9","standalone","FreeBSD 10.1-RELEASE-p26 amd64",,"kqueue","4.2.1",957,"48f37d15b3f5169f11aa5d7194fdfccc7f8df20b",6364747,1, -"2016-07-24 01:25:30","198.51.100.211","tcp",6379,"198-51-100-118.example.net","redis","2.8.17",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"e4968abcd4b78b2e","standalone","Linux 3.13.0-36-generic x86_64",,"epoll","4.8.2",1643,"665565b1b1fb6e773039707a0f680bbc417186be",20180649,4,"Information Technology" -"2016-07-24 01:25:35","198.51.100.249","tcp",6379,,"redis","3.0.2",28753,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"6be7fc9e6b88f79","standalone","Linux 2.6.32-71.29.1.el6.x86_64 x86_64",,"epoll","4.4.7",811,"bbd4dce247ab51d029a64243810aeb900e00d1d6",10082265,15, -"2016-07-24 01:25:40","198.51.100.55","tcp",6379,,"redis","3.2.1",3320,"DE","NORDRHEIN-WESTFALEN","SOLINGEN",518210,737415,00000000,0,"e19bb8c3d1c28291","standalone","Linux 4.4.0-24-generic x86_64",,"epoll","5.3.0",1,"49687ba2a5be7f7b6cdf0c837e06307442f6a369",494739,1, -"2016-07-24 01:25:42","198.51.100.62","tcp",6379,"198-51-100-62.example.net","redis","3.0.7",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"2b87841ee28adfc3","standalone","Linux 3.13.0-042stab113.11 x86_64",,"epoll","4.8.4",525,"4045d68fd2e59a1135bb303206d7cd0439ba7ffd",6971251,4, -"2016-07-24 01:25:55","198.51.100.127","tcp",6379,"198-51-100-25.example.net","redis","2.8.4",20473,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.13.0-87-generic x86_64",,"epoll","4.8.2",11492,"3de3e977405eef9392a77db4a50d99a5caa2f2d9",2194103,3,"Information Technology" -"2016-07-24 01:26:08","198.51.100.92","tcp",6379,"198-51-100-92.example.net","redis","2.8.10",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"5fce0c4aab65e01","standalone","Linux 2.6.32-042stab113.11 x86_64",,"epoll","4.6.3",490,"15abe68a10b011972f50d0abb3bb18f1735994a5",7505621,4, -"2016-07-24 01:26:17","198.51.100.218","tcp",6379,,"redis","3.0.7",34011,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"dc142e699f115c40","standalone","Linux 3.2.60-grsec-x86_64 x86_64",,"epoll","4.7.3",8006,"53a093bd4d0a7b72b2d084ec3767d23b18b8b947",4024979,7, -"2016-07-24 01:26:29","198.51.100.168","tcp",6379,"198-51-100-168.example.net","redis","3.0.6",51167,"DE","BAYERN","MUNICH",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.13.0-37-generic x86_64",,"epoll","4.8.4",1279,"8218bd77a0dcb0e00bd77dbb9478115757c70ba5",2405965,1, -"2016-07-24 01:26:29","198.51.100.155","tcp",6379,"198-51-100-155.example.net","redis","3.0.7",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"d9155128f7b25ea0","standalone","Linux 3.19.0-25-generic x86_64",,"epoll","4.8.4",27030,"0ede623cb268643672abc04d0267f684a5ee7a0d",6880190,5,"Information Technology" -"2016-07-24 01:26:34","198.51.100.185","tcp",6379,,"redis","2.8.4",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.13.0-43-generic x86_64",,"epoll","4.8.2",1196,"ae80fcbb54017f521212caf257418885cd6836a0",5412584,5, -"2016-07-24 01:26:34","198.51.100.1","tcp",6379,"198-51-100-1.example.net","redis","3.2.0",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"5382f69a4e75566b","standalone","Linux 4.2.0-19-generic x86_64",,"epoll","4.9.2",1,"ff8990f109ff5b2d4e0eee47e5ebc66acc43f9e3",4615889,4,"Chemical" -"2016-07-24 01:26:39","198.51.100.51","tcp",6379,"198-51-100-164.example.net","redis","3.0.0",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"9526f4809583faaa","standalone","Linux 2.6.32-042stab113.21 x86_64",,"epoll","4.4.5",14528,"d7271feff55175f434ace92d199f332ad35776a9",7440370,16, -"2016-07-24 01:26:44","198.51.100.138","tcp",6379,,"redis","3.0.6",12586,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",313197,26452, -"2016-07-24 01:26:47","198.51.100.16","tcp",6379,,"redis","2.8.17",25074,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"5937320cdd80c1e4","standalone","Linux 2.6.32-43-pve x86_64",,"epoll","4.9.2",266,"e1d403f2daff849a64b178f74c672db6712f217a",351253,1, -"2016-07-24 01:26:54","198.51.100.171","tcp",6379,"198-51-100-171.example.net","redis","3.0.6",12586,"DE","BERLIN","BERLIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",313207,26601, -"2016-07-24 01:27:14","198.51.100.89","tcp",6379,"198-51-100-89.example.net","redis","3.0.6",12586,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",313227,26358, -"2016-07-24 01:27:24","198.51.100.65","tcp",6379,,"redis","3.0.7",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"869e89100d5ea8c2","standalone","Linux 3.13.0-85-generic x86_64",,"epoll","4.8.4",21575,"3ec40168300e14f5776d82a48ba873a3999caec1",1897530,1, -"2016-07-24 01:27:24","198.51.100.248","tcp",6379,"198-51-100-248.example.net","redis","3.0.6",12586,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",313237,25902, -"2016-07-24 01:27:33","198.51.100.17","tcp",6379,,"redis","2.8.17",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"43dd9e14444e6aea","standalone","Linux 3.16.0-4-amd64 x86_64",,"epoll","4.9.2",556,"3e8fc2878511cc72f79b765fca86cefe21346912",2607965,72, -"2016-07-24 01:27:33","198.51.100.134","tcp",6379,"198-51-100-134.example.net","redis","3.0.7",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"6f8b503a2787e3a6","standalone","Linux 3.16.0-4-amd64 x86_64",,"epoll","4.9.2",1,"b85b2419cf35dd81ff5b9ba6e8bf802cf1d439f6",128621,33, -"2016-07-24 01:27:42","198.51.100.186","tcp",6379,"198-51-100-186.example.net","redis","2.8.13",34011,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"d588bf57ea0dfa69","standalone","Linux 4.4.8-jb1 i686",,"epoll","4.6.3",2460,"97b8d49e62d340d94a38c96c5104abfcacbfa4cb",181557,1, -"2016-07-24 01:27:42","198.51.100.21","tcp",6379,"198-51-100-21.example.net","redis","2.8.19",34011,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"920d7eda78149e99","standalone","Linux 4.4.8-x86_64-jb1 x86_64",,"epoll","4.7.2",3722,"74dfd8a7d87cbb9ecc590ceafd438c85d5073903",183984,1, -"2016-07-24 01:27:43","198.51.100.128","tcp",6379,"198-51-100-203.example.net","redis","3.0.5",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"f3bd5bc2b8b4c486","standalone","Linux 2.6.32-573.8.1.el6.x86_64 x86_64",,"epoll","4.4.7",1968,"0d92b1323fea791ba4b0a43435a156b6ec0aac1c",2967611,2,"Information Technology" -"2016-07-24 01:27:44","198.51.100.216","tcp",6379,"198-51-100-229.example.net","redis","2.8.4",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.16.0-30-generic x86_64",,"epoll","4.8.2",1470,"e76cd0cf25eec5d254c880965189ae011a119220",302420,1, -"2016-07-24 01:27:53","198.51.100.242","tcp",6379,"198-51-100-242.example.net","redis","3.0.2",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"6a04b5ede30cd4cd","standalone","Linux 3.13.0-32-generic x86_64",,"epoll","4.8.4",29725,"1b7e8dc53dec8fb29a8a2d76f516fd3dcb8df652",5815739,7, -"2016-07-24 01:27:53","198.51.100.54","tcp",6379,"198-51-100-54.example.net","redis","2.8.4",8560,"DE","BADEN-WURTTEMBERG","KARLSRUHE",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.13.0-91-generic x86_64",,"epoll","4.8.2",2903,"0e02514dec6031018eb148b13a4a9639cab3e8aa",905886,1, -"2016-07-24 01:27:54","198.51.100.225","tcp",6379,"198-51-100-225.example.net","redis","3.0.6",12586,"DE","BERLIN","BERLIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",313267,25281, -"2016-07-24 01:27:57","198.51.100.38","tcp",6379,"198-51-100-38.example.net","redis","3.0.5",6724,"DE","BERLIN","BERLIN",0,0,00000000,0,"3b863f97501297e9","standalone","Linux 3.13.0-042stab111.12 x86_64",,"epoll","4.8.4",2088,"31a8cececad2e4a33310a741143d85cdef3479b4",11906868,10, -"2016-07-24 01:27:58","198.51.100.22","tcp",6379,"198-51-100-22.example.net","redis","2.8.9",51167,"DE","BAYERN","MUNICH",0,0,00000000,0,"2ac6afaedfd3ea15","standalone","Linux 3.13.0-86-generic x86_64",,"epoll","4.8.4",9082,"8e5d9d74c86a9f148a7012733eb52a21938c3c04",5833880,5, -"2016-07-24 01:28:05","198.51.100.106","tcp",6379,"198-51-100-106.example.net","redis","2.8.19",36351,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"9968db13395be4aa","standalone","Windows",,"winsock_IOCP","0.0.0",4372,"89716352a10cd53b5c10e6d5e6cd1d46f5f53a30",485031,4,"Information Technology" -"2016-07-24 01:28:06","198.51.100.130","tcp",6379,"198-51-100-130.example.net","redis","2.8.3",51167,"DE","BAYERN","MUNICH",0,0,00000000,0,"542faa6f897d2236","standalone","Linux 2.6.32-573.3.1.el6.x86_64 x86_64",,"epoll","4.4.7",25531,"9d7606a883f764e744d766b7bf0036ba61f7fb6e",496133,5, -"2016-07-24 01:28:08","198.51.100.37","tcp",6379,"198-51-100-37.example.net","redis","2.8.23",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"50630e46be5feb4f","standalone","Linux 3.13.0-74-generic x86_64",,"epoll","4.9.2",1,"62d16be721c3c62d6c4d080a9bdbe9502c57ca86",3481683,9,"Communications" -"2016-07-24 01:28:32","198.51.100.148","tcp",6379,"198-51-100-148.example.net","redis","3.0.5",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"83dc15dcf8ee3eb8","standalone","Linux 4.1.7-15.23.amzn1.x86_64 x86_64",,"epoll","4.8.3",2304,"883accf76dc364c60902b4eab7861dd1a7eac71d",10981957,10,"Communications" -"2016-07-24 01:28:49","198.51.100.247","tcp",6379,"198-51-100-247.example.net","redis","3.0.7",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"3e971e94fbe2eaa6","standalone","Linux 3.2.0-4-amd64 x86_64",,"epoll","4.7.2",2535,"d223aab0621cdd2e4ab752978ad3009ad3814d8b",7715188,57, -"2016-07-24 02:08:46","198.51.100.220","tcp",6379,"198-51-100-220.example.net","redis","3.0.6",51167,"DE","BAYERN","MUNICH",0,0,00000000,0,"1f8e4c92f1ca309","standalone","Linux 3.13.0-74-generic x86_64",,"epoll","4.8.4",3355,"dd517756bb6ee81e1929fa605972318b2baebb93",5211978,10, -"2016-07-24 02:08:46","198.51.100.239","tcp",6379,"198-51-100-239.example.net","redis","2.8.23",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"83a5616190c5a1aa","standalone","Linux 3.16.0-4-amd64 x86_64",,"epoll","4.9.2",711,"4117960b13fa313b823c79b0e9f188d8ec6aa3ac",10156283,6, -"2016-07-24 02:08:50","198.51.100.233","tcp",6379,,"redis","2.8.11",8560,"DE","BADEN-WURTTEMBERG","KARLSRUHE",0,0,00000000,0,"f26bfdf4b8265fc","standalone","Linux 3.8.0-29-generic x86_64",,"epoll","4.6.3",14551,"0d001175cf26cee88486d814b4f0c972a5aa89b9",21043417,9, -"2016-07-24 02:08:51","198.51.100.208","tcp",6379,"198-51-100-181.example.net","redis","3.0.6",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 4.2.0-38-generic x86_64",,"epoll","4.8.4",809,"14c5ec7f9669e42ea45a40ff26a6501d593695c0",2405839,19, -"2016-07-24 02:08:51","198.51.100.60","tcp",6379,"198-51-100-60.example.net","redis","3.0.7",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"4ed99bd9c45dfc14","standalone","Linux 3.13.0-57-generic x86_64",,"epoll","4.8.4",1144,"9e28c29ff40017e2fbe32fb97755caf801f95793",843538,2, -"2016-07-24 02:08:51","198.51.100.107","tcp",6379,"198-51-100-39.example.net","redis","3.2.0",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"82b2619163aabc80","standalone","Linux 4.2.0-25-generic x86_64",,"epoll","4.9.2",1,"98f6640bbde04b1214730937212e1fd4e58d03a8",2195657,12, -"2016-07-24 02:08:54","198.51.100.31","tcp",6379,,"redis","2.8.4",6724,"DE","BERLIN","BERLIN",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.13.0-042stab111.12 x86_64",,"epoll","4.8.2",1112,"9c4e55b5ebd06045c5d89d43fa202e219ec8b42c",8839783,7, -"2016-07-24 02:08:56","198.51.100.221","tcp",6379,,"redis","3.0.7",44066,"DE","BAYERN","MUNICH",0,0,00000000,0,"49f951dce0725d71","standalone","FreeBSD 10.0-RELEASE-p7 amd64",,"kqueue","4.2.1",932,"28c6af3c4dedcd9b71cf51a7ebc4e84899196aee",8000949,1, -"2016-07-24 02:09:01","198.51.100.155","tcp",6379,"198-51-100-155.example.net","redis","2.8.22",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"fcdf45e47686c89b","standalone","Linux 3.13.0-57-generic x86_64",,"epoll","4.8.4",7,"946ec6b96fe9925d2b677ce02b6c56097c5e69a8",8449694,6, -"2016-07-24 02:09:02","198.51.100.219","tcp",6379,"198-51-100-219.example.net","redis","2.8.4",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.13.0-74-generic x86_64",,"epoll","4.8.2",1047,"9b83d6a6e7a6ffe50e75dac88cdc5e06f6203c9c",966148,1,"Chemical" -"2016-07-24 02:09:02","198.51.100.193","tcp",6379,"198-51-100-193.example.net","redis","3.0.7",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"fd640d8ef55a22dd","standalone","Linux 4.2.0-42-generic x86_64",,"epoll","4.8.4",1397,"ed5ec17d78d089af53afd4abc339f7decf4641d4",651175,2,"Information Technology" -"2016-07-24 02:09:20","198.51.100.120","tcp",6379,"198-51-100-120.example.net","redis","3.2.1",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"ed627d97d5dc311e","standalone","Linux 3.16.0-4-amd64 x86_64",,"epoll","4.9.2",1,"f524508ad29334eee2fcf7bdda5c80b9f99d3dfe",987580,167, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv deleted file mode 100644 index a61e4573ec..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","module","motd","has_password" -"2010-02-10 00:00:00",192.168.0.1,tcp,873,node01.example.com,rsync,64512,ZZ,Region,City,0,0,"system|Backup system;system_full|Backup full system;mysql|Backup virtual mysql;netadmin|Backup virtual netadmin;",,N -"2010-02-10 00:00:01",192.168.0.2,tcp,873,node02.example.com,rsync,64512,ZZ,Region,City,0,0,"system|Backup system;system_full|Backup full system;mysql|Backup virtual mysql;netadmin|Backup virtual netadmin;",,N -"2010-02-10 00:00:02",192.168.0.3,tcp,873,node03.example.com,rsync,64512,ZZ,Region,City,0,0,"system|Backup system;system_full|Backup full system;mysql|Backup virtual mysql;netadmin|Backup virtual netadmin;",,N diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv deleted file mode 100644 index ee0a625e55..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","sip","sip_code","sip_reason","user_agent","sip_via","sip_to","sip_from","content_length","content_type","server","contact","cseq","call_id","allow","amplification","response_size" -"2010-02-10 00:00:00",192.168.0.1,udp,5060,node01.example.com,sip,64512,ZZ,Region,City,SIP/2.0,489,"Event Package Not Supported",,,,,0,,,,,,"INVITE,ACK,BYE,CANCEL,REGISTER",15.57,109 -"2010-02-10 00:00:01",192.168.0.2,udp,5060,node02.example.com,sip,64512,ZZ,Region,City,SIP/2.0,400,"Bad Request",,,,,364,text/plain,,,,,,62.57,438 -"2010-02-10 00:00:02",192.168.0.3,udp,5060,node03.example.com,sip,64512,ZZ,Region,City,SIP/2.0,400,"Bad Request",,,,,0,,,,,,,6.57,46 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv.license deleted file mode 100644 index 9f58c89ef0..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2023 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv deleted file mode 100644 index 256dd78f60..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","version","function","function_text","flags","next_extension_offset","xid","language_tag_length","language_tag","error_code","error_code_text","response_size","raw_response" -"2010-02-10 00:00:00",192.168.0.1,tcp,427,node01.example.com,slp,64512,ZZ,Region,City,0,0,,2,2,"Service reply",0x0000,0,5,2,en,5,"Unsupported SLP SPI",40,MDIwMjAwMDAxNDAwMDAwMDAwMDAwMDA1MDAwMjY1NmUwMDA1MDAwMA== -"2010-02-10 00:00:01",192.168.0.2,tcp,427,node02.example.com,slp,64512,ZZ,Region,City,0,0,,2,2,"Service reply",0x0000,0,5,2,en,5,"Unsupported SLP SPI",40,MDIwMjAwMDAxNDAwMDAwMDAwMDAwMDA1MDAwMjY1NmUwMDA1MDAwMA== -"2010-02-10 00:00:02",192.168.0.3,tcp,427,node03.example.com,slp,64512,ZZ,Region,City,0,0,,2,2,"Service reply",0x0000,0,5,2,en,5,"Unsupported SLP SPI",40,MDIwMjAwMDAxNDAwMDAwMDAwMDAwMDA1MDAwMjY1NmUwMDA1MDAwMA== diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv.license deleted file mode 100644 index 9f58c89ef0..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2023 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv deleted file mode 100644 index fc7fe2fff6..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","port","hostname","tag","asn","geo","region","city","naics","sic","smb_implant","arch","key","smbv1_support","smb_major_number","smb_minor_number","smb_revision","smb_version_string" -"2010-02-10 00:00:00",192.168.0.1,445,node01.example.com,smb,64512,ZZ,Region,City,0,0,N,,,N,2,1,0,"SMB 2.1" -"2010-02-10 00:00:01",192.168.0.2,445,node02.example.com,smb,64512,ZZ,Region,City,0,0,N,,,N,2,1,0,"SMB 2.1" -"2010-02-10 00:00:02",192.168.0.3,445,node03.example.com,smb,64512,ZZ,Region,City,0,0,N,,,N,2,1,0,"SMB 2.1" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv deleted file mode 100644 index 19eb560538..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","banner" -"2021-07-08 11:58:42","1.2.3.4","tcp",25,"smtp-server.invalid","smtp;21nails",12345,"EE","HARJUMAA","TALLINN",,,"220 smtp-server.invalid ESMTP Exim 4.80 Wed, 11 Jun 2021 10:00:00 +0300|" -"2021-07-08 11:58:44","5.6.7.8","tcp",25,"smtp-out.invalid","smtp;21nails",23456,"EE","HARJUMAA","TALLINN",,,"220 smtp-out.invalid, ESMTP EXIM 4.86_2|" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv.license deleted file mode 100644 index c1900637ff..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Mikk Margus Möll -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv deleted file mode 100644 index f489261c42..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","sysdesc","sysname","asn","geo","region","city","version","naics","sic","sector","device_vendor","device_type","device_model","device_version","device_sector","tag","community","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,161,node01.example.com,"Linux localhost 3.18.20 #1 SMP Mon Jul 9 14:11:21 CST 2018 armv7l",,64512,ZZ,Region,City,2,0,0,,,,,,,snmp,public,165,1.94 -"2010-02-10 00:00:01",192.168.0.2,udp,161,node02.example.com,"RouterOS CCR1009-8G-1S-1S+",,64512,ZZ,Region,City,2,0,0,,MikroTik,router,,,consumer,"snmp,iot",public,115,1.35 -"2010-02-10 00:00:02",192.168.0.3,udp,161,node03.example.com,,,64512,ZZ,Region,City,2,0,0,,,,,,,snmp,public,85,1.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv deleted file mode 100644 index c591a5c099..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector" -"2010-02-10 00:00:00",192.168.0.1,tcp,1080,node01.example.com,socks4,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service" -"2010-02-10 00:00:01",192.168.0.2,tcp,1080,node02.example.com,socks5,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service" -"2010-02-10 00:00:02",192.168.0.3,tcp,1080,node03.example.com,socks4,64512,ZZ,Region,City,0,0,"Retail Trade" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv deleted file mode 100644 index 460be32c50..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","header","asn","geo","region","city","systime","cache_control","location","server","search_target","unique_service_name","host","nts","nt","content_type","naics","sic","sector","server_port","instance","version","updated_at","resource_identifier","amplification","response_size" -"2010-02-10 00:00:00",192.168.0.1,udp,60194,node01.example.com,ssdp,"HTTP/1.1 200 OK",64512,ZZ,Region,City,"Sun, 21 Aug 2022 09:51:13 GMT",max-age=100,http://192.168.200.254:49152/description.xml,"Linux/2.6.26, UPnP/1.0, Portable SDK for UPnP devices/1.3.1",upnp:rootdevice,uuid:28802880-2880-1880-a880-001bc502f600::upnp:rootdevice,node01.example.com,,,,0,0,Government,,,,,,3.35,325 -"2010-02-10 00:00:01",192.168.0.2,udp,38732,node02.example.com,ssdp,"HTTP/1.1 200 OK",64512,ZZ,Region,City,,"max-age = 1800",http://95.160.216.14:52235/dmr/SamsungMRDesc.xml,"Linux/9.0 UPnP/1.0 PROTOTYPE/1.0",upnp:rootdevice,uuid:f144ca92-6816-94b5-b95f-b58180834044::upnp:rootdevice,node02.example.com,,,,0,0,,,,,,,2.71,263 -"2010-02-10 00:00:02",192.168.0.3,udp,57626,node03.example.com,ssdp,"HTTP/1.1 200 OK",64512,ZZ,Region,City,"Sun, 03 Jan 2016 21:37:50 GMT",max-age=1800,http://192.168.1.3:8008/ssdp/device-desc.xml,"Linux/3.10.79, UPnP/1.0, Portable SDK for UPnP devices/1.6.18",upnp:rootdevice,uuid:62fa0fc8-079d-d00f-2e22-59b49fb488f9::upnp:rootdevice,node03.example.com,,,,0,0,Government,,,,,,4.79,465 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv deleted file mode 100644 index 837adbad10..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","serverid_raw","serverid_version","serverid_software","serverid_comment","server_cookie","available_kex","available_ciphers","available_mac","available_compression","selected_kex","algorithm","selected_cipher","selected_mac","selected_compression","server_signature_value","server_signature_raw","server_host_key","server_host_key_sha256","rsa_prime","rsa_prime_length","rsa_generator","rsa_generator_length","rsa_public_key","rsa_public_key_length","rsa_exponent","rsa_modulus","rsa_length","dss_prime","dss_prime_length","dss_generator","dss_generator_length","dss_public_key","dss_public_key_length","dss_dsa_public_g","dss_dsa_public_p","dss_dsa_public_q","dss_dsa_public_y","ecdsa_curve25519","ecdsa_curve","ecdsa_public_key_length","ecdsa_public_key_b","ecdsa_public_key_gx","ecdsa_public_key_gy","ecdsa_public_key_n","ecdsa_public_key_p","ecdsa_public_key_x","ecdsa_public_key_y","ed25519_curve25519","ed25519_cert_public_key_nonce","ed25519_cert_public_key_bytes","ed25519_cert_public_key_raw","ed25519_cert_public_key_sha256","ed25519_cert_public_key_serial","ed25519_cert_public_key_type_id","ed25519_cert_public_key_type_name","ed25519_cert_public_key_keyid","ed25519_cert_public_key_principles","ed25519_cert_public_key_valid_after","ed25519_cert_public_key_valid_before","ed25519_cert_public_key_duration","ed25519_cert_public_key_sigkey_bytes","ed25519_cert_public_key_sigkey_raw","ed25519_cert_public_key_sigkey_sha256","ed25519_cert_public_key_sigkey_value","ed25519_cert_public_key_sig_raw","banner","userauth_methods","device_vendor","device_type","device_model","device_version","device_sector" -"2022-01-10 02:20:37","18.179.0.0","tcp",22,"ec2-18-179-0-0.ap-northeast-1.compute.amazonaws.com","ssh",16509,"JP","TOKYO","TOKYO",454110,,"SSH-2.0-OpenSSH_7.4","2.0","OpenSSH_7.4",,"bGjsifbPIDWT7tAu8BMjyg==","curve25519-sha256, curve25519-sha256@libssh.org, ecdh-sha2-nistp256, ecdh-sha2-nistp384, ecdh-sha2-nistp521, diffie-hellman-group-exchange-sha256, diffie-hellman-group16-sha512, diffie-hellman-group18-sha512, diffie-hellman-group-exchange-sha1, diffie-hellman-group14-sha256, diffie-hellman-group14-sha1, diffie-hellman-group1-sha1","chacha20-poly1305@openssh.com, aes128-ctr, aes192-ctr, aes256-ctr, aes128-gcm@openssh.com, aes256-gcm@openssh.com, aes128-cbc, aes192-cbc, aes256-cbc, blowfish-cbc, cast128-cbc, 3des-cbc","umac-64-etm@openssh.com, umac-128-etm@openssh.com, hmac-sha2-256-etm@openssh.com, hmac-sha2-512-etm@openssh.com, hmac-sha1-etm@openssh.com, umac-64@openssh.com, umac-128@openssh.com, hmac-sha2-256, hmac-sha2-512, hmac-sha1","none, zlib@openssh.com","curve25519-sha256@libssh.org","ecdsa-sha2-nistp256","aes128-ctr","hmac-sha2-256","none","AAAAIQCd+X/B/OEx+FrwJSlVecOvNMuS5w2vTRz0z4prM+5VBwAAACEArU60b9CHs/d5BgyaOd7vmFygTMK5SyL90bS8VIztX/4=","AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAABKAAAAIQCd+X/B/OEx+FrwJSlVecOvNMuS5w2vTRz0z4prM+5VBwAAACEArU60b9CHs/d5BgyaOd7vmFygTMK5SyL90bS8VIztX/4=","AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDSEHaLacthwB30rtA4xJgN3G9zXkCmm2WhV/TlNBrD20fuNQAZX7XciX2YkqIHtK2dWLBYwVCCqvl//zoM42kI=","a6e4e1c16ba25d51bcddc58a6e16797144575dd18d02d9dedf75093d2b15c557",,,,,,,,,,,,,,,,,,,,"1xx7ASut7BF4ED8b592bebZBMBKTCzOsmbH4cjwx/0U=","P-256",256,"WsY12Ko6k+ez671VdpiGvGUdBrDMU7D2O848PifSYEs=","axfR8uEsQkf4vOblY6RA8ncDfYEt6zOg9KE5RdiYwpY=","T+NC4v4af5uO5+tKfA+eFivOM1drMV7Oy7ZAaDe/UfU=","/////wAAAAD//////////7zm+q2nF56E87nKwvxjJVE=","/////wAAAAEAAAAAAAAAAAAAAAD///////////////8=","NIQdotpy2HAHfSu0DjEmA3cb3NeQKabZaFX9OU0GsPY=","0fuNQAZX7XciX2YkqIHtK2dWLBYwVCCqvl//zoM42kI=",,,,,,,,,,,,,,,,,,,,"publickey",,,,, -"2022-01-10 02:20:37","170.10.0.0","tcp",22,"170-10-0-0.example.com","ssh",11976,"US","TEXAS","MARSHALL",,,"SSH-2.0-ARRIS_0.50","2.0","ARRIS_0.50",,"Y4RQS9sdRgEFwNJKVP6bZg==","diffie-hellman-group1-sha1","aes128-cbc, 3des-cbc, aes256-cbc, twofish256-cbc, twofish-cbc, twofish128-cbc, blowfish-cbc","hmac-sha1-96, hmac-sha1, hmac-md5","none","diffie-hellman-group1-sha1","ssh-rsa","aes128-cbc","hmac-sha1","none","LQj+UTJEQqdb/p/c/19yVc63eo+rnedwXKjP6eNNxxijN2cFoOjVMeqT2QTBjyoN7yRWBU2EID+3y2jUYT8mCqmqfyUv1eEbiCfLVlUyQ0X/CY9I5DDb5l6yEjNkuH2xVNNV6R7GFRwyYKAsYzfy+i9o1OORlUh3tozkkPfA9z/NlA==","AAAAB3NzaC1yc2EAAACCLQj+UTJEQqdb/p/c/19yVc63eo+rnedwXKjP6eNNxxijN2cFoOjVMeqT2QTBjyoN7yRWBU2EID+3y2jUYT8mCqmqfyUv1eEbiCfLVlUyQ0X/CY9I5DDb5l6yEjNkuH2xVNNV6R7GFRwyYKAsYzfy+i9o1OORlUh3tozkkPfA9z/NlA==","AAAAB3NzaC1yc2EAAAADAQABAAAAgwCDq1kGqqwdQVryCNcoyDbBpnL/okvM2d9NmR0OjprcToCZ2TZ5WUZt2BGwPE1QLJYskjhv7GwlfQ4qhEqHDg35wMrkO7j9LTQC7KW3xisOLuUil4FmMxPkol6s39945zBGpjw0l/BmJnUDlutxavkdd84fppFMwXNp2vbjxV1SYVc9","d53fedbfe92e631264629882b2e85bfd213ca4b07b824cd31f8de1fcb8d0ddcb",,,,,,,65537,"g6tZBqqsHUFa8gjXKMg2waZy/6JLzNnfTZkdDo6a3E6Amdk2eVlGbdgRsDxNUCyWLJI4b+xsJX0OKoRKhw4N+cDK5Du4/S00Auylt8YrDi7lIpeBZjMT5KJerN/feOcwRqY8NJfwZiZ1A5brcWr5HXfOH6aRTMFzadr248VdUmFXPQ==",1040,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"publickey, password","Arris",,,, -"2022-01-10 02:20:37","72.17.0.0","tcp",22,"072-017-0-0.example.com","ssh",33363,"US","FLORIDA","ORLANDO",517311,,"SSH-1.99-Cisco-1.25","1.99","Cisco-1.25",,"Z2fOfWsrLlh76Y0bOqa1cw==","diffie-hellman-group-exchange-sha1, diffie-hellman-group14-sha1, diffie-hellman-group1-sha1","aes128-cbc, 3des-cbc, aes192-cbc, aes256-cbc","hmac-sha1, hmac-sha1-96, hmac-md5, hmac-md5-96","none","diffie-hellman-group14-sha1","ssh-rsa","aes128-cbc","hmac-sha1","none","lrzL2DY9fVvwYg6CgB75uf2s8CLo+rL8Mp9tU1Ja3sDfBzj9QJjVDykupiy8s3usHfxMrHS2v3DhTiZjz/b5K6tVTgUBTXL94JfM4lwB+3EbLggPzKnlm1jQgnnU9c+tb7RX3IhBqU9Yj1gqxhErv9NFotgajQOOLgY0Ua5C0Ee+AIaMlLaNZe3LTejMsNUZMN5tl+sEmtutMHkGQsmjJxiJ3feF+Pys0I2+ojiiAfzqlMYar/5xOPl4Dj+HO+h91xVQ1/8nQRBc082fM7+ZJtDbRLtt4G8srlB5gew26jqfVASc/ui5gx4+BR9DG9VH8w+rJWBGfhOAaWqLFE2M3YuEWkjEmQMR1SQK1WFQ/oNiWJO2K5L3rk2LcAmyR6nQMtClVxYZ7CQOwa3uFL+JNXp9AhiiAtVaqhrEK81NJrJNh/+egTBl5STphxIShXd4KI9wyvkGlCIvNIMO94iXPVaWUXXbsGnU03+dsUkBzGf0eJ4DePInCk/RtunlSmOsjGld+rpS9g0VRxPrzbQRWuhpkgpV+CldyrI3C/rOxJRs2vSAKXocRsGwhqEKseAJzHXmiZ5ncsaGKoeB5lUkWLwcKjyok2tHVCDlzDUpE4aA/JHNEhT48det9RqtjC71yz8m0PeK2ySI/I+Qb7eBgevgduBmt+OUxgvfKi2UB6s=","AAAAB3NzaC1yc2EAAAIAlrzL2DY9fVvwYg6CgB75uf2s8CLo+rL8Mp9tU1Ja3sDfBzj9QJjVDykupiy8s3usHfxMrHS2v3DhTiZjz/b5K6tVTgUBTXL94JfM4lwB+3EbLggPzKnlm1jQgnnU9c+tb7RX3IhBqU9Yj1gqxhErv9NFotgajQOOLgY0Ua5C0Ee+AIaMlLaNZe3LTejMsNUZMN5tl+sEmtutMHkGQsmjJxiJ3feF+Pys0I2+ojiiAfzqlMYar/5xOPl4Dj+HO+h91xVQ1/8nQRBc082fM7+ZJtDbRLtt4G8srlB5gew26jqfVASc/ui5gx4+BR9DG9VH8w+rJWBGfhOAaWqLFE2M3YuEWkjEmQMR1SQK1WFQ/oNiWJO2K5L3rk2LcAmyR6nQMtClVxYZ7CQOwa3uFL+JNXp9AhiiAtVaqhrEK81NJrJNh/+egTBl5STphxIShXd4KI9wyvkGlCIvNIMO94iXPVaWUXXbsGnU03+dsUkBzGf0eJ4DePInCk/RtunlSmOsjGld+rpS9g0VRxPrzbQRWuhpkgpV+CldyrI3C/rOxJRs2vSAKXocRsGwhqEKseAJzHXmiZ5ncsaGKoeB5lUkWLwcKjyok2tHVCDlzDUpE4aA/JHNEhT48det9RqtjC71yz8m0PeK2ySI/I+Qb7eBgevgduBmt+OUxgvfKi2UB6s=","AAAAB3NzaC1yc2EAAAADAQABAAACAQDIVXBwKGhi35gabwHNZi6Bxls1BGtDVVZFhwvhTpJKTKhV4T2HnDFG7+FBpYejc92wH026Wf+uJHIpnKkVQRnnOV98zKXW68Tz+OnwT8aBQdLI+QYDC7wLwGRf+cOiXEAkpMrp2OJme+GwQ97oBccEwdu2j9vcYAFQ0+eCPNfwPrcZhwVb00kt/moLVSxWRdsDMzQiNDZf2zel+FQIAl5cCfaLSAQa1TIXy8SM13B0brnlpdyIqukQS0zUv/PL/6AsfhgLXeQBgjs1XIf6qL+ZdtQss5AKUDuJgrWDcS3nyNZQg/CAt8XdIsLntu3bCn+VGA1O/gUGLS1a9GoGd/lRArlmODNtbds74m7hxaAf/gzg0LFJx6HhwubmVCzTXEHl95KHYHKoDvCtUOgUm7zUugxWjhsLPfT6UfZCwvCY21SGVYsoEPiTT2DhuAFriM+PT83JresFHgZDosbqW0VCi2bzAKSBu/vphaqTbSdDo0xhkW9JCb3zUkW2ge/e/GrjxV4cNXRC9XQ/XYEIWmtF/gHSi0i9KweX4sN5TEkB/41vDvyDOdyPJ8Jta0I9vBolDwJ6qdMHOPlOW5oW83yCgbmUJNYkZ+MivABlc6iS/006qYiIwknHezbY5foYd8kDON7YAssOwCJcG5viII50Z1N9VsGkUv5sZMr2p9ry8Q==","06ff3cce443ed832927576d982b69d5a526d0e63334c72e87201deda61679406",,,,,,,65537,"yFVwcChoYt+YGm8BzWYugcZbNQRrQ1VWRYcL4U6SSkyoVeE9h5wxRu/hQaWHo3PdsB9Nuln/riRyKZypFUEZ5zlffMyl1uvE8/jp8E/GgUHSyPkGAwu8C8BkX/nDolxAJKTK6djiZnvhsEPe6AXHBMHbto/b3GABUNPngjzX8D63GYcFW9NJLf5qC1UsVkXbAzM0IjQ2X9s3pfhUCAJeXAn2i0gEGtUyF8vEjNdwdG655aXciKrpEEtM1L/zy/+gLH4YC13kAYI7NVyH+qi/mXbULLOQClA7iYK1g3Et58jWUIPwgLfF3SLC57bt2wp/lRgNTv4FBi0tWvRqBnf5UQK5ZjgzbW3bO+Ju4cWgH/4M4NCxSceh4cLm5lQs01xB5feSh2ByqA7wrVDoFJu81LoMVo4bCz30+lH2QsLwmNtUhlWLKBD4k09g4bgBa4jPj0/Nya3rBR4GQ6LG6ltFQotm8wCkgbv76YWqk20nQ6NMYZFvSQm981JFtoHv3vxq48VeHDV0QvV0P12BCFprRf4B0otIvSsHl+LDeUxJAf+Nbw78gzncjyfCbWtCPbwaJQ8CeqnTBzj5TluaFvN8goG5lCTWJGfjIrwAZXOokv9NOqmIiMJJx3s22OX6GHfJAzje2ALLDsAiXBub4iCOdGdTfVbBpFL+bGTK9qfa8vE=",4096,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"publickey, keyboard-interactive, password","Cisco",,,,"enterprise" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv deleted file mode 100644 index 0b125001be..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","port","hostname","tag","handshake","asn","geo","region","city","cipher_suite","ssl_poodle","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serial_number","naics","sic","freak_vulnerable","freak_cipher_suite","sector","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","http_response_type","http_code","http_reason","content_type","http_connection","www_authenticate","set_cookie","server_type","content_length","transfer_encoding","http_date","cert_valid","self_signed","cert_expired","browser_trusted","validation_level","browser_error","tlsv13_support","tlsv13_cipher","jarm" -"2022-01-10 00:01:42","96.60.0.0",10443,"96-60-0-0.example.com","ssl,vpn","TLSv1.2",4181,"US","WISCONSIN","MILWAUKEE","TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA","N",1024,"FGT60D4614030700","support","2014-06-23 09:56:32","2038-01-19 03:14:07","5A:3D:FF:06:F9:E9:25:37:57:F9:09:52:33:A4:85:15:24:2D:88:7F","168CAE",2,"sha1WithRSAEncryption","rsaEncryption","Fortinet","FortiGate","US","California","Sunnyvale",,,,,"support@fortinet.com",,,"Fortinet","Certificate Authority","US","California","Sunnyvale",,,,,"support@fortinet.com",,,517311,,"N",,,"35:AB:B6:76:2A:3D:17:B2:FB:40:45:1B:FC:0A:99:0A:6E:48:57:F7:30:0A:3B:B1:1A:E6:99:70:5B:7C:32:41","88:7B:16:DB:39:44:0C:47:0E:4A:8F:0B:C5:FB:4D:45:BC:93:5A:00:43:A1:D9:7F:05:1D:86:33:02:F8:FC:57:67:A6:1D:C0:FF:F7:D2:40:D8:9A:21:AE:4E:6D:DC:E7:FF:72:BF:13:CB:EE:A7:5F:CD:83:EA:8A:5E:FB:87:DD","99:45:1F:2E:AE:EB:88:91:27:43:33:79:FA:93:7D:CA","HTTP/1.1",200,"OK","text/html",,,,"xxxxxxxx-xxxxx",131,,"Mon, 10 Jan 2022 00:01:44 GMT","Y","N","N","N","unknown","x509: unknown error",,, -"2022-01-10 00:01:42","113.160.0.0",10443,"","ssl","TLSv1.2",45899,"VN","THAI BINH","THAI BINH","TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256","N",2048,"1078-btb-tbi-HungHa-61d39c6d5a7e2","1078-btb-tbi-HungHa-61d39c6d5a7e2","2022-01-04 01:01:34","2023-02-06 01:01:34","A9:00:BB:E1:54:4D:56:54:59:F1:B7:EA:F1:1A:D5:36:5C:63:90:8E","36974C4C6B1B3785",2,"sha256WithRSAEncryption","rsaEncryption","pfSense webConfigurator Self-Signed Certificate",,,,,,,,,,,,"pfSense webConfigurator Self-Signed Certificate",,,,,,,,,,,,517311,,"N",,,"38:85:F0:44:1E:AD:84:B8:2F:43:68:BA:AC:EE:17:13:A4:BF:86:1D:48:75:7E:22:FA:08:4C:28:5F:AC:3E:5F","AE:1B:4F:D1:E4:C0:35:9D:2A:4F:7A:37:B8:7B:11:9D:84:25:23:21:AB:EF:B2:0F:DC:C9:F2:A3:72:28:92:E1:74:72:FA:E1:09:6C:E1:F6:B6:E3:A7:61:1C:58:89:34:D7:06:5C:3D:0A:A7:F6:CC:8A:D6:24:D0:04:4C:03:02","16:93:9A:F4:35:7F:9A:85:45:71:91:C7:7C:80:88:00","HTTP/1.1",200,"OK","text/html; charset=UTF-8","keep-alive",,"PHPSESSID=e15bdfa5739c36877608eb4cf46cc388; path=/; secure; HttpO","nginx",,"chunked","Mon, 10 Jan 2022 00:01:44 GMT","N","Y","N","N","unknown","x509: unknown error",,, -"2022-01-10 00:01:42","34.224.0.0",10443,"","ssl,vpn","TLSv1.2",14618,"US","VIRGINIA","ASHBURN","TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA","N",2048,"","Entrust Certification Authority - L1K","2021-10-07 15:30:28","2022-11-06 15:30:28","AD:19:B2:1C:CB:88:70:9B:DB:8E:7E:F5:65:50:13:D6:43:6C:BE:6E","7B388364A24B88E77E5553B5C6748100",2,"sha256WithRSAEncryption","rsaEncryption","Ciena Corporation",,"US","Maryland","Hanover",,,,,,,,"Entrust, Inc.","(c) 2012 Entrust, Inc. - for authorized use only","US",,,,,,,,,,454110,,"N",,"Retail Trade","9A:64:73:0B:8A:FA:DE:22:D4:6D:5A:C6:C4:6F:D4:A4:2A:28:FA:41:1E:FF:81:DC:D4:D9:00:FD:78:DF:C4:DD","9A:B7:BD:68:7D:F3:E7:C1:B7:D3:F4:2F:01:B6:C4:77:90:A3:2B:1E:C0:89:F5:08:EC:43:87:35:60:36:D4:87:61:AA:B8:A8:B3:8A:E9:F1:04:AA:5B:67:12:FF:63:D5:14:80:77:6E:8F:7D:C3:E2:3A:F3:13:DF:08:43:6C:B0","E7:34:BC:92:84:FA:39:DE:E1:46:6C:27:DA:5A:01:F4","HTTP/1.1",200,"OK","text/html",,,,"xxxxxxxx-xxxxx",131,,"Mon, 10 Jan 2022 00:01:44 GMT","Y","N","N","Y","OV",,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv deleted file mode 100644 index ab28456b4e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv +++ /dev/null @@ -1,46 +0,0 @@ -"timestamp","ip","port","hostname","tag","handshake","asn","geo","region","city","cipher_suite","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serial_number","naics","sic","freak_vulnerable","freak_cipher_suite","sector","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","http_response_type","http_code","http_reason","content_type","http_connection","www_authenticate","set_cookie","server_type","content_length","transfer_encoding","http_date","cert_valid","self_signed","cert_expired","browser_trusted","validation_level","browser_error","device_model","device_sector","device_type","device_vendor","device_version","jarm","page_sha256fp","raw_cert","raw_cert_chain","tlsv13_cipher","tlsv13_support" -"2018-04-23 13:25:21","198.51.100.232","443",,"ssl-freak","TLSv1.0","8447","AT","WIEN","VIENNA","TLS_RSA_WITH_RC4_128_SHA","1024","usg50_B0B2DC2FA69D","usg50_B0B2DC2FA69D","2012-05-10 00:01:19","2032-05-05 00:01:19","14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2","4FAB054F","sha1WithRSAEncryption","rsaEncryption",,,,,,,,,,,,,,,,,,,,,,,,,"0","0","Y","TLS_RSA_EXPORT_WITH_RC4_40_MD5",,"57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1","E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87","1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE","HTTP/1.1","200","OK","text/html",,,,,,"chunked","Mon, 23 Apr 2018 13:25:26 GMT","Y","Y","N","N","unknown","x509: unknown error",,,,,,,,,,, -"2018-04-23 13:25:26","198.51.100.224","443","198-51-100-224.example.net","ssl-freak","TLSv1.0","12577","AT","NIEDEROSTERREICH","BADEN","TLS_RSA_WITH_RC4_128_SHA","1024","usg20w_C86C870287EC","usg20w_C86C870287EC","2010-01-01 00:00:53","2029-12-27 00:00:53","14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2","4B3D3B35","sha1WithRSAEncryption","rsaEncryption",,,,,,,,,,,,,,,,,,,,,,,,,"0","0","Y","TLS_RSA_EXPORT_WITH_RC4_40_MD5",,"57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1","E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87","1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE","HTTP/1.1","200","OK","text/html",,,,,,"chunked","Mon, 23 Apr 2018 13:25:29 GMT","Y","Y","N","N","unknown","x509: unknown error",,,,,,,,,,, -2018-04-23 13:25:21,198.51.100.232,443,,ssl-freak,TLSv1.0,8447,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg50_B0B2DC2FA69D,usg50_B0B2DC2FA69D,2012-05-10 00:01:19,2032-05-05 00:01:19,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4FAB054F,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:25:26 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:25:26,198.51.100.224,443,198-51-100-224.example.net,ssl-freak,TLSv1.0,12577,AT,NIEDEROSTERREICH,BADEN,TLS_RSA_WITH_RC4_128_SHA,1024,usg20w_C86C870287EC,usg20w_C86C870287EC,2010-01-01 00:00:53,2029-12-27 00:00:53,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4B3D3B35,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:25:29 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:25:33,198.51.100.67,443,,ssl-freak,TLSv1.0,8447,AT,NIEDEROSTERREICH,WAIDHOFEN AN DER THAYA,TLS_RSA_WITH_RC4_128_SHA,1024,Technicolor TG670,Technicolor TG670,2005-01-01 00:00:00,2024-12-31 00:00:00,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,-7A2C610E,sha1WithRSAEncryption,rsaEncryption,Technicolor,1112WT0YK,,,,,,,,,,,Technicolor,1112WT0YK,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Moved Temporarily,,keep-alive,,xAuth_SESSION_ID=bm90aGluZyBoZXJlCg==; path=/;,,0,,"Mon, 23 Apr 2018 14:25:37 GMT",N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:25:36,198.51.100.3,443,,ssl-freak,TLSv1.2,8445,AT,SALZBURG,HINTERGLEMM,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,2048,uag2100_04BF6D22A5A9,uag2100_04BF6D22A5A9,2016-03-08 20:27:08,2026-03-06 20:27:08,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,B0F07D300BDB4FC4,sha256WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_DES40_CBC_SHA,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:25:39 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:25:38,198.51.100.198,443,198-51-100-198.example.net,ssl-freak,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,2048,198-51-100-198.example.net,Go Daddy Secure Certificate Authority - G2,2016-12-29 08:51:00,2019-12-29 08:51:00,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,AEA6D3637023B56B,sha256WithRSAEncryption,rsaEncryption,,Domain Control Validated,,,,,,,,,,,198-51-100-198.example.net," Inc.""",http://certs.godaddy.com/repository/,SomeState,SomeCity,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,403,Forbidden ( The server,text/html,close,,,,2024,,,Y,N,N,Y,DV,,,,,,,,,,,, -2018-04-23 13:25:38,198.51.100.98,443,198-51-100-98.example.net,ssl-freak,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg50_FCF528743754,usg50_FCF528743754,2013-04-29 00:00:26,2033-04-24 00:00:26,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,517DB81A,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:25:40 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:25:41,198.51.100.156,443,198-51-100-156.example.net,ssl-freak,TLSv1.0,8339,AT,NIEDEROSTERREICH,SCHWECHAT,TLS_RSA_WITH_AES_128_CBC_SHA,1024,usg200_404A036775FC,usg200_404A036775FC,2010-05-01 00:04:04,2030-04-26 00:04:04,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4BDB6FF4,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_DES40_CBC_SHA,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:25:43 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:25:53,198.51.100.200,443,,ssl-freak,TLSv1.2,8447,AT,NIEDEROSTERREICH,KREMS AN DER DONAU,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_5CF4AB67FC6F,usg20_5CF4AB67FC6F,2015-12-02 00:00:47,2035-11-27 00:00:47,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,565E34AF,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:25:56 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:02,198.51.100.83,443,198-51-100-83.example.net,ssl-freak,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg20w_FCF5286F5972,usg20w_FCF5286F5972,2013-03-23 00:00:43,2033-03-18 00:00:43,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,514CF0AB,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:05 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:03,198.51.100.155,443,198-51-100-155.example.net,ssl-freak,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-155.example.net,198-51-100-155.example.net,2018-03-19 19:47:07,2023-03-19 19:47:07,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,2DF52AA905C7A2B44C2B9F0012FD5745,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,403,Forbidden,text/html,,,,Microsoft-IIS/6.0,1939,,"Mon, 23 Apr 2018 13:11:52 GMT",N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:03,198.51.100.129,443,198-51-100-129.example.net,ssl-freak,TLSv1.0,29654,AT,SALZBURG,SALZBURG,TLS_RSA_WITH_RC4_128_SHA,1024,localhost,localhost,2007-01-31 19:00:29,2008-01-31 19:00:29,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,2,sha1WithRSAEncryption,rsaEncryption,Apache HTTP Server,Test Certificate,,,,,,,,,,,Apache HTTP Server,For testing purposes only,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Apache,318,,"Mon, 23 Apr 2018 17:42:37 GMT",N,N,Y,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:26:12,198.51.100.7,443,198-51-100-7.example.net,ssl-freak,TLSv1.0,8445,AT,SALZBURG,ALTENMARKT IM PONGAU,TLS_RSA_WITH_RC4_128_SHA,2048,IMM2-5cf3fcaf3abd,IMM2-5cf3fcaf3abd,2013-03-22 14:32:06,2023-03-20 14:32:06,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,D8C631398B585F10,sha1WithRSAEncryption,rsaEncryption,System X,,US,SomeState,SomeCity,,,,,,,,System X,,US,SomeState,SomeCity,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,301,Moved Permanently,application/x-appweb-php,keep-alive,,,Mbedthis-Appweb/2.4.2,0,,"Mon, 23 Apr 2018 13:37:08 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:15,198.51.100.93,443,,ssl-freak,TLSv1.2,8447,AT,KARNTEN,SPITTAL AN DER DRAU,TLS_RSA_WITH_RC4_128_SHA,1024,usg50_B0B2DC3308EF,usg50_B0B2DC3308EF,2012-05-25 00:00:39,2032-05-20 00:00:39,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4FBECBA7,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:17 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:16,198.51.100.81,443,198-51-100-81.example.net,ssl-freak,TLSv1.0,5385,AT,VORARLBERG,FELDKIRCH,TLS_RSA_WITH_RC4_128_SHA,1024,usg100_5067F03642A5,usg100_5067F03642A5,2010-10-01 00:04:48,2030-09-26 00:04:48,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4CA525A0,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,518210,737415,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:19 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:17,198.51.100.162,443,198-51-100-162.example.net,ssl-freak,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,rc1,Peppercon CA,2003-05-08 16:30:05,2008-05-06 16:30:05,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,18,md5WithRSAEncryption,rsaEncryption,,R&D,DE,SomeState,,,,,,198-51-100-162.example.net,,,,Security Department,DE,SomeState,SomeCity,,,,,198-51-100-162.example.net,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Redirect,,,,,,,,,N,N,Y,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:26:22,198.51.100.57,443,,ssl-freak,TLSv1.0,8447,AT,STEIERMARK,GLEISDORF,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_5CF4AB661192,usg20_5CF4AB661192,2015-09-22 00:00:46,2035-09-17 00:00:46,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,56009A2E,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:25 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:28,198.51.100.146,443,198-51-100-146.example.net,ssl-freak,TLSv1.0,8447,AT,WIEN,VIENNA,TLS_ECDHE_RSA_WITH_RC4_128_SHA,1024,zywall_110_A0E4CB7CE5AF,zywall_110_A0E4CB7CE5AF,2015-01-26 17:19:56,2025-01-23 17:19:56,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,54C6773C,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:31 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:34,198.51.100.233,443,,ssl-freak,TLSv1.0,8447,AT,KARNTEN,KLAGENFURT AM WORTHERSEE,TLS_RSA_WITH_RC4_128_SHA,1024,198.51.100.174,198-51-100-174.example.net,2009-04-14 07:26:09,2025-04-15 07:26:09,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,A0571920C03C9EE0DA1168E586E0E8D440E42EA69D898AC829,sha1WithRSAEncryption,rsaEncryption,,General,DE,SomeState,SomeCity,,,,,198-51-100-174.example.net,,,,General,DE,SomeState,SomeCity,,,,,198-51-100-174.example.net,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,Close,,,LANCOM 1781A 8.50.0161 / 09.08.2011,,,,Y,N,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:35,198.51.100.106,443,198-51-100-106.example.net,ssl-freak,TLSv1.0,12793,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-106.example.net,SHT-Gruppe CA,2004-07-20 07:28:10,2006-07-20 07:38:10,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,190DBE75000000000007,sha1WithRSAEncryption,rsaEncryption,,,AT,SomeState,SomeCity,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Microsoft-IIS/6.0,1508,,"Mon, 23 Apr 2018 13:26:37 GMT",N,N,Y,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:26:37,198.51.100.191,443,,ssl-freak,TLSv1.0,8447,AT,STEIERMARK,LEBRING,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_5CF4AB669448,usg20_5CF4AB669448,2015-10-01 00:00:38,2035-09-26 00:00:38,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,560C77A6,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:40 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:42,198.51.100.235,443,198-51-100-235.example.net,ssl-freak,TLSv1.2,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg50_107BEF33651A,usg50_107BEF33651A,2014-04-24 00:00:27,2034-04-19 00:00:27,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,5358541B,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:45 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:43,198.51.100.167,443,198-51-100-167.example.net,ssl-freak,TLSv1.0,8412,AT,BURGENLAND,ELTENDORF,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-167.example.net,198-51-100-167.example.net,2008-08-19 06:57:11,2010-08-19 06:57:11,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,2,sha1WithRSAEncryption,rsaEncryption,SuSE Linux Web Server,web server,XY,unknown,unknown,,,,,198-51-100-167.example.net,,,SuSE Linux Web Server,CA,XY,SomeState,unknown,,,,,198-51-100-167.example.net,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Apache/2.2.3 (Linux/SUSE),80,,"Mon, 23 Apr 2018 13:26:45 GMT",N,N,Y,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:26:47,198.51.100.42,443,198-51-100-42.example.net,ssl-freak,TLSv1.0,8437,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-42.example.net,iLO Default Issuer (Do not trust),2013-11-05 00:00:00,2028-11-04 00:00:00,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,72FD09EF,sha1WithRSAEncryption,rsaEncryption,,,US,SomeState,SomeCity,,,,,,,,,,US,SomeState,Houston,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,,,,,,,,,N,N,N,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:26:48,198.51.100.177,443,198-51-100-177.example.net,ssl-freak,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_5CF4AB625772,usg20_5CF4AB625772,2015-03-04 00:00:39,2035-02-27 00:00:39,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,54F64B27,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:50 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:48,198.51.100.66,443,198-51-100-66.example.net,ssl-freak,TLSv1.0,5385,AT,VORARLBERG,DORNBIRN,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-66.example.net,198-51-100-66.example.net,2009-10-06 11:23:48,2015-03-29 11:23:48,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,98B18BCD61B0CD5D,sha1WithRSAEncryption,rsaEncryption,,??,??,??,??,,,,,??,,,,??,??,??,??,,,,,??,,,518210,737415,Y,TLS_RSA_EXPORT_WITH_DES40_CBC_SHA,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Found,text/html; charset=utf-8,close,,DSSignInURL=/; path=/; secure,,,,,Y,Y,Y,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:48,198.51.100.29,443,198-51-100-29.example.net,ssl-freak,TLSv1.0,6830,AT,NIEDEROSTERREICH,GUNTRAMSDORF,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_FCF52878354B,usg20_FCF52878354B,2013-05-20 00:00:39,2033-05-15 00:00:39,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,519967A7,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:50 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:49,198.51.100.235,443,,ssl-freak,TLSv1.0,8447,AT,TIROL,KITZBUHEL,TLS_RSA_WITH_RC4_128_SHA,1024,usg50_B0B2DC3AEFE7,usg50_B0B2DC3AEFE7,2012-10-30 00:02:36,2032-10-25 00:02:36,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,508F191C,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:52 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:50,198.51.100.159,443,,ssl-freak,TLSv1.0,8218,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-159.example.net,198-51-100-159.example.net,2002-01-09 20:22:25,2003-01-09 20:22:25,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,0,md5WithRSAEncryption,rsaEncryption,SomeOrganization,SomeOrganizationalUnit,--,SomeState,SomeCity,,,,,198-51-100-159.example.net,,,SomeOrganization,SomeOrganizationalUnit,--,SomeState,SomeCity,,,,,198-51-100-159.example.net,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,close,,,Apache/1.3.33 (Unix) (Red-Hat/Linux) FrontPage/4.,31,,"Mon, 23 Apr 2018 13:26:52 GMT",Y,Y,Y,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:51,198.51.100.138,443,198-51-100-138.example.net,ssl-freak,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_B0B2DC34A1F6,usg20_B0B2DC34A1F6,2012-06-16 00:00:58,2032-06-11 00:00:58,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4FDBCCBA,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:54 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:52,198.51.100.64,443,,ssl-freak,TLSv1.0,1853,AT,OBEROSTERREICH,WILHERING,TLS_RSA_WITH_RC4_128_SHA,1024,198.51.100.171,198.51.100.117,2017-08-10 10:48:40,2020-08-09 10:48:40,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,598C3A08,sha1WithRSAEncryption,rsaEncryption,,,,,SomeCity,,,,,,,,,,,,SomeCity,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,5597,,"Mon, 23 Apr 2018 13:26:54 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:55,198.51.100.189,443,198-51-100-62.example.net,ssl-freak,TLSv1.0,25255,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg20w_107BEF3A4C9E,usg20w_107BEF3A4C9E,2014-07-04 00:00:43,2034-06-29 00:00:43,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,53B5EEAB,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:57 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:56,198.51.100.17,443,198-51-100-17.example.net,ssl-freak,TLSv1.0,8447,AT,STEIERMARK,SOEDING,TLS_RSA_WITH_AES_256_CBC_SHA,1024,Vimar By-Web,Vimar By-Web,2011-10-27 09:19:55,2016-10-25 09:19:55,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,B82B13ED1FB0FD71,sha1WithRSAEncryption,rsaEncryption,,R&D,IT,SomeState,SomeCity,,,,,,,,,R&D,IT,SomeState,SomeCity,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_DES40_CBC_SHA,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Moved Temporarily,text/html,keep-alive,,,nginx/0.6.32,,chunked,"Mon, 23 Apr 2018 13:26:56 GMT",Y,Y,Y,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:56,198.51.100.111,443,,ssl-freak,TLSv1.0,8218,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-111.example.net,198-51-100-111.example.net,2002-01-09 20:22:25,2003-01-09 20:22:25,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,0,md5WithRSAEncryption,rsaEncryption,SomeOrganization,SomeOrganizationalUnit,--,SomeState,SomeCity,,,,,198-51-100-111.example.net,,,SomeOrganization,SomeOrganizationalUnit,--,SomeState,SomeCity,,,,,198-51-100-111.example.net,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,close,,,Apache/1.3.33 (Unix) (Red-Hat/Linux) FrontPage/4.,31,,"Mon, 23 Apr 2018 13:26:58 GMT",Y,Y,Y,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:56,198.51.100.179,443,198-51-100-179.example.net,ssl-freak,TLSv1.0,12605,AT,OBEROSTERREICH,LINZ,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_5CF4AB665FB9,usg20_5CF4AB665FB9,2015-09-25 00:00:42,2035-09-20 00:00:42,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,56048EAA,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:58 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:58,198.51.100.143,443,,ssl-freak,TLSv1.0,8447,AT,KARNTEN,GLAN,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_FCF5285DEDC4,usg20_FCF5285DEDC4,2012-11-09 00:00:44,2032-11-04 00:00:44,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,509C47AC,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:27:00 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:58,198.51.100.111,443,198-51-100-111.example.net,ssl-freak,TLSv1.0,1901,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,*.*,198-51-100-111.example.net,2009-01-16 12:51:43,2010-01-16 12:51:43,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,6,md5WithRSAEncryption,rsaEncryption,,,IL,SomeState,,,,,,,,,,Visonic CA,IL,SomeState,,,,,,198-51-100-111.example.net,,,518210,737415,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Found,text/html,close,,PowerLink=226002836046b4bddcd2d16b809f76d9; path=/,Apache/1.3.31 (Unix) PHP/4.3.9 mod_ssl/2.8.20 Open,,chunked,"Wed, 23 Jan 2002 10:17:09 GMT",N,N,Y,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:26:59,198.51.100.79,443,,ssl-freak,TLSv1.0,8447,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_5CF4AB65A17C,usg20_5CF4AB65A17C,2015-09-01 00:00:51,2035-08-27 00:00:51,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,55E4EAB3,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:27:02 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:59,198.51.100.90,443,,ssl-freak,TLSv1.0,8218,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-90.example.net,198-51-100-90.example.net,2002-01-09 20:22:25,2003-01-09 20:22:25,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,0,md5WithRSAEncryption,rsaEncryption,SomeOrganization,SomeOrganizationalUnit,--,SomeState,SomeCity,,,,,198-51-100-90.example.net,,,SomeOrganization,SomeOrganizationalUnit,--,SomeState,SomeCity,,,,,198-51-100-90.example.net,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,close,,,Apache/1.3.33 (Unix) (Red-Hat/Linux) FrontPage/4.,31,,"Mon, 23 Apr 2018 13:27:02 GMT",Y,Y,Y,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:27:03,198.51.100.186,443,198-51-100-186.example.net,ssl-freak,TLSv1.0,31125,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-186.example.net,198-51-100-186.example.net,2013-07-11 12:20:19,2021-07-09 12:20:19,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,D947ED19BEAB28E6,sha1WithRSAEncryption,rsaEncryption,,IT,PL,SomeState,SomeCity,,,,,198-51-100-186.example.net,,,,IT,PL,SomeState,SomeCity,,,,,198-51-100-186.example.net,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,401,Unauthorized,text/plain,close,"Basic realm=""example.com""",,Microsoft-IIS/7.5,0,,"Mon, 23 Apr 2018 14:03:40 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:27:03,198.51.100.150,443,198-51-100-150.example.net,ssl-freak,TLSv1.0,8559,AT,BURGENLAND,NEUSIEDL AM SEE,TLS_ECDHE_RSA_WITH_RC4_128_SHA,2048,198-51-100-150.example.net,COMODO RSA Domain Validation Secure Server CA,2017-02-08 00:00:00,2019-05-09 23:59:59,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,B6EF6CF436532F0252627393BD7311FD,sha256WithRSAEncryption,rsaEncryption,,Domain Control Validated,,,,,,,,,,,,,GB,SomeState,SomeCity,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:27:06 GMT",N,N,N,N,DV,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:27:03,198.51.100.141,443,198-51-100-141.example.net,ssl-freak,TLSv1.0,39372,AT,OBEROSTERREICH,HINTERSTODER,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-141.example.net,iLO Default Issuer (Do not trust),2014-01-14 00:00:00,2029-01-13 00:00:00,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,7852761B,sha1WithRSAEncryption,rsaEncryption,,,US,SomeState,SomeCity,,,,,,,,,,US,SomeState,SomeCity,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,,,,,,,,,N,N,N,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:27:04,198.51.100.194,443,198-51-100-194.example.net,ssl-freak,TLSv1.0,8447,AT,KARNTEN,GLAN,TLS_RSA_WITH_RC4_128_SHA,1024,iDRAC6 default certificate,iDRAC6 default certificate,2009-09-17 22:47:28,2019-09-15 22:47:28,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,1,sha1WithRSAEncryption,rsaEncryption,,Remote Access Group,US,SomeState,SomeCity,,,,,,,,,Remote Access Group,US,SomeState,SomeCity,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Moved Temporarily,,keep-alive,,,Mbedthis-Appweb/2.4.2,0,,"Mon, 23 Apr 2018 13:25:57 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -"2022-02-07 00:01:41","2.136.0.0",10443,"2-136-0-0.example.com","ssl,ssl-freak,ssl-poodle,vpn","TLSv1.0",12345,"ES","MADRID","MADRID","TLS_RSA_WITH_RC4_128_SHA",1024,"usg50_107BEF336340","usg50_107BEF336340","2014-04-24 00:00:32","2034-04-19 00:00:32","F5:04:98:CD:D4:67:13:E1:77:B7:38:D4:B9:43:C0:72:50:6C:0D:58",53585420,"sha1WithRSAEncryption","rsaEncryption",,,,,,,,,,,,,,,,,,,,,,,,,517311,,"Y","TLS_RSA_EXPORT_WITH_RC4_40_MD5","Communications, Service Provider, and Hosting Service","AF:3A:71:B7:1B:A2:62:4E:87:22:FF:19:3F:84:1F:7F:CC:DC:06:E0:AF:80:E2:5D:33:A5:68:9A:E3:81:25:45","14:92:CC:6B:C7:B3:09:31:50:8C:1C:8D:5B:FD:D1:BE:41:78:80:97:E0:10:11:48:1F:EE:D6:CB:4F:F0:13:D5:05:56:AC:BA:12:12:02:F7:0F:03:40:95:17:8A:5F:79:98:E1:44:EF:E6:5A:44:E3:AC:3A:F8:49:F7:AC:B6:52","E8:5F:96:16:3F:76:35:F0:07:4F:4C:2C:38:FC:27:6B","HTTP/1.1",200,"OK","text/html",,,,"",,"chunked","Mon, 07 Feb 2022 00:01:43 GMT","Y","Y","N","N","unknown","x509: unknown error",,,,,,"Zyxel","firewall","ZyWALL USG 50",,"enterprise", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv deleted file mode 100644 index 4bcc6758ac..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv +++ /dev/null @@ -1,32 +0,0 @@ -"timestamp","ip","port","hostname","tag","handshake","asn","geo","region","city","cipher_suite","ssl_poodle","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serial_number","naics","sic","sector","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","http_response_type","http_code","http_reason","content_type","http_connection","www_authenticate","set_cookie","server_type","content_length","transfer_encoding","http_date","cert_valid","self_signed","cert_expired","browser_trusted","validation_level","browser_error","tlsv13_support","tlsv13_cipher","device_model","device_sector","device_type","device_vendor","device_version","jarm","page_sha256fp","raw_cert","raw_cert_chain" -"2018-08-08 00:51:42","203.0.113.85",8443,"example.com","ssl-poodle","TLSv1.0",65540,"AT","WIEN","VIENNA","TLS_RSA_WITH_RC4_128_SHA","Y",1024,"usg20_107BEF394BA5","usg20_107BEF394BA5","2014-06-25 00:00:42","2034-06-20 00:00:42","04:FA:DE:1D:BD:4A:05:25:61:FB:F3:D6:64:74:66:44:01:22:D7:C3","53AA112A",2,"sha1WithRSAEncryption","rsaEncryption",,,,,,,,,,,,,,,,,,,,,,,,,0,0,,"16:25:9F:C7:A1:8D:64:1F:D9:25:42:BF:87:5C:4F:F3:63:14:97:21:EC:B6:67:10:F2:CA:52:37:C9:FE:49:2E","0B:2D:48:8C:4B:55:8B:F3:AB:F8:45:ED:E0:A0:63:F4:84:2F:4C:19:DC:A8:6F:7D:6A:AF:61:D7:98:AA:58:0F:CB:CA:87:D2:C3:0B:C5:DF:49:A7:84:7C:47:58:89:7D:92:B6:7B:98:7D:B1:64:4B:DC:DD:BE:9D:11:2A:D1:AE","33:E3:61:3F:5D:AA:96:99:38:A5:D6:F1:11:C7:ED:FC","HTTP/1.1",200,"OK","text/html",,,,,,"chunked","Wed, 08 Aug 2018 00:51:44 GMT","Y","Y","N","N","unknown","x509: unknown error",,,,,,,,,,, -2018-04-19 13:32:27,198.51.100.147,443,,ssl-poodle,TLSv1.0,8445,AT,SALZBURG,SALZBURG,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-147.example.net,some_issuer,2017-09-18 08:22:17,2019-09-18 08:22:17,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,746481F100000000000C,2,sha1WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,AT,Tirol,Ehrwald,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Microsoft-IIS/7.5,689,,"Thu, 19 Apr 2018 13:32:32 GMT",N,N,N,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-19 13:32:32,198.51.100.207,443,198-51-100-94.example.net,ssl-poodle,TLSv1.0,25255,AT,SALZBURG,SALZBURG,TLS_RSA_WITH_RC4_128_SHA,Y,1024,example,some_issuer,2004-06-03 11:11:43,2024-05-29 11:11:43,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,0,2,md5WithRSAEncryption,rsaEncryption,,,US,,,,,,,,,,,,US,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,close,,"rg_cookie_session_id=1654544029; path=/; expires=Fri, 01 Jan 2038",,,,"Thu, 19 Apr 2018 13:32:34 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:32,198.51.100.200,443,198-51-100-200.example.net,ssl-poodle,TLSv1.2,8445,AT,SALZBURG,,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-200.example.net,198-51-100-200.example.net,2016-10-01 14:09:12,2020-10-02 14:09:12,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,2E8C9E4A2C7D3EDC,2,sha256WithRSAEncryption,rsaEncryption,some_org_name,,AT,,,,,,,,,,some_org_name,,AT,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html; charset=utf-8,close,,,,,,,N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:33,198.51.100.239,443,198-51-100-239.example.net,ssl-poodle,TLSv1.0,8437,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,Y,1024,198-51-100-239.example.net,198-51-100-239.example.net,2011-07-27 13:30:18,2012-07-26 13:30:18,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,7C91,2,sha1WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,--,SomeState,SomeCity,,,,,198-51-100-239.example.net,,,SomeOrganization,SomeOrganizationalUnit,--,SomeState,SomeCity,,,,,198-51-100-239.example.net,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,403,Forbidden,text/html; charset=UTF-8,close,,,Apache/2.2.3 (CentOS),4958,,"Thu, 19 Apr 2018 13:32:35 GMT",Y,Y,Y,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:35,198.51.100.156,443,,ssl-poodle,TLSv1.0,8447,AT,NIEDEROSTERREICH,,TLS_RSA_WITH_RC4_128_SHA,Y,1024,example,some_issuer,2010-01-01 00:00:52,2029-12-27 00:00:52,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4B3D3B34,2,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Thu, 19 Apr 2018 13:32:37 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:36,198.51.100.122,443,198-51-100-122.example.net,ssl-poodle,TLSv1.2,36351,AT,AUSTRIA,?,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,Y,2048,198-51-100-122.example.net,COMODO RSA Organization Validation Secure Server CA,2017-04-06 00:00:00,2019-04-06 23:59:59,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,CAB81F32F3FF4766BC545A2C14DF34B5,2,sha256WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,AT,Wien,Wien,,1130,,,,,,COMODO CA Limited,,GB,Greater Manchester,Salford,,,,,,,,518210,737401,Information Technology,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Apache,0,,"Thu, 19 Apr 2018 13:32:20 GMT",Y,N,N,Y,OV,,,,,,,,,,,, -2018-04-19 13:32:37,198.51.100.58,443,198-51-100-58.example.net,ssl-poodle,TLSv1.2,12605,AT,OBEROSTERREICH,LINZ,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,example,some_issuer,2015-01-17 16:11:24,2020-01-17 16:11:24,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,6D9E2D4443F1D69E4A8865CC1C5B6963,2,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Microsoft-IIS/8.5,701,,"Thu, 19 Apr 2018 13:34:53 GMT",N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:38,198.51.100.18,443,198-51-100-18.example.net,ssl-poodle,TLSv1.2,6830,AT,OBEROSTERREICH,LINZ,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,Y,2048,198-51-100-18.example.net,TERENA SSL CA 3,2017-07-14 00:00:00,2020-07-22 12:00:00,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,0386AD387BEC13878473D23C8C786ECE,2,sha256WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,AT,,Linz,,,,,,,,TERENA,,NL,Noord-Holland,Amsterdam,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Found,,Close,,BNIS_ChallengeState=Bqyd+IQebjQwiiYNKBJkA5Ta0spL1gX5; Path=/; Exp,,61,,,Y,N,N,Y,OV,,,,,,,,,,,, -2018-04-19 13:32:38,198.51.100.246,443,,ssl-poodle,TLSv1.2,8447,AT,SALZBURG,SALZBURG,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,example,some_issuer,2014-09-01 16:18:46,2054-08-24 16:18:46,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,129FA64A4BE039B54E850F1AA65AD835,2,sha256WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Found,text/html; charset=utf-8,,,ASP.NET_SessionId=e3qfk1dfz2mtqwzoym3gul3r; path=/; HttpOnly,Microsoft-IIS/8.5,145,,"Thu, 19 Apr 2018 13:32:40 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:39,198.51.100.35,443,198-51-100-35.example.net,ssl-poodle,TLSv1.0,12605,AT,OBEROSTERREICH,LINZ,TLS_RSA_WITH_AES_128_CBC_SHA,Y,2048,198-51-100-35.example.net,Go Daddy Secure Certificate Authority - G2,2017-08-28 13:29:01,2018-09-10 06:28:49,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,90B22B4CEF57C0FC,2,sha256WithRSAEncryption,rsaEncryption,,some_org_name,,,,,,,,198-51-100-35.example.net,,,,,US,Arizona,Scottsdale,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Microsoft-IIS/7.5,266,,"Thu, 19 Apr 2018 13:35:03 GMT",Y,N,N,Y,DV,,,,,,,,,,,, -2018-04-19 13:32:39,198.51.100.142,443,,ssl-poodle,TLSv1.0,8447,AT,WIEN,VIENNA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,Y,2048,198.51.100.19,198-51-100-19.example.net,2014-12-11 09:57:33,2024-12-08 09:57:33,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,A0571DCBE5E1A2C062D8FB7001271581B5F69824157E385563FA23527E0B,2,sha256WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,DE,,,,,,,198-51-100-19.example.net,,,some_org_name,Engineering,DE,NRW,Wuerselen,,,,,198-51-100-19.example.net,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,Close,,,LANCOM,,,"Thur, 19 Apr 2018 13:32:41 GMT",Y,N,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:39,198.51.100.178,443,,ssl-poodle,TLSv1.0,8447,AT,NIEDEROSTERREICH,,TLS_RSA_WITH_RC4_128_SHA,Y,1024,example,some_issuer,2012-05-30 00:00:44,2032-05-25 00:00:44,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4FC5632C,2,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Thu, 19 Apr 2018 13:32:41 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:40,198.51.100.99,443,198-51-100-99.example.net,ssl-poodle,TLSv1.2,6830,AT,WIEN,VIENNA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-99.example.net,RapidSSL RSA CA 2018,2018-03-30 00:00:00,2019-04-29 12:00:00,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,0BDCB5D6D4C22BD2A1CF55584B6DE09C,2,sha256WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,DigiCert Inc,198-51-100-99.example.net,US,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,404,Not Found,text/html; charset=us-ascii,close,,,Microsoft-HTTPAPI/2.0,315,,"Thu, 19 Apr 2018 13:32:43 GMT",Y,N,N,Y,DV,,,,,,,,,,,, -2018-04-19 13:32:40,198.51.100.235,443,198-51-100-235.example.net,ssl-poodle,TLSv1.0,25255,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,Y,1024,Nextcloud,Nextcloud,2016-12-13 20:28:39,2017-01-12 20:28:39,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,CDE5769D28C80B6B,2,sha1WithRSAEncryption,rsaEncryption,some_org_name,,AU,Some-State,,,,,,,,,Internet Widgits Pty Ltd,,AU,Some-State,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,400,Bad Request,text/html; charset=UTF-8,close,,nc_sameSiteCookiestrict=true; path=/; httponly;secure; expires=Fr,Apache/2.4.10 (FreeBSD) OpenSSL/0.9.8zd-freebsd PH,6939,,"Thu, 19 Apr 2018 13:32:42 GMT",Y,Y,Y,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:41,198.51.100.187,443,198-51-100-187.example.net,ssl-poodle,TLSv1.2,28760,AT,OBEROSTERREICH,,TLS_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-187.example.net,Go Daddy Secure Certificate Authority - G2,2018-02-12 17:56:01,2020-02-12 17:56:01,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,03BA30FF4972177C,2,sha256WithRSAEncryption,rsaEncryption,,some_org_name,,,,,,,,198-51-100-187.example.net,,,,,US,Arizona,Scottsdale,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,400,No parameters passed t,text/html,,,,Microsoft-IIS/10.0,11,,"Thu, 19 Apr 2018 13:32:42 GMT",Y,N,N,Y,DV,,,,,,,,,,,, -2018-04-19 13:32:42,198.51.100.213,443,198-51-100-213.example.net,ssl-poodle,TLSv1.2,8447,AT,OBEROSTERREICH,,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-213.example.net,some_issuer,2016-09-22 08:12:17,2018-09-22 08:12:17,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,770000000EBB9429663601BAB700000000000E,2,sha1WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,AT,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,301,Moved Permanently,,close,,,Microsoft-IIS/8.5,0,,"Thu, 19 Apr 2018 13:32:44 GMT",N,N,N,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-19 13:32:42,198.51.100.74,443,198-51-100-74.example.net,ssl-poodle,TLSv1.0,62363,AT,STEIERMARK,,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,DC,DC,2016-12-30 17:15:38,2021-12-30 17:15:38,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,7753CCEB55990A834E15DAC5707D403A,2,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Microsoft-IIS/7.5,689,,"Thu, 19 Apr 2018 13:32:44 GMT",N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:43,198.51.100.145,443,198-51-100-145.example.net,ssl-poodle,TLSv1.0,8447,AT,KARNTEN,KLAGENFURT AM WORTHERSEE,TLS_RSA_WITH_RC4_128_SHA,Y,1024,localdomain,localdomain,2008-10-07 20:12:54,2018-10-07 20:12:54,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,91B04FFCF174CCFF,0,sha1WithRSAEncryption,rsaEncryption,some_org_name,,CA,,,,,,,198-51-100-145.example.net,,,some_org_name,,CA,Quebec,Gatineau,,,,,198-51-100-145.example.net,,,518210,737415,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.0,302,Found,text/html; charset=UTF-8,close,,"HOMEBASEID=658512b32961b9b6f8df7a3d4de7fa01; expires=Tue, 19-Jan-",Apache/2.2.3 (Red Hat),0,,"Thu, 19 Apr 2018 12:52:32 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:44,198.51.100.48,443,198-51-100-48.example.net,ssl-poodle,TLSv1.0,1901,AT,NIEDEROSTERREICH,,TLS_RSA_WITH_RC4_128_SHA,Y,1024,198-51-100-48.example.net,198-51-100-48.example.net,2013-06-15 20:10:49,2023-06-15 20:10:49,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,013F49762DAE,0,sha1WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,US,,,,,,,198-51-100-48.example.net,,,Western Digital,Branded Products,US,CS,Mountain View,,,,,,,,518210,737415,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Apache,225,,"Thu, 19 Apr 2018 03:08:06 GMT",N,N,N,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-19 13:32:45,198.51.100.94,443,198-51-100-94.example.net,ssl-poodle,TLSv1.2,6830,AT,WIEN,VIENNA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-94.example.net,RapidSSL CA,2013-04-03 17:02:33,2014-04-07 03:32:33,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,0B697D,2,sha1WithRSAEncryption,rsaEncryption,,some_org_name,,,,,,,,,,KtAjvog6HgAsml0cyxE4hpc9kv8dhgWZ,"GeoTrust, Inc.",,US,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Found,text/html; charset=utf-8,,,ASP.NET_SessionId=z5lph4ufefkvg1xzmd4q2m33; path=/; HttpOnly,Microsoft-IIS/8.0,144,,"Thu, 19 Apr 2018 13:32:48 GMT",Y,N,Y,N,unknown,x509: certificate has expired or is not yet valid,,,,,,,,,,, -2018-04-19 13:32:45,198.51.100.53,443,198-51-100-53.example.net,ssl-poodle,TLSv1.0,8447,AT,TIROL,,TLS_RSA_WITH_RC4_128_SHA,Y,1024,example,some_issuer,2008-11-13 13:47:18,2028-11-08 13:47:18,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,BE2B43544C0AFF2E,0,sha1WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,DE,,,,,,,198-51-100-53.example.net,,,some_org_name,some_org_name,DE,Niedersachsen,38162 Cremlingen (OT Schandelah),,,,,198-51-100-53.example.net,,,518210,737415,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html; charset=iso-8859-1;,,,,GoAhead-Webs,,,,Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:46,198.51.100.56,443,198-51-100-56.example.net,ssl-poodle,TLSv1.0,8445,AT,TIROL,,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-56.example.net,some_issuer,2016-11-28 08:05:12,2018-11-28 08:05:12,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,637D34F100010000000E,2,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Microsoft-IIS/7.5,689,,"Thu, 19 Apr 2018 13:32:49 GMT",N,N,N,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-19 13:32:46,198.51.100.82,443,198-51-100-82.example.net,ssl-poodle,TLSv1.0,6830,AT,OBEROSTERREICH,,TLS_RSA_WITH_AES_128_CBC_SHA,Y,1024,123AFG,7426AC8186F3,2011-01-01 00:00:06,2020-12-29 00:00:06,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,8186F3,2,sha1WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,US,,,,,,,,,,"Cisco Systems, Inc.",some_org_name,US,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.0,200,OK,text/html,close,,,Embedded HTTP Server.,107,,"Sat, 01 Jan 2011 00:00:45 GMT",N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:48,198.51.100.29,443,198-51-100-29.example.net,ssl-poodle,TLSv1.0,6830,AT,STEIERMARK,GRAZ,TLS_RSA_WITH_RC4_128_SHA,Y,1024,198.51.100.43,198.51.100.22,2018-04-18 13:32:09,2038-01-15 13:32:09,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,862D98F4B99D0042,2,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,518210,737415,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.0,200,OK,text/html; charset=utf-8,,,,,,,,Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:49,198.51.100.114,443,198-51-100-114.example.net,ssl-poodle,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_AES_256_CBC_SHA,Y,1024,198-51-100-114.example.net,198-51-100-114.example.net,2009-08-25 17:47:57,2019-05-25 17:47:57,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,FB09C7848A7F4D77,0,sha1WithRSAEncryption,rsaEncryption,some_org_name,,AT,Vienna,Vienna,,,,,198-51-100-114.example.net,,,Digispectrum,,AT,Vienna,Vienna,,,,,198-51-100-114.example.net,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html; charset=utf-8,close,,b69223925949d45306d32f1a3d23c011=6a01vehilfpml41pl3pq3oth52; path,Apache/2.2.3 (CentOS),,chunked,"Thu, 19 Apr 2018 13:32:52 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:49,198.51.100.11,443,198-51-100-11.example.net,ssl-poodle,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,Y,2048,FGT60C3G12019794,FGT60C3G12019794,2012-08-10 07:17:11,2022-08-11 07:17:11,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,-6CD83A89,2,sha1WithRSAEncryption,rsaEncryption,some_org_name,,,,,,,,,,,,Fortinet Ltd.,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,79,,"Thu, 19 Apr 2018 13:32:08 GMT",N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:49,198.51.100.49,443,198-51-100-49.example.net,ssl-poodle,TLSv1.2,8447,AT,NIEDEROSTERREICH,,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,Y,1024,localhost,localhost,2009-11-10 23:48:47,2019-11-08 23:48:47,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,B5C752C98781B503,0,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,518210,737415,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Apache/2.4.10 (Win32) OpenSSL/1.0.1i PHP/5.5.15,2190,,"Thu, 19 Apr 2018 13:32:55 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:50,198.51.100.236,443,,ssl-poodle,TLSv1.0,8447,AT,NIEDEROSTERREICH,,TLS_RSA_WITH_AES_128_CBC_SHA,Y,1024,example,some_issuer,2013-01-30 12:00:08,2023-01-28 12:00:08,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,-462A1420,2,sha1WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,US,,,,,,,,,,Netgear Inc.,Netgear Prosafe,US,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.0,200,OK,text/html,close,,,Embedded HTTP Server.,107,,"Sat, 01 Jan 2011 00:00:21 GMT",N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:50,198.51.100.224,443,198-51-100-224.example.net,ssl-poodle,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-224.example.net,some_issuer,2017-08-03 10:21:50,2019-08-03 10:21:50,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,6126D181000300000041,2,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,401,Unauthorized,text/html,,NTLM,,Microsoft-IIS/7.5,1344,,"Thu, 19 Apr 2018 13:32:52 GMT",N,N,N,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -"2022-02-07 00:01:41","206.162.0.0",10443,,"ssl,ssl-poodle,vpn","TLSv1.2",12345,"CA","BRITISH COLUMBIA","BURNABY","TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA","Y",1024,"FWF60D4615000455","support","2015-01-28 18:14:33","2038-01-19 03:14:07","C9:B0:4E:B7:79:94:B4:DD:A7:15:21:86:43:F9:6E:4B:C9:A2:87:D9","1CA40F",2,"sha1WithRSAEncryption","rsaEncryption","Fortinet","FortiGate","US","California","Sunnyvale",,,,,"support@fortinet.com",,,"Fortinet","Certificate Authority","US","California","Sunnyvale",,,,,"support@fortinet.com",,,517311,,"Communications, Service Provider, and Hosting Service","38:F7:E0:92:24:8C:CB:28:43:93:0B:91:17:30:B1:41:8F:4E:2D:E5:A8:93:AE:4D:FE:53:00:D3:0E:53:02:16","0C:F0:37:3F:A8:93:AE:4D:FE:53:00:D3:2A:E6:6D:0B:02:9D:B9:46:58:A6:9E:5A:35:40:FB:62:9C:81:47:0A:4F:15:5D:53:D9:2F:36:4A:0B:3B:10:61:A9:07:EE:94:EC:00:B8:9C:F7:E0:92:24:8C:CB:28:2C:DD:E7:07:C6","8A:B3:08:20:34:79:94:B4:DD:A7:36:D7:14:6E:33:50","HTTP/1.1",200,"OK","text/html",,,,,131,,"Mon, 07 Feb 2022 00:01:43 GMT","Y","N","N","N","unknown","x509: unknown error",,,,,,"Fortinet","firewall","FortiGate",,"enterprise", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv deleted file mode 100644 index fd671ec904..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","transaction_id","magic_cookie","message_length","message_type","mapped_family","mapped_address","mapped_port","xor_mapped_family","xor_mapped_address","xor_mapped_port","software","fingerprint","amplification","response_size" -"2010-02-10 00:00:00",192.168.0.1,udp,3478,node01.example.com,stun,64512,ZZ,Region,City,0,0,,000000000000000000000000,2112a442,88,0101,01,192.168.0.1,3243,01,192.168.0.1,3243,"Coturn-4.5.1.1 'dan Eider'",0xfaedd06e,5.40,108 -"2010-02-10 00:00:01",192.168.0.2,udp,3478,node02.example.com,stun,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",000000000000000000000000,2112a442,88,0101,01,51.77.39.195,45877,01,192.168.0.2,45877,"Coturn-4.5.1.1 'dan Eider'",0x21128641,5.40,108 -"2010-02-10 00:00:02",192.168.0.3,udp,3478,node03.example.com,stun,64512,ZZ,Region,City,0,0,,000000000000000000000000,2112a442,76,0101,01,192.168.0.3,16321,01,188.68.240.32,16321,"ApolloProxy-1.20.1.28 'sunflower'",,4.80,96 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv deleted file mode 100644 index 8f63554910..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sequence_number","ack_number","window_size","urgent_pointer","tcp_flags","raw_packet","sector" -"2022-01-10 09:18:23","66.9.0.0","tcp",80,,"synfulknock",18885,"US","NEW JERSEY","JERSEY CITY",,,0,791102,8192,0,4608,"3cfdfec601e4700f6a9a2000080045000034c3780000f706789442099555b869f7ee0050b20800000000000c123e8012200002aa0000020405b40101040201030305", -"2022-01-10 09:19:17","213.131.0.0","tcp",80,"host-213-131-55-210-customer.wanex.net","synfulknock",35805,"GE","TBILISI","TBILISI",,,0,791102,8192,0,4608,"90e2baaf0b84700f6a9a200008004500003434100000f2064382d58337d2b8698b720050916200000000000c123e8012200059d50000020405b40101040201030305", -"2022-01-10 09:27:39","213.178.0.0","tcp",80,,"synfulknock",29256,"SY","DIMASHQ","DAMASCUS",,,0,791102,8192,0,4608,"90e2bab9cfd4700f6a9a20000800450000340f1d0000ea068bdad5b2e6914a522f360050eb5200000000000c123e801220001b4a0000020405b40101040201030305" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv deleted file mode 100644 index 3309e9a3d8..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","banner" -"2019-09-04 12:27:34","198.123.245.145","tcp",5678,"example.local","telnet-alt",20255,"AA","LOCATION","LOCATION",0,0,"|MikroTik v6.5|Login:" -"2019-09-04 12:27:40","198.123.245.145","tcp",5678,"example.local","telnet-alt",20255,"AA","LOCATION","LOCATION",0,0,"|MikroTik v6.45.3 (stable)|Login:" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv deleted file mode 100644 index 3dde133d4e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","opcode","errorcode","error","errormessage","size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,35067,node01.example.com,tftp,64512,ZZ,Region,City,0,0,5,0,"Not defined","Get not supported",22,1.57 -"2010-02-10 00:00:01",192.168.0.2,udp,56709,node02.example.com,tftp,64512,ZZ,Region,City,0,0,5,1,"File not found","File not found",19,1.36 -"2010-02-10 00:00:02",192.168.0.3,udp,32785,node03.example.com,tftp,64512,ZZ,Region,City,0,0,5,2,"Access violation","Access violation",21,1.50 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv deleted file mode 100644 index efeab02c49..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","mac","radioname","essid","modelshort","modelfull","firmware","size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,10001,node01.example.com,"ubiquiti,iot",64512,ZZ,Region,City,0,0,00156db98c3a,kachine.meta.lidia.tereixa,Kachine-Meta-Lidia-Tereixa,NS5,,XS5.ar2313.v3.5.4494.091109.1459,148,37.00 -"2010-02-10 00:00:01",192.168.0.2,udp,10001,node02.example.com,"ubiquiti,iot",64512,ZZ,Region,City,0,0,00156d7c9188,adana.mason.lanikai.ozaner,Adana-Mason-Lanikai-Ozaner,LM5,"NanoStation Loco M5",XM.ar7240.v5.6.3.28591.151130.1749,156,39.00 -"2010-02-10 00:00:02",192.168.0.3,udp,10001,node03.example.com,"ubiquiti,iot",64512,ZZ,Region,City,0,0,0418d6000fd5,tailynn.kadija.noreen.dinkar,Tailynn-Kadija-Noreen-Dinkar,P2B-400,"PowerBeam M2 400",XW.ar934x.v5.6.5.29033.160515.2108,145,36.25 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv deleted file mode 100644 index 000f5ed42d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","port","hostname","asn","geo","region","city","naics","sic","product","banner","sector" -"2019-09-04 14:51:44","198.123.245.53",5678,,5678,"AA","LOCATION","LOCATION",0,0,"Apple remote desktop vnc","RFB 003.889", -"2019-09-04 14:51:44","198.123.245.112",5678,"localhost.localdomain",5678,"AA","LOCATION","LOCATION",517311,0,"RealVNC Enterprise v5.3 or later","RFB 005.000", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv deleted file mode 100644 index 7e279ca3e6..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","response_size","amplification","error","raw_response" -"2010-02-10 00:00:00",192.168.0.1,udp,3702,node01.example.com,ws-discovery,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",989,164.83,"Validation constraint violation: SOAP message expected",c2FtcGxlIHJlc3BvbnNlIGRhdGEK -"2010-02-10 00:00:01",192.168.0.2,udp,3702,node02.example.com,ws-discovery,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",918,183.60,"Validation constraint violation: missing root element",c2FtcGxlIHJlc3BvbnNlIGRhdGEK -"2010-02-10 00:00:02",192.168.0.3,udp,3702,node03.example.com,ws-discovery,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",989,197.80,"Validation constraint violation: SOAP message expected",c2FtcGxlIHJlc3BvbnNlIGRhdGEK diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv.license deleted file mode 100644 index 9f58c89ef0..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2023 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv deleted file mode 100644 index 7e83bbaf8f..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","opcode","reported_hostname","status","size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,177,node01.example.com,xdmcp,64512,ZZ,Region,City,0,0,Willing,node01.example.com,"Linux 3.0.101-100-default",44,6.29 -"2010-02-10 00:00:01",192.168.0.2,udp,47074,node02.example.com,xdmcp,64512,ZZ,Region,City,0,0,Willing,node02.example.com,"Linux 2.6.9-103.ELsmp",48,6.86 -"2010-02-10 00:00:02",192.168.0.3,udp,177,node03.example.com,xdmcp,64512,ZZ,Region,City,0,0,Willing,node03.example.com,"1 user, load: 6,5, 6,6, 6,6",46,6.57 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/sinkhole_http_drone.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/sinkhole_http_drone.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/sinkhole_http_drone.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/special.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/special.csv deleted file mode 100644 index 2e7b591582..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/special.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","port","protocol","asn","geo","region","city","hostname","naics","sector","tag","public_source","status","detail","method","device_vendor" -"2010-02-10 00:00:00",192.168.0.1,,,64512,ZZ,Region,City,node01.example.com,0,"Communications, Service Provider, and Hosting Service",cyclops-blink,,"likely compromised",,, -"2010-02-10 00:00:01",192.168.0.2,,,64512,ZZ,Region,City,node02.example.com,0,"Communications, Service Provider, and Hosting Service",cyclops-blink,,"likely compromised",,, -"2010-02-10 00:00:02",192.168.0.3,,,64512,ZZ,Region,City,node03.example.com,0,"Professional, Scientific, and Technical Services",cyclops-blink,,"likely compromised",,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/special.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/special.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/special.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later From 2dec6ec3ed4e693a539c52a9ce5bb3424b040465 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 11 Apr 2023 23:57:12 +0000 Subject: [PATCH 02/76] remove json parser - csv provides better performance --- .../shadowserver/collector_reports_api.py | 7 +- .../bots/parsers/shadowserver/parser_json.py | 171 ------------------ .../test_collector_reports_api.py | 7 +- 3 files changed, 7 insertions(+), 178 deletions(-) delete mode 100644 intelmq/bots/parsers/shadowserver/parser_json.py diff --git a/intelmq/bots/collectors/shadowserver/collector_reports_api.py b/intelmq/bots/collectors/shadowserver/collector_reports_api.py index 5e0b045c88..dc8bd6b420 100644 --- a/intelmq/bots/collectors/shadowserver/collector_reports_api.py +++ b/intelmq/bots/collectors/shadowserver/collector_reports_api.py @@ -34,7 +34,7 @@ class ShadowServerAPICollectorBot(CollectorBot, HttpMixin, CacheMixin): A list of strings or a comma-separated list of the mailing lists you want to process. types (list): A list of strings or a string of comma-separated values with the names of reporttypes you want to process. If you leave this empty, all the available reports will be downloaded and processed (i.e. 'scan', 'drones', 'intel', 'sandbox_connection', 'sinkhole_combined'). - file_format (str): File format to download ('csv' or 'json'). The default is 'json' for compatibility. Using 'csv' is recommended for best performance. + file_format (str): File format to download ('csv'). The 'json' option is not longer supported. """ country = None @@ -67,11 +67,10 @@ def init(self): self._report_list.append(self.country) if self.file_format is not None: - if not (self.file_format == 'csv' or self.file_format == 'json'): + if not (self.file_format == 'csv'): raise ValueError('Invalid file_format') else: - self.file_format = 'json' - self.logger.info("For best performance, set 'file_format' to 'csv' and use intelmq.bots.parsers.shadowserver.parser.") + self.file_format = 'csv' self.preamble = f'{{ "apikey": "{self.api_key}" ' diff --git a/intelmq/bots/parsers/shadowserver/parser_json.py b/intelmq/bots/parsers/shadowserver/parser_json.py deleted file mode 100644 index 893ad877b8..0000000000 --- a/intelmq/bots/parsers/shadowserver/parser_json.py +++ /dev/null @@ -1,171 +0,0 @@ -""" -Shadowserver JSON Parser - -SPDX-FileCopyrightText: 2020 Intelmq Team -SPDX-License-Identifier: AGPL-3.0-or-later -""" -import re -from typing import Any - -from intelmq.lib.bot import ParserBot -from intelmq.lib.exceptions import InvalidKey, InvalidValue -import intelmq.lib.message as libmessage -import intelmq.bots.parsers.shadowserver._config as config - - -class ShadowserverJSONParserBot(ParserBot): - """Parse all Shadowserver feeds in JSON format (data coming from the reports API) - Shadowserver JSON Parser - - Parameters: - feedname (str): The name of the feed - """ - __is_filename_regex = re.compile(r'^(?:\d{4}-\d{2}-\d{2}-)?(\w+)(-\w+)*\.json$') - feedname = None - _sparser_config = None - recover_line = ParserBot.recover_line_json - overwrite = True - - def init(self): - if self.feedname is not None: - feedname = self.feedname - self._sparser_config = config.get_feed_by_feedname(feedname) - if self._sparser_config: - self.logger.info('Using fixed feed name %r for parsing reports.', feedname) - else: - self.logger.info('Could not determine the feed by the feed name %r given by parameter. ' - 'Will determine the feed from the file names.', feedname) - - def parse(self, report): - report_name = report.get('extra.file_name') - if not report_name: - raise ValueError("No feedname given as parameter and the " - "processed report has no 'extra.file_name'. " - "Ensure that at least one is given. " - "Also have a look at the documentation of the bot.") - - filename_search = self.__is_filename_regex.search(report_name) - - if not filename_search: - raise ValueError(f"Report's 'extra.file_name' {report_name!r} is not valid.") - report_name = filename_search.group(1) - - self.logger.debug("Detected report's file name: %s.", report_name) - retval = config.get_feed_by_filename(report_name) - - if not retval: - raise ValueError('Could not get a config for {!r}, check the documentation.' - ''.format(report_name)) - self.feedname, self._sparser_config = retval - - return self.parse_json(report) - - def parse_line(self, line: Any, report: libmessage.Report): - conf = self._sparser_config - processedkeys = [] - - event = self.new_event(report) - event.add('feed.name', self.feedname, overwrite=self.overwrite) - - extra = {} - - for entry in conf.get('required_fields'): - intelmqkey, shadowserverkey = entry[0], entry[1] - value = self.get_value_from_config(line, entry) - - if value is not None: - event.add(intelmqkey, value) - processedkeys.append(shadowserverkey) - - # Now add optional fields. - # This action may fail, the value is added to - # extra if an add operation failed - for entry in conf.get('optional_fields'): - intelmqkey, shadowserverkey = entry[0], entry[1] - try: - value = self.get_value_from_config(line, entry) - except ValueError: - self.logger.warning('Optional key %s not found in feed %s. Possible change in data' - ' format or misconfiguration.', shadowserverkey, self.feedname) - continue - - intelmqkey, shadowserverkey = entry[0], entry[1] - if value is not None: - if intelmqkey == 'extra.': - extra[shadowserverkey] = value - processedkeys.append(shadowserverkey) - continue - elif intelmqkey and intelmqkey.startswith('extra.'): - extra[intelmqkey.replace('extra.', '', 1)] = value - processedkeys.append(shadowserverkey) - continue - elif intelmqkey is False: - # ignore it explicitly - processedkeys.append(shadowserverkey) - continue - try: - event.add(intelmqkey, value) - processedkeys.append(shadowserverkey) - except InvalidValue: - self.logger.debug('Could not add key %r in feed %r, adding it to extras.', - shadowserverkey, self.feedname) - except InvalidKey: - extra[intelmqkey] = value - processedkeys.append(shadowserverkey) - else: - processedkeys.append(shadowserverkey) - - # Now add additional constant fields. - event.update(conf.get('constant_fields', {})) - - event.add('raw', self.recover_line_json(line)) - - # Add everything which could not be resolved to extra. - for key in line: - if key not in processedkeys: - val = line[key] - if not val == "": - extra[key] = val - - if extra: - event.add('extra', extra) - - yield event - - def get_value_from_config(self, data, entry): - """ - Given a specific config, get the value for that data based on the entry - """ - conv_fun = None - - shadowserverkey = entry[1] - raw_value = data.get(shadowserverkey, None) - value = raw_value - - if raw_value is None: - raise ValueError('Key {!r} not found in feed {!r}. Possible change in data' - ' format or misconfiguration.'.format(shadowserverkey, self.feedname)) - if len(entry) > 2: - conv_fun = entry[2] - - if conv_fun is not None and raw_value is not None: - if len(entry) == 4 and entry[3]: - try: - value = conv_fun(raw_value, data) - except Exception: - self.logger.error('Could not convert shadowserverkey: %r in feed %r, ' - 'value: %r via conversion function %r.', - shadowserverkey, self.feedname, raw_value, conv_fun.__name__) - raise - else: - try: - value = conv_fun(raw_value) - except Exception: - self.logger.error('Could not convert shadowserverkey: %r in feed %r, ' - 'value: %r via conversion function %r.', - shadowserverkey, self.feedname, raw_value, conv_fun.__name__) - raise - return value - - -BOT = ShadowserverJSONParserBot diff --git a/intelmq/tests/bots/collectors/shadowserver/test_collector_reports_api.py b/intelmq/tests/bots/collectors/shadowserver/test_collector_reports_api.py index a625c9d34f..2bf6e61e9a 100644 --- a/intelmq/tests/bots/collectors/shadowserver/test_collector_reports_api.py +++ b/intelmq/tests/bots/collectors/shadowserver/test_collector_reports_api.py @@ -14,12 +14,13 @@ RANDSTR = secrets.token_urlsafe(50) ASSET_PATH = pathlib.Path(__file__).parent / 'reports-list.json' PARAMETERS = {'reports': 'anarres', 'api_key': RANDSTR, 'secret': RANDSTR, 'logging_level': 'DEBUG', 'types': ['scan_smb', 'cisco_smart_install', 'nonexistent'], 'name': 'shadowservercollector'} -REPORT = {'__type': 'Report', 'extra.file_name': '2020-08-02-scan_smb-anarres-geo.json', 'feed.accuracy': 100.0, 'feed.name': 'shadowservercollector', 'raw': 'e30='} +REPORT = {'__type': 'Report', 'extra.file_name': '2020-08-02-scan_smb-anarres-geo.csv', 'feed.accuracy': 100.0, 'feed.name': 'shadowservercollector', 'raw': 'e30='} def prepare_mocker(mocker): mocker.post('https://transform.shadowserver.org/api2/reports/list', content=ASSET_PATH.read_bytes()) - mocker.post('https://transform.shadowserver.org/api2/reports/download', text='{}') + mocker.get('https://dl.shadowserver.org/xNDSuwXrKnrLrDopU926rR75CAESMWesVCKsuyI8b8ncTv7GCX', text='{}') + mocker.get('https://dl.shadowserver.org/unnzVtn92tS9459rKIEz2J8qb7oJDv0Fa2feGUOiJLCDLqBXnN', text='{}') # Explicit skip_redis is required (although implicitly called by no_cache), otherwise fails in package build environments @@ -80,7 +81,7 @@ def test_report_sent(self, mocker): self.cache.flushdb() prepare_mocker(mocker) self.run_bot(iterations=1, parameters=PARAMETERS) - self.assertAnyLoglineEqual("Sent report: '2020-08-02-cisco_smart_install-anarres-geo.csv' (fixed: '2020-08-02-cisco_smart_install-anarres-geo.json', size: 0.00195 KiB).", 'DEBUG') + self.assertAnyLoglineEqual("Sent report: '2020-08-02-cisco_smart_install-anarres-geo.csv' (fixed: '2020-08-02-cisco_smart_install-anarres-geo.csv', size: 0.00195 KiB).", 'DEBUG') def test_report_content(self, mocker): self.cache.flushdb() From 04549613e9ace976d4f22f1ee3f6d7e20ee0a025 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 11 Apr 2023 23:59:42 +0000 Subject: [PATCH 03/76] dynamic configuration model --- intelmq/bots/parsers/shadowserver/README.md | 7 + intelmq/bots/parsers/shadowserver/_config.py | 4202 +---------------- intelmq/bots/parsers/shadowserver/parser.py | 46 +- .../parsers/shadowserver/schema.json.test | 180 + .../parsers/shadowserver/update_schema.py | 12 + 5 files changed, 303 insertions(+), 4144 deletions(-) create mode 100644 intelmq/bots/parsers/shadowserver/schema.json.test create mode 100644 intelmq/bots/parsers/shadowserver/update_schema.py diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md index eb0ddfb4a7..297930861b 100644 --- a/intelmq/bots/parsers/shadowserver/README.md +++ b/intelmq/bots/parsers/shadowserver/README.md @@ -7,3 +7,10 @@ This module is maintained by [The Shadowserver Foundation](https://www.shadowser Please contact intelmq@shadowserver.org with any issues or concerns. +The report configuration is now stored in a _schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/_version_. + +For environments that have internet connectivity the `update_schema.py` script should be setup as a cron job to obtain the latest revision. + +For air-gapped systems automation will be required to download and copy the _schema.json_ file into this directory + +The parser will automatically reload the configuration when the file changes. diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index bea3d0c0b8..a7b80b7a6c 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -77,20 +77,34 @@ feed_idx is not complete. """ +import os import re import base64 import binascii +import json +import urllib.request +import tempfile from typing import Optional, Dict, Tuple, Any import intelmq.lib.harmonization as harmonization +class __Container: + pass + +__config = __Container() +__config.schema_file = os.path.join(os.path.dirname(__file__), 'schema.json') +__config.schema_mtime = 0.0 +__config.feedname_mapping = {} +__config.filename_mapping = {} def get_feed_by_feedname(given_feedname: str) -> Optional[Dict[str, Any]]: - return feedname_mapping.get(given_feedname, None) + reload() + return __config.feedname_mapping.get(given_feedname, None) def get_feed_by_filename(given_filename: str) -> Optional[Tuple[str, Dict[str, Any]]]: - return filename_mapping.get(given_filename, None) + reload() + return __config.filename_mapping.get(given_filename, None) def add_UTC_to_timestamp(value: str) -> str: @@ -165,11 +179,6 @@ def invalidate_zero(value: str) -> Optional[int]: return int(value) if value and int(value) != 0 else None -# TODO this function is a wild guess... -def set_tor_node(value: str) -> Optional[bool]: - return True if value else None - - def validate_ip(value: str) -> Optional[str]: """Remove "invalid" IP.""" # FIX: https://github.com/certtools/intelmq/issues/1720 # TODO: Find better fix @@ -240,4126 +249,63 @@ def scan_exchange_identifier(field): return 'exchange-server-webshell' return 'vulnerable-exchange-server' +functions = { + 'add_UTC_to_timestamp': add_UTC_to_timestamp, + 'convert_bool': convert_bool, + 'validate_to_none': validate_to_none, + 'convert_int': convert_int, + 'convert_float': convert_float, + 'convert_http_host_and_url': convert_http_host_and_url, + 'invalidate_zero': invalidate_zero, + 'validate_ip': validate_ip, + 'validate_network': validate_network, + 'validate_fqdn': validate_fqdn, + 'convert_date': convert_date, + 'convert_date_utc': convert_date_utc, + 'force_base64': force_base64, + 'scan_exchange_taxonomy': scan_exchange_taxonomy, + 'scan_exchange_type': scan_exchange_type, + 'scan_exchange_identifier': scan_exchange_identifier, + } + + +def reload (): + """ reload the configuration if it has changed """ + mtime = 0.0 + + if (os.path.isfile(__config.schema_file)): + mtime = os.path.getmtime(__config.schema_file) + if __config.schema_mtime == mtime: + return + schema_file = __config.schema_file + else: + # load a test schema if one has not been downloaded yet + schema_file = __config.schema_file + schema_file += '.test' + + __config.feedname_mapping.clear() + __config.filename_mapping.clear() + with open(schema_file) as fh: + schema = json.load(fh) + for report in schema: + __config.feedname_mapping[schema[report]['feed_name']] = (schema[report]['feed_name'], schema[report]) + __config.filename_mapping[schema[report]['file_name']] = (schema[report]['feed_name'], schema[report]) + __config.schema_mtime = mtime + +def update_schema (version): + """ download the latest configuration """ + (th, tmp) = tempfile.mkstemp() + url = 'https://interchange.shadowserver.org/intelmq/'+version + try: + urllib.request.urlretrieve(url, tmp) + except: + raise ValueError("Failed to download %r" % url) -# BEGIN CONFGEN - -# https://www.shadowserver.org/what-we-do/network-reporting/blocklist-report/ -blocklist = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ], - 'optional_fields': [ - ('source.network', 'ip', validate_network), - ('extra.', 'tag', validate_to_none), - ('source.reverse_dns', 'hostname'), - ('extra.', 'source', validate_to_none), - ('extra.', 'reason', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'blacklisted-ip', - 'classification.taxonomy': 'other', - 'classification.type': 'blacklist', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/compromised-website-report/ -compromised_website = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.application', 'application', validate_to_none), - ('source.url', 'url', convert_http_host_and_url, True), - ('source.fqdn', 'http_host', validate_fqdn), - ('source.reverse_dns', 'hostname'), - ('malware.name', 'tag'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('event_description.text', 'category', validate_to_none), - ('extra.', 'system', validate_to_none), - ('extra.', 'detected_since', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'redirect_target', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'cc_url', validate_to_none), - ('extra.', 'family', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'intrusions', - 'classification.type': 'system-compromise', - 'classification.identifier': 'compromised-website', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/device-identification-report/ -device_id = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'undetermined', - 'classification.identifier': 'device-id', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/ddos-participant-report/ -event_ddos_participant = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'duration', convert_int), - ('extra.', 'attack_src_port', convert_int), - ('extra.', 'http_usessl', convert_bool), - ('extra.', 'ip_header_seqnum', convert_int), - ('extra.', 'ip_header_ttl', convert_int), - ('extra.', 'number_of_connections', convert_int), - ('extra.', 'packet_length', convert_int), - ('extra.', 'packet_randomized', convert_bool), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'domain_source', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'dst_network', validate_to_none), - ('extra.', 'dst_netmask', validate_to_none), - ('extra.', 'attack', validate_to_none), - ('extra.', 'attack_src_ip', validate_to_none), - ('extra.', 'domain', validate_to_none), - ('extra.', 'domain_transaction_id', validate_to_none), - ('extra.', 'gcip', validate_to_none), - ('extra.', 'http_method', validate_to_none), - ('extra.', 'http_path', validate_to_none), - ('extra.', 'http_postdata', validate_to_none), - ('extra.', 'ip_header_ack', validate_to_none), - ('extra.', 'ip_header_acknum', validate_to_none), - ('extra.', 'ip_header_dont_fragment', validate_to_none), - ('extra.', 'ip_header_fin', validate_to_none), - ('extra.', 'ip_header_identity', validate_to_none), - ('extra.', 'ip_header_psh', validate_to_none), - ('extra.', 'ip_header_rst', validate_to_none), - ('extra.', 'ip_header_syn', validate_to_none), - ('extra.', 'ip_header_tos', validate_to_none), - ('extra.', 'ip_header_urg', validate_to_none), - ('extra.', 'http_agent', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'classification.identifier': 'ddos-participant', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-brute-force-events-report/ -event_honeypot_brute_force = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('classification.identifier', 'application'), - ('destination.account', 'username', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'service', validate_to_none), - ('extra.', 'start_time', convert_date_utc), - ('extra.', 'end_time', convert_date_utc), - ('extra.', 'client_version', validate_to_none), - ('extra.', 'password', validate_to_none), - ('extra.', 'payload_url', validate_to_none), - ('extra.', 'payload_md5', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'intrusion-attempts', - 'classification.type': 'brute-force', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-darknet-events-report/ -event_honeypot_darknet = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('classification.identifier', 'tag', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'count', convert_int), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-ddos-events/ -event_honeypot_ddos = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'duration', convert_int), - ('extra.', 'attack_src_port', convert_int), - ('extra.', 'http_usessl', convert_bool), - ('extra.', 'ip_header_seqnum', convert_int), - ('extra.', 'ip_header_ttl', convert_int), - ('extra.', 'number_of_connections', convert_int), - ('extra.', 'packet_length', convert_int), - ('extra.', 'packet_randomized', convert_bool), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'domain_source', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'dst_network', validate_to_none), - ('extra.', 'dst_netmask', validate_to_none), - ('extra.', 'attack', validate_to_none), - ('extra.', 'attack_src_ip', validate_to_none), - ('extra.', 'domain', validate_to_none), - ('extra.', 'domain_transaction_id', validate_to_none), - ('extra.', 'gcip', validate_to_none), - ('extra.', 'http_method', validate_to_none), - ('extra.', 'http_path', validate_to_none), - ('extra.', 'http_postdata', validate_to_none), - ('extra.', 'ip_header_ack', validate_to_none), - ('extra.', 'ip_header_acknum', validate_to_none), - ('extra.', 'ip_header_dont_fragment', validate_to_none), - ('extra.', 'ip_header_fin', validate_to_none), - ('extra.', 'ip_header_identity', validate_to_none), - ('extra.', 'ip_header_psh', validate_to_none), - ('extra.', 'ip_header_rst', validate_to_none), - ('extra.', 'ip_header_syn', validate_to_none), - ('extra.', 'ip_header_tos', validate_to_none), - ('extra.', 'ip_header_urg', validate_to_none), - ('extra.', 'http_agent', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'classification.identifier': 'honeypot-ddos', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-amplification-ddos-events-report/ -event_honeypot_ddos_amp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'avg_pps', convert_float), - ('extra.', 'max_pps', convert_float), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'request', validate_to_none), - ('extra.', 'count', convert_int), - ('extra.', 'bytes', convert_int), - ('extra.', 'end_time', convert_date_utc), - ('extra.', 'duration', convert_int), - ], - 'constant_fields': { - 'classification.identifier': 'amplification-ddos-victim', - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-ddos-target-events-report/ -event_honeypot_ddos_target = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'attack_src_port', convert_int), - ('extra.', 'http_usessl', convert_bool), - ('extra.', 'ip_header_seqnum', convert_int), - ('extra.', 'ip_header_ttl', convert_int), - ('extra.', 'number_of_connections', convert_int), - ('extra.', 'packet_length', convert_int), - ('extra.', 'packet_randomized', convert_bool), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'domain_source', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'dst_network', validate_to_none), - ('extra.', 'dst_netmask', validate_to_none), - ('extra.', 'attack', validate_to_none), - ('extra.', 'duration', convert_int), - ('extra.', 'attack_src_ip', validate_to_none), - ('extra.', 'domain', validate_to_none), - ('extra.', 'domain_transaction_id', validate_to_none), - ('extra.', 'gcip', validate_to_none), - ('extra.', 'http_method', validate_to_none), - ('extra.', 'http_path', validate_to_none), - ('extra.', 'http_postdata', validate_to_none), - ('extra.', 'ip_header_ack', validate_to_none), - ('extra.', 'ip_header_acknum', validate_to_none), - ('extra.', 'ip_header_dont_fragment', validate_to_none), - ('extra.', 'ip_header_fin', validate_to_none), - ('extra.', 'ip_header_identity', validate_to_none), - ('extra.', 'ip_header_psh', validate_to_none), - ('extra.', 'ip_header_rst', validate_to_none), - ('extra.', 'ip_header_syn', validate_to_none), - ('extra.', 'ip_header_tos', validate_to_none), - ('extra.', 'ip_header_urg', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'classification.identifier': 'honeypot-ddos-target', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-http-scanner-events/ -event_honeypot_http_scan = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('user_agent', 'http_agent', validate_to_none), - ('extra.method', 'http_request_method', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'pattern', validate_to_none), - ('destination.url', 'http_url', convert_http_host_and_url, True), - ('extra.', 'url_scheme', validate_to_none), - ('extra.', 'session_tags', validate_to_none), - ('extra.', 'vulnerability_enum', validate_to_none), - ('extra.', 'vulnerability_id', validate_to_none), - ('extra.', 'vulnerability_class', validate_to_none), - ('extra.', 'vulnerability_score', validate_to_none), - ('extra.', 'vulnerability_severity', validate_to_none), - ('extra.', 'vulnerability_version', validate_to_none), - ('extra.', 'threat_framework', validate_to_none), - ('extra.', 'threat_tactic_id', validate_to_none), - ('extra.', 'threat_technique_id', validate_to_none), - ('extra.', 'target_vendor', validate_to_none), - ('extra.', 'target_product', validate_to_none), - ('extra.', 'target_class', validate_to_none), - ('extra.', 'file_md5', validate_to_none), - ('extra.', 'file_sha256', validate_to_none), - ('extra.', 'request_raw', force_base64), - ('extra.', 'body_raw', force_base64), - ], - 'constant_fields': { - 'classification.taxonomy': 'information-gathering', - 'classification.type': 'scanner', - 'protocol.application': 'http', - 'classification.identifier': 'honeypot-http-scan', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-ics-scanner-events-report/ -event_honeypot_ics_scan = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'state', validate_to_none), - ('extra.', 'sensor_id', validate_to_none), - ('extra.', 'slave_id', validate_to_none), - ('extra.', 'function_code', validate_to_none), - ('extra.', 'request', validate_to_none), - ('extra.', 'response', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'ics', - 'classification.taxonomy': 'information-gathering', - 'classification.type': 'scanner', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/ip-spoofer-events-report/ -event_ip_spoofer = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'infection', validate_to_none), - ('source.network', 'network', validate_network), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'routedspoof', validate_to_none), - ('extra.', 'session', validate_to_none), - ('extra.', 'nat', convert_bool), - ], - 'constant_fields': { - 'classification.taxonomy': 'fraud', - 'classification.type': 'masquerade', - 'classification.identifier': 'ip-spoofer', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/sinkhole-events-report/ -event_sinkhole = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('classification.identifier', 'infection', validate_to_none), - ('malware.name', 'family', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('extra.', 'infection', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/sinkhole-dns-events-report/ -event_sinkhole_dns = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('extra.naics', 'src_naics', invalidate_zero), - ('extra.sector', 'src_sector', validate_to_none), - ('extra.dns_query_type', 'query_type'), - ('extra.dns_query', 'query'), - ('malware.name', 'family', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('extra.', 'infection', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'count', convert_int), - ], - 'constant_fields': { - 'classification.identifier': 'sinkholedns', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'dns', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/sinkhole-http-events-report/ -event_sinkhole_http = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('classification.identifier', 'tag'), - ('malware.name', 'family', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('extra.', 'infection', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('destination.url', 'http_url', convert_http_host_and_url, True), - ('destination.fqdn', 'http_host', validate_fqdn), - ('extra.', 'http_agent', validate_to_none), - ('extra.', 'forwarded_by', validate_to_none), - ('extra.', 'ssl_cipher', validate_to_none), - ('extra.', 'http_referer', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'protocol.application': 'http', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/sinkhole-http-referer-events-report/ -event_sinkhole_http_referer = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ], - 'optional_fields': [ - ('malware.name', 'family', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('extra.', 'infection', validate_to_none), - ('protocol.transport', 'protocol'), - ('extra.', 'http_referer_ip', validate_ip), - ('extra.', 'http_referer_port', convert_int), - ('extra.', 'http_referer_asn', invalidate_zero), - ('extra.', 'http_referer_geo', validate_to_none), - ('extra.', 'http_referer_region', validate_to_none), - ('extra.', 'http_referer_city', validate_to_none), - ('extra.', 'http_referer_hostname', validate_to_none), - ('extra.', 'http_referer_naics', invalidate_zero), - ('extra.', 'http_referer_sector', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('destination.url', 'http_url', convert_http_host_and_url, True), - ('destination.fqdn', 'http_host', validate_fqdn), - ('extra.', 'http_referer', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'classification.identifier': 'sinkhole-http-referer', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/malware-url-report/ -malware_url = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ], - 'optional_fields': [ - ('source.url', 'url', convert_http_host_and_url, True), - ('source.fqdn', 'host', validate_fqdn), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('malware.name', 'tag'), - ('extra.', 'source', validate_to_none), - ('malware.hash.sha256', 'sha256', validate_to_none), - ('extra.', 'application', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'malware-distribution', - 'classification.identifier': 'malware-url', - }, -} - -phish_url = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ], - 'optional_fields': [ - ('source.url', 'url', convert_http_host_and_url, True), - ('source.fqdn', 'host', validate_fqdn), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'source', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'fraud', - 'classification.type': 'phishing', - 'classification.identifier': 'phish-url', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-http-proxy-report/ -population_http_proxy = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('malware.name', 'tag'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'http', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'connection', validate_to_none), - ('extra.', 'proxy_authenticate', validate_to_none), - ('extra.', 'via', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-http-proxy', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'http', - }, -} - -# http://www.shadowserver.org/wiki/pmwiki.php/Services/Sandbox-Connection -sandbox_conn = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('destination.fqdn', 'host', validate_fqdn), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('malware.hash.md5', 'md5', validate_to_none), - ('protocol.transport', 'protocol'), - ('extra.', 'bytes_in', validate_to_none), - ('extra.', 'bytes_out', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'malware-distribution', - 'classification.identifier': 'sandbox-conn', - }, -} - -sandbox_dns = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ], - 'optional_fields': [ - ('extra.dns_query_type', 'type', validate_to_none), - ('malware.hash.md5', 'md5hash', validate_to_none), - ('extra.', 'request', validate_to_none), - ('extra.', 'response', validate_to_none), - ('extra.', 'family', validate_to_none), - ('malware.name', 'tag'), - ('extra.', 'source', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'dns', - 'classification.identifier': 'sandbox-dns', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/sandbox-url-report/ -sandbox_url = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ], - 'optional_fields': [ - ('destination.fqdn', 'host', validate_fqdn), - ('extra.http_request_method', 'method', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('malware.hash.md5', 'md5', validate_to_none), - ('destination.url', 'url', convert_http_host_and_url, True), - ('user_agent', 'user_agent', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'malware-distribution', - 'classification.identifier': 'sandbox-url', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-adb-report/ -scan_adb = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'name', validate_to_none), - ('extra.', 'model', validate_to_none), - ('extra.', 'device', validate_to_none), - ('extra.', 'features', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_sector', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-adb', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'adb', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-afp-report/ -scan_afp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'machine_type', validate_to_none), - ('extra.', 'afp_versions', validate_to_none), - ('extra.', 'uams', validate_to_none), - ('extra.', 'flags', validate_to_none), - ('extra.', 'server_name', validate_to_none), - ('extra.', 'signature', validate_to_none), - ('extra.', 'directory_service', validate_to_none), - ('extra.', 'utf8_servername', validate_to_none), - ('extra.', 'network_address', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-afp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'afp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-amqp-report/ -scan_amqp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'channel', validate_to_none), - ('extra.', 'message_length', convert_int), - ('extra.', 'class', validate_to_none), - ('extra.', 'method', validate_to_none), - ('extra.', 'version_major', validate_to_none), - ('extra.', 'version_minor', validate_to_none), - ('extra.', 'capabilities', validate_to_none), - ('extra.', 'cluster_name', validate_to_none), - ('extra.', 'platform', validate_to_none), - ('extra.', 'product', validate_to_none), - ('extra.', 'product_version', validate_to_none), - ('extra.', 'mechanisms', validate_to_none), - ('extra.', 'locales', validate_to_none), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-amqp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'amqp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-apple-remote-desktop-ard-report/ -scan_ard = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'machine_name', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-ard', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-chargen-report/ -scan_chargen = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.response_size', 'size', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'chargen', - 'classification.identifier': 'open-chargen', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-cisco-smart-install-report/ -scan_cisco_smart_install = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-cisco-smart-install', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'cisco-smart-install', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-coap-report/ -scan_coap = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'response', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-coap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'coap', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-couchdb-report/ -scan_couchdb = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'server_version', validate_to_none), - ('extra.', 'couchdb_message', validate_to_none), - ('extra.', 'couchdb_version', validate_to_none), - ('extra.', 'git_sha', validate_to_none), - ('extra.', 'features', validate_to_none), - ('extra.', 'vendor', validate_to_none), - ('extra.', 'visible_databases', validate_to_none), - ('extra.', 'error', validate_to_none), - ('extra.', 'error_reason', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'CouchDB', - 'classification.identifier': 'open-couchdb', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-cwmp-report/ -scan_cwmp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'http', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'connection', validate_to_none), - ('extra.', 'www_authenticate', validate_to_none), - ('extra.', 'set_cookie', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'date', validate_to_none), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'cwmp', - 'classification.identifier': 'open-cwmp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-db2-discovery-service-report/ -scan_db2 = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'size', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'db2_hostname', validate_to_none), - ('extra.', 'servername', validate_to_none), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'open-db2-discovery-service', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'db2', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/vulnerable-ddos-middlebox-report/ -scan_ddos_middlebox = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.application', 'tag'), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'source_port', validate_to_none), - ('extra.', 'bytes', convert_int), - ('extra.', 'amplification', convert_float), - ('extra.', 'method', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-ddos-middlebox', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/dns-open-resolvers-report/ -scan_dns = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'min_amplification', convert_float), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'dns_version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'dns-open-resolver', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'dns', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-docker-service-report/ -scan_docker = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'http', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'date', validate_to_none), - ('extra.', 'experimental', validate_to_none), - ('extra.', 'api_version', validate_to_none), - ('extra.', 'arch', validate_to_none), - ('extra.', 'go_version', validate_to_none), - ('extra.os.name', 'os', validate_to_none), - ('extra.', 'kernel_version', validate_to_none), - ('extra.', 'git_commit', validate_to_none), - ('extra.', 'min_api_version', validate_to_none), - ('extra.', 'build_time', validate_to_none), - ('extra.', 'pkg_version', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'docker', - 'classification.identifier': 'open-docker', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-dvr-dhcpdiscover-report/ -scan_dvr_dhcpdiscover = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.application', 'tag'), - ('extra.', 'video_input_channels', convert_int), - ('extra.', 'alarm_input_channels', convert_int), - ('extra.', 'video_output_channels', convert_int), - ('extra.', 'alarm_output_channels', convert_int), - ('extra.', 'remote_video_input_channels', convert_int), - ('extra.', 'ipv4_dhcp_enable', convert_bool), - ('extra.', 'ipv6_dhcp_enable', convert_bool), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_id', validate_to_none), - ('extra.', 'device_serial', validate_to_none), - ('extra.', 'machine_name', validate_to_none), - ('extra.', 'manufacturer', validate_to_none), - ('extra.', 'method', validate_to_none), - ('extra.', 'http_port', convert_int), - ('extra.', 'internal_port', convert_int), - ('extra.', 'mac_address', validate_to_none), - ('extra.', 'ipv4_address', validate_to_none), - ('extra.', 'ipv4_gateway', validate_to_none), - ('extra.', 'ipv4_subnet_mask', validate_to_none), - ('extra.', 'ipv6_address', validate_to_none), - ('extra.', 'ipv6_link_local', validate_to_none), - ('extra.', 'ipv6_gateway', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-dvr-dhcpdiscover', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-elasticsearch-report/ -scan_elasticsearch = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'build_snapshot', convert_bool), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'ok', convert_bool), - ('extra.', 'name', validate_to_none), - ('extra.', 'cluster_name', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'build_hash', validate_to_none), - ('extra.', 'build_timestamp', validate_to_none), - ('extra.', 'lucene_version', validate_to_none), - ('extra.', 'tagline', validate_to_none), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'elasticsearch', - 'classification.identifier': 'open-elasticsearch', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-erlang-port-mapper-report-daemon/ -scan_epmd = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'nodes', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'Erlang Port Mapper Daemon', - 'classification.identifier': 'open-epmd', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/vulnerable-exchange-server-report/ -scan_exchange = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('classification.taxonomy', 'tag', scan_exchange_taxonomy), - ('classification.type', 'tag', scan_exchange_type), - ('classification.identifier', 'tag', scan_exchange_identifier), - ('extra.', 'tag', validate_to_none), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'servername', validate_to_none), - ('destination.url', 'url', convert_http_host_and_url, True), - ], - 'constant_fields': { - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-ftp-report/ -scan_ftp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'banner', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'auth_tls_response', validate_to_none), - ('extra.', 'auth_ssl_response', validate_to_none), - ('extra.', 'tlsv13_support', validate_to_none), - ('extra.', 'tlsv13_cipher', validate_to_none), - ('extra.', 'jarm', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_sector', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-ftp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ftp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-hadoop-report/ -scan_hadoop = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'total_disk', convert_int), - ('extra.', 'used_disk', convert_int), - ('extra.', 'free_disk', convert_int), - ('source.reverse_dns', 'hostname'), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'server_type', validate_to_none), - ('extra.', 'clusterid', validate_to_none), - ('extra.', 'livenodes', validate_to_none), - ('extra.', 'namenodeaddress', validate_to_none), - ('extra.', 'volumeinfo', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-hadoop', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'hadoop', - 'protocol.transport': 'tcp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-http-report/ -scan_http = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'http', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'connection', validate_to_none), - ('extra.', 'www_authenticate', validate_to_none), - ('extra.', 'set_cookie', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-http', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'http', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-http-proxy-report/ -scan_http_proxy = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'http', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'connection', validate_to_none), - ('extra.', 'proxy_authenticate', validate_to_none), - ('extra.', 'via', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), - ], - 'constant_fields': { - 'classification.identifier': 'open-http-proxy', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'http', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/vulnerable-http-report/ -scan_http_vulnerable = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'http', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'connection', validate_to_none), - ('extra.', 'www_authenticate', validate_to_none), - ('extra.', 'set_cookie', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), - ('extra.', 'version', validate_to_none), - ('extra.', 'build_date', validate_to_none), - ('extra.', 'detail', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-http', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'http', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-ics-report/ -scan_ics = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.application', 'tag'), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_id', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'raw_response', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-ics', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-ipmi-report/ -scan_ipmi = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'none_auth', convert_bool), - ('extra.', 'md2_auth', convert_bool), - ('extra.', 'md5_auth', convert_bool), - ('extra.', 'passkey_auth', convert_bool), - ('extra.', 'oem_auth', convert_bool), - ('extra.', 'permessage_auth', convert_bool), - ('extra.', 'userlevel_auth', convert_bool), - ('extra.', 'usernames', convert_bool), - ('extra.', 'nulluser', convert_bool), - ('extra.', 'anon_login', convert_bool), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'ipmi_version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'defaultkg', validate_to_none), - ('extra.', 'error', validate_to_none), - ('extra.', 'deviceid', validate_to_none), - ('extra.', 'devicerev', validate_to_none), - ('extra.', 'firmwarerev', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'manufacturerid', validate_to_none), - ('extra.', 'manufacturername', validate_to_none), - ('extra.', 'productid', validate_to_none), - ('extra.', 'productname', validate_to_none), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ipmi', - 'protocol.transport': 'udp', - 'classification.identifier': 'open-ipmi', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-ipp-report/ -scan_ipp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'ipp_version', validate_to_none), - ('extra.', 'cups_version', validate_to_none), - ('extra.', 'printer_uris', validate_to_none), - ('extra.', 'printer_name', validate_to_none), - ('extra.', 'printer_info', validate_to_none), - ('extra.', 'printer_more_info', validate_to_none), - ('extra.', 'printer_make_and_model', validate_to_none), - ('extra.', 'printer_firmware_name', validate_to_none), - ('extra.', 'printer_firmware_string_version', validate_to_none), - ('extra.', 'printer_firmware_version', validate_to_none), - ('extra.', 'printer_organization', validate_to_none), - ('extra.', 'printer_organization_unit', validate_to_none), - ('extra.', 'printer_uuid', validate_to_none), - ('extra.', 'printer_wifi_ssid', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ipp', - 'classification.identifier': 'open-ipp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/vulnerable-isakmp-report/ -scan_isakmp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'spi_size', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'initiator_spi', validate_to_none), - ('extra.', 'responder_spi', validate_to_none), - ('extra.', 'next_payload', validate_to_none), - ('extra.', 'exchange_type', validate_to_none), - ('extra.', 'flags', validate_to_none), - ('extra.', 'message_id', validate_to_none), - ('extra.', 'next_payload2', validate_to_none), - ('extra.', 'domain_of_interpretation', validate_to_none), - ('extra.', 'protocol_id', validate_to_none), - ('extra.', 'notify_message_type', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'open-ike', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ipsec', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-kubernetes-api-server-report/ -scan_kubernetes = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'http', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'date', validate_to_none), - ('extra.', 'major', validate_to_none), - ('extra.', 'minor', validate_to_none), - ('extra.', 'git_version', validate_to_none), - ('extra.', 'git_commit', validate_to_none), - ('extra.', 'git_tree_state', validate_to_none), - ('extra.', 'build_date', validate_to_none), - ('extra.', 'go_version', validate_to_none), - ('extra.', 'compiler', validate_to_none), - ('extra.', 'platform', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'browser_trusted', convert_bool), - ('extra.', 'browser_error', validate_to_none), - ('extra.', 'raw_cert', validate_to_none), - ('extra.', 'raw_cert_chain', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'kubernetes', - 'classification.identifier': 'open-kubernetes', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-ldap-tcp-report/ -scan_ldap_tcp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.local_hostname', 'dns_host_name', validate_to_none), - ('extra.', 'domain_controller_functionality', convert_int), - ('extra.', 'domain_functionality', convert_int), - ('extra.', 'forest_functionality', convert_int), - ('extra.', 'highest_committed_usn', convert_int), - ('extra.', 'is_global_catalog_ready', convert_bool), - ('extra.', 'is_synchronized', convert_bool), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'size', convert_int), - ('extra.', 'configuration_naming_context', validate_to_none), - ('extra.', 'current_time', validate_to_none), - ('extra.', 'default_naming_context', validate_to_none), - ('extra.', 'ds_service_name', validate_to_none), - ('extra.', 'ldap_service_name', validate_to_none), - ('extra.', 'naming_contexts', validate_to_none), - ('extra.', 'root_domain_naming_context', validate_to_none), - ('extra.', 'schema_naming_context', validate_to_none), - ('extra.', 'server_name', validate_to_none), - ('extra.', 'subschema_subentry', validate_to_none), - ('extra.', 'supported_capabilities', validate_to_none), - ('extra.', 'supported_control', validate_to_none), - ('extra.', 'supported_ldap_policies', validate_to_none), - ('extra.', 'supported_ldap_version', validate_to_none), - ('extra.', 'supported_sasl_mechanisms', validate_to_none), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ldap', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-ldap-report/ -scan_ldap_udp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.local_hostname', 'dns_host_name', validate_to_none), - ('extra.', 'domain_controller_functionality', convert_int), - ('extra.', 'domain_functionality', convert_int), - ('extra.', 'forest_functionality', convert_int), - ('extra.', 'highest_committed_usn', convert_int), - ('extra.', 'is_global_catalog_ready', convert_bool), - ('extra.', 'is_synchronized', convert_bool), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'size', convert_int), - ('extra.', 'configuration_naming_context', validate_to_none), - ('extra.', 'current_time', validate_to_none), - ('extra.', 'default_naming_context', validate_to_none), - ('extra.', 'ds_service_name', validate_to_none), - ('extra.', 'ldap_service_name', validate_to_none), - ('extra.', 'naming_contexts', validate_to_none), - ('extra.', 'root_domain_naming_context', validate_to_none), - ('extra.', 'schema_naming_context', validate_to_none), - ('extra.', 'server_name', validate_to_none), - ('extra.', 'subschema_subentry', validate_to_none), - ('extra.', 'supported_capabilities', validate_to_none), - ('extra.', 'supported_control', validate_to_none), - ('extra.', 'supported_ldap_policies', validate_to_none), - ('extra.', 'supported_ldap_version', validate_to_none), - ('extra.', 'supported_sasl_mechanisms', validate_to_none), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ldap', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-mdns-report/ -scan_mdns = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'mdns_name', validate_to_none), - ('extra.', 'mdns_ipv4', validate_to_none), - ('extra.', 'mdns_ipv6', validate_to_none), - ('extra.', 'services', validate_to_none), - ('extra.', 'workstation_name', validate_to_none), - ('extra.', 'workstation_ipv4', validate_to_none), - ('extra.', 'workstation_ipv6', validate_to_none), - ('extra.', 'workstation_info', validate_to_none), - ('extra.', 'http_name', validate_to_none), - ('extra.', 'http_ipv4', validate_to_none), - ('extra.', 'http_ipv6', validate_to_none), - ('extra.', 'http_ptr', validate_to_none), - ('extra.', 'http_info', validate_to_none), - ('extra.', 'http_target', validate_to_none), - ('extra.', 'http_port', convert_int), - ('extra.', 'spotify_name', validate_to_none), - ('extra.', 'spotify_ipv4', validate_to_none), - ('extra.', 'spotify_ipv6', validate_to_none), - ('extra.', 'opc_ua_discovery', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'mdns', - 'classification.identifier': 'open-mdns', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-memcached-report/ -scan_memcached = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'pid', convert_int), - ('extra.', 'pointer_size', convert_int), - ('extra.', 'uptime', convert_int), - ('extra.', 'curr_connections', convert_int), - ('extra.', 'total_connections', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'time', validate_to_none), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'memcached', - 'classification.identifier': 'open-memcached', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-mongodb-report/ -scan_mongodb = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'gitversion', validate_to_none), - ('extra.', 'sysinfo', validate_to_none), - ('extra.', 'opensslversion', validate_to_none), - ('extra.', 'allocator', validate_to_none), - ('extra.', 'javascriptengine', validate_to_none), - ('extra.', 'bits', validate_to_none), - ('extra.', 'maxbsonobjectsize', validate_to_none), - ('extra.', 'ok', convert_bool), - ('extra.', 'visible_databases', validate_to_none), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'mongodb', - 'classification.identifier': 'open-mongodb', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-mqtt-report/ -scan_mqtt = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'anonymous_access', convert_bool), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'raw_response', validate_to_none), - ('extra.', 'hex_code', validate_to_none), - ('extra.', 'code', validate_to_none), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serialNumber', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'mqtt', - 'classification.identifier': 'open-mqtt', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-mqtt-report/ -scan_mqtt_anon = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'raw_response', validate_to_none), - ('extra.', 'hex_code', validate_to_none), - ('extra.', 'code', validate_to_none), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serialNumber', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'mqtt', - 'classification.identifier': 'open-mqtt-anon', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-MSSQL -scan_mssql = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.local_hostname', 'server_name', validate_to_none), - ('extra.', 'tcp_port', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'instance_name', validate_to_none), - ('extra.', 'named_pipe', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'mssql', - 'classification.identifier': 'open-mssql', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-mysql-server-report/ -scan_mysql = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'client_can_handle_expired_passwords', convert_bool), - ('extra.', 'client_compress', convert_bool), - ('extra.', 'client_connect_attrs', convert_bool), - ('extra.', 'client_connect_with_db', convert_bool), - ('extra.', 'client_deprecated_eof', convert_bool), - ('extra.', 'client_found_rows', convert_bool), - ('extra.', 'client_ignore_sigpipe', convert_bool), - ('extra.', 'client_ignore_space', convert_bool), - ('extra.', 'client_interactive', convert_bool), - ('extra.', 'client_local_files', convert_bool), - ('extra.', 'client_long_flag', convert_bool), - ('extra.', 'client_long_password', convert_bool), - ('extra.', 'client_multi_results', convert_bool), - ('extra.', 'client_multi_statements', convert_bool), - ('extra.', 'client_no_schema', convert_bool), - ('extra.', 'client_odbc', convert_bool), - ('extra.', 'client_plugin_auth', convert_bool), - ('extra.', 'client_plugin_auth_len_enc_client_data', convert_bool), - ('extra.', 'client_protocol_41', convert_bool), - ('extra.', 'client_ps_multi_results', convert_bool), - ('extra.', 'client_reserved', convert_bool), - ('extra.', 'client_secure_connection', convert_bool), - ('extra.', 'client_session_track', convert_bool), - ('extra.', 'client_ssl', convert_bool), - ('extra.', 'client_transactions', convert_bool), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'mysql_protocol_version', validate_to_none), - ('extra.', 'server_version', validate_to_none), - ('extra.', 'error_code', validate_to_none), - ('extra.', 'error_id', validate_to_none), - ('extra.', 'error_message', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'browser_trusted', convert_bool), - ('extra.', 'browser_error', validate_to_none), - ('extra.', 'raw_cert', validate_to_none), - ('extra.', 'raw_cert_chain', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'mysql', - 'classification.identifier': 'open-mysql', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-NATPMP -scan_nat_pmp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'opcode', validate_to_none), - ('extra.', 'uptime', convert_int), - ('extra.', 'external_ip', validate_to_none), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'open-natpmp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'natpmp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-netbios-report/ -scan_netbios = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.account', 'username'), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'mac_address', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'workgroup', validate_to_none), - ('extra.', 'machine_name', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'open-netbios-nameservice', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'netbios-nameservice', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/netcore-netis-router-vulnerability-scan-report/ -scan_netis_router = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'response', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'open-netis', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.transport': 'udp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/ntp-version-report/ -scan_ntp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'clk_wander', convert_float), - ('extra.', 'frequency', convert_float), - ('extra.', 'jitter', convert_float), - ('extra.', 'leap', convert_float), - ('extra.', 'offset', convert_float), - ('extra.', 'peer', convert_int), - ('extra.', 'poll', convert_int), - ('extra.', 'precision', convert_int), - ('extra.', 'rootdelay', convert_float), - ('extra.', 'rootdispersion', convert_float), - ('extra.', 'stratum', convert_int), - ('extra.', 'tc', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'version', validate_to_none), - ('extra.', 'clock', validate_to_none), - ('extra.', 'error', validate_to_none), - ('extra.', 'mintc', validate_to_none), - ('extra.', 'noise', validate_to_none), - ('extra.', 'phase', validate_to_none), - ('extra.', 'processor', validate_to_none), - ('extra.', 'refid', validate_to_none), - ('extra.', 'reftime', validate_to_none), - ('extra.', 'stability', validate_to_none), - ('extra.', 'state', validate_to_none), - ('extra.', 'system', validate_to_none), - ('extra.', 'tai', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'ntp-version', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ntp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/ntp-monitor-report/ -scan_ntpmonitor = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'packets', convert_int), - ('extra.', 'size', convert_int), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'ntp-monitor', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ntp', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-Portmapper -scan_portmapper = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'programs', validate_to_none), - ('extra.', 'mountd_port', validate_to_none), - ('extra.', 'exports', validate_to_none), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'portmapper', - 'classification.identifier': 'open-portmapper', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-postgresql-server-report/ -scan_postgres = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'startup_error_line', convert_int), - ('extra.', 'client_ssl', convert_bool), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'supported_protocols', validate_to_none), - ('extra.', 'protocol_error_code', validate_to_none), - ('extra.', 'protocol_error_file', validate_to_none), - ('extra.', 'protocol_error_line', validate_to_none), - ('extra.', 'protocol_error_message', validate_to_none), - ('extra.', 'protocol_error_routine', validate_to_none), - ('extra.', 'protocol_error_severity', validate_to_none), - ('extra.', 'protocol_error_severity_v', validate_to_none), - ('extra.', 'startup_error_code', validate_to_none), - ('extra.', 'startup_error_file', validate_to_none), - ('extra.', 'startup_error_message', validate_to_none), - ('extra.', 'startup_error_routine', validate_to_none), - ('extra.', 'startup_error_severity', validate_to_none), - ('extra.', 'startup_error_severity_v', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'browser_trusted', convert_bool), - ('extra.', 'browser_error', validate_to_none), - ('extra.', 'raw_cert', validate_to_none), - ('extra.', 'raw_cert_chain', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'postgres', - 'classification.identifier': 'open-postgres', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-QOTD -scan_qotd = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'quote', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'qotd', - 'classification.identifier': 'open-qotd', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-quic-report/ -scan_quic = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'version_field_1', validate_to_none), - ('extra.', 'version_field_2', validate_to_none), - ('extra.', 'version_field_3', validate_to_none), - ('extra.', 'version_field_4', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'classification.identifier': 'open-quic', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-radmin-report/ -scan_radmin = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-radmin', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-rdp-report/ -scan_rdp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'cve20190708_vulnerable', convert_bool), - ('extra.', 'bluekeep_vulnerable', convert_bool), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'rdp_protocol', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'tlsv13_support', validate_to_none), - ('extra.', 'tlsv13_cipher', validate_to_none), - ('extra.', 'jarm', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'rdp', - 'protocol.transport': 'tcp', - 'classification.identifier': 'open-rdp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-ms-rdpeudp/ -scan_rdpeudp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sessionid', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-msrdpeudp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-Redis -scan_redis = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'git_sha1', validate_to_none), - ('extra.', 'git_dirty_flag', validate_to_none), - ('extra.', 'build_id', validate_to_none), - ('extra.', 'mode', validate_to_none), - ('extra.os.name', 'os', validate_to_none), - ('extra.', 'architecture', validate_to_none), - ('extra.', 'multiplexing_api', validate_to_none), - ('extra.', 'gcc_version', validate_to_none), - ('extra.', 'process_id', validate_to_none), - ('extra.', 'run_id', validate_to_none), - ('extra.', 'uptime', convert_int), - ('extra.', 'connected_clients', validate_to_none), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'redis', - 'classification.identifier': 'open-redis', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-rsync-report/ -scan_rsync = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'module', validate_to_none), - ('extra.', 'motd', validate_to_none), - ('extra.', 'has_password', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-rsync', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'rsync', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-sip-report/ -scan_sip = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'sip', validate_to_none), - ('extra.', 'sip_code', validate_to_none), - ('extra.', 'sip_reason', validate_to_none), - ('user_agent', 'user_agent', validate_to_none), - ('extra.', 'sip_via', validate_to_none), - ('extra.', 'sip_to', validate_to_none), - ('extra.', 'sip_from', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'content_type', validate_to_none), - ('extra.sip_server', 'server', validate_to_none), - ('extra.sip_contact', 'contact', validate_to_none), - ('extra.sip_cseq', 'cseq', validate_to_none), - ('extra.sip_call_id', 'call_id', validate_to_none), - ('extra.sip_allow', 'allow', validate_to_none), - ('extra.', 'amplification', convert_float), - ('extra.', 'response_size', convert_int), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'sip', - 'classification.identifier': 'open-sip', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-slp-service-report/ -scan_slp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'function', validate_to_none), - ('extra.', 'function_text', validate_to_none), - ('extra.', 'flags', validate_to_none), - ('extra.', 'next_extension_offset', validate_to_none), - ('extra.', 'xid', validate_to_none), - ('extra.', 'language_tag_length', validate_to_none), - ('extra.', 'language_tag', validate_to_none), - ('extra.', 'error_code', validate_to_none), - ('extra.', 'error_code_text', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'raw_response', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'slp', - 'classification.identifier': 'open-slp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-smb-report/ -scan_smb = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'smb_implant', convert_bool), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'arch', validate_to_none), - ('extra.', 'key', validate_to_none), - ('extra.', 'smbv1_support', validate_to_none), - ('extra.', 'smb_major_number', validate_to_none), - ('extra.', 'smb_minor_number', validate_to_none), - ('extra.', 'smb_revision', validate_to_none), - ('extra.', 'smb_version_string', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'smb', - 'protocol.transport': 'tcp', - 'classification.identifier': 'open-smb', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-smtp-report/ -scan_smtp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'banner', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'smtp', - 'classification.identifier': 'open-smtp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/vulnerable-smtp-report/ -scan_smtp_vulnerable = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'banner', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'smtp', - 'classification.identifier': 'vulnerable-smtp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-snmp-report/ -scan_snmp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'sysdesc', validate_to_none), - ('extra.', 'sysname', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'version', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_sector', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('extra.', 'community', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'snmp', - 'classification.identifier': 'open-snmp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-socks4-5-proxy-report/ -scan_socks = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.application', 'tag'), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-socks', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-SSDP -scan_ssdp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'header', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'systime', validate_to_none), - ('extra.', 'cache_control', validate_to_none), - ('extra.', 'location', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'search_target', validate_to_none), - ('extra.', 'unique_service_name', validate_to_none), - ('extra.', 'host', validate_to_none), - ('extra.', 'nts', validate_to_none), - ('extra.', 'nt', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'server_port', validate_to_none), - ('extra.', 'instance', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'updated_at', validate_to_none), - ('extra.', 'resource_identifier', validate_to_none), - ('extra.', 'amplification', convert_float), - ('extra.', 'response_size', convert_int), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ssdp', - 'classification.identifier': 'open-ssdp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-ssh-report/ -scan_ssh = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'serverid_raw', validate_to_none), - ('extra.', 'serverid_version', validate_to_none), - ('extra.', 'serverid_software', validate_to_none), - ('extra.', 'serverid_comment', validate_to_none), - ('extra.', 'server_cookie', validate_to_none), - ('extra.', 'available_kex', validate_to_none), - ('extra.', 'available_ciphers', validate_to_none), - ('extra.', 'available_mac', validate_to_none), - ('extra.', 'available_compression', validate_to_none), - ('extra.', 'selected_kex', validate_to_none), - ('extra.', 'algorithm', validate_to_none), - ('extra.', 'selected_cipher', validate_to_none), - ('extra.', 'selected_mac', validate_to_none), - ('extra.', 'selected_compression', validate_to_none), - ('extra.', 'server_signature_value', validate_to_none), - ('extra.', 'server_signature_raw', validate_to_none), - ('extra.', 'server_host_key', validate_to_none), - ('extra.', 'server_host_key_sha256', validate_to_none), - ('extra.', 'rsa_prime', validate_to_none), - ('extra.', 'rsa_prime_length', validate_to_none), - ('extra.', 'rsa_generator', validate_to_none), - ('extra.', 'rsa_generator_length', validate_to_none), - ('extra.', 'rsa_public_key', validate_to_none), - ('extra.', 'rsa_public_key_length', validate_to_none), - ('extra.', 'rsa_exponent', validate_to_none), - ('extra.', 'rsa_modulus', validate_to_none), - ('extra.', 'rsa_length', validate_to_none), - ('extra.', 'dss_prime', validate_to_none), - ('extra.', 'dss_prime_length', validate_to_none), - ('extra.', 'dss_generator', validate_to_none), - ('extra.', 'dss_generator_length', validate_to_none), - ('extra.', 'dss_public_key', validate_to_none), - ('extra.', 'dss_public_key_length', validate_to_none), - ('extra.', 'dss_dsa_public_g', validate_to_none), - ('extra.', 'dss_dsa_public_p', validate_to_none), - ('extra.', 'dss_dsa_public_q', validate_to_none), - ('extra.', 'dss_dsa_public_y', validate_to_none), - ('extra.', 'ecdsa_curve25519', validate_to_none), - ('extra.', 'ecdsa_curve', validate_to_none), - ('extra.', 'ecdsa_public_key_length', validate_to_none), - ('extra.', 'ecdsa_public_key_b', validate_to_none), - ('extra.', 'ecdsa_public_key_gx', validate_to_none), - ('extra.', 'ecdsa_public_key_gy', validate_to_none), - ('extra.', 'ecdsa_public_key_n', validate_to_none), - ('extra.', 'ecdsa_public_key_p', validate_to_none), - ('extra.', 'ecdsa_public_key_x', validate_to_none), - ('extra.', 'ecdsa_public_key_y', validate_to_none), - ('extra.', 'ed25519_curve25519', validate_to_none), - ('extra.', 'ed25519_cert_public_key_nonce', validate_to_none), - ('extra.', 'ed25519_cert_public_key_bytes', validate_to_none), - ('extra.', 'ed25519_cert_public_key_raw', validate_to_none), - ('extra.', 'ed25519_cert_public_key_sha256', validate_to_none), - ('extra.', 'ed25519_cert_public_key_serial', validate_to_none), - ('extra.', 'ed25519_cert_public_key_type_id', validate_to_none), - ('extra.', 'ed25519_cert_public_key_type_name', validate_to_none), - ('extra.', 'ed25519_cert_public_key_keyid', validate_to_none), - ('extra.', 'ed25519_cert_public_key_principles', validate_to_none), - ('extra.', 'ed25519_cert_public_key_valid_after', validate_to_none), - ('extra.', 'ed25519_cert_public_key_valid_before', validate_to_none), - ('extra.', 'ed25519_cert_public_key_duration', validate_to_none), - ('extra.', 'ed25519_cert_public_key_sigkey_bytes', validate_to_none), - ('extra.', 'ed25519_cert_public_key_sigkey_raw', validate_to_none), - ('extra.', 'ed25519_cert_public_key_sigkey_sha256', validate_to_none), - ('extra.', 'ed25519_cert_public_key_sigkey_value', validate_to_none), - ('extra.', 'ed25519_cert_public_key_sig_raw', validate_to_none), - ('extra.', 'banner', validate_to_none), - ('extra.', 'userauth_methods', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'classification.identifier': 'open-ssh', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-ssl-report/ -scan_ssl = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'ssl_poodle', convert_bool), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'freak_vulnerable', convert_bool), - ('extra.', 'freak_cipher_suite', validate_to_none), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'http_response_type', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'http_connection', validate_to_none), - ('extra.', 'www_authenticate', validate_to_none), - ('extra.', 'set_cookie', validate_to_none), - ('extra.', 'server_type', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'browser_trusted', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'browser_error', validate_to_none), - ('extra.', 'tlsv13_support', validate_to_none), - ('extra.', 'tlsv13_cipher', validate_to_none), - ('extra.', 'jarm', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'https', - 'classification.identifier': 'open-ssl', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Ssl-Freak-Scan -scan_ssl_freak = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'freak_vulnerable', convert_bool), - ('extra.', 'freak_cipher_suite', validate_to_none), - ('extra.', 'sector', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'http_response_type', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'http_connection', validate_to_none), - ('extra.', 'www_authenticate', validate_to_none), - ('extra.', 'set_cookie', validate_to_none), - ('extra.', 'server_type', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'browser_trusted', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'browser_error', validate_to_none), - ('extra.', 'tlsv13_support', validate_to_none), - ('extra.', 'tlsv13_cipher', validate_to_none), - ('extra.', 'raw_cert', validate_to_none), - ('extra.', 'raw_cert_chain', validate_to_none), - ('extra.', 'jarm', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_sector', validate_to_none), - ('extra.', 'page_sha256fp', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'https', - 'classification.identifier': 'ssl-freak', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Ssl-Scan -scan_ssl_poodle = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'ssl_poodle', convert_bool), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'http_response_type', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'http_connection', validate_to_none), - ('extra.', 'www_authenticate', validate_to_none), - ('extra.', 'set_cookie', validate_to_none), - ('extra.', 'server_type', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'browser_trusted', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'browser_error', validate_to_none), - ('extra.', 'tlsv13_support', validate_to_none), - ('extra.', 'tlsv13_cipher', validate_to_none), - ('extra.', 'raw_cert', validate_to_none), - ('extra.', 'raw_cert_chain', validate_to_none), - ('extra.', 'jarm', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_sector', validate_to_none), - ('extra.', 'page_sha256fp', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'https', - 'classification.identifier': 'ssl-poodle', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-stun-service-report/ -scan_stun = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'mapped_port', convert_int), - ('extra.', 'xor_mapped_port', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'transaction_id', validate_to_none), - ('extra.', 'magic_cookie', validate_to_none), - ('extra.', 'message_length', convert_int), - ('extra.', 'message_type', validate_to_none), - ('extra.', 'mapped_family', validate_to_none), - ('extra.', 'mapped_address', validate_to_none), - ('extra.', 'xor_mapped_family', validate_to_none), - ('extra.', 'xor_mapped_address', validate_to_none), - ('extra.', 'software', validate_to_none), - ('extra.', 'fingerprint', validate_to_none), - ('extra.', 'amplification', convert_float), - ('extra.', 'response_size', convert_int), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'Session Traversal Utilities for NAT', - 'classification.identifier': 'open-stun', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/synful-scan-report/ -scan_synfulknock = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'ack_number', convert_int), - ('extra.', 'window_size', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'sequence_number', validate_to_none), - ('extra.', 'urgent_pointer', validate_to_none), - ('extra.', 'tcp_flags', validate_to_none), - ('extra.', 'raw_packet', validate_to_none), - ('extra.source.sector', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-synfulknock', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-telnet-report/ -scan_telnet = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'banner', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'telnet', - 'classification.identifier': 'open-telnet', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-TFTP -scan_tftp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'opcode', validate_to_none), - ('extra.', 'errorcode', validate_to_none), - ('extra.', 'error', validate_to_none), - ('extra.', 'errormessage', validate_to_none), - ('extra.', 'size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'tftp', - 'classification.identifier': 'open-tftp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-ubiquiti-report/ -scan_ubiquiti = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.mac_address', 'mac', validate_to_none), - ('extra.radio_name', 'radioname', validate_to_none), - ('extra.model', 'modelshort', validate_to_none), - ('extra.model_full', 'modelfull', validate_to_none), - ('extra.firmwarerev', 'firmware', validate_to_none), - ('extra.response_size', 'size', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'essid', validate_to_none), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-ubiquiti-discovery-service', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-vnc-report/ -scan_vnc = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.reverse_dns', 'hostname'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'product', validate_to_none), - ('extra.', 'banner', validate_to_none), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'vnc', - 'protocol.transport': 'tcp', - 'classification.identifier': 'open-vnc', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-ws-discovery-service-report/ -scan_ws_discovery = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ('extra.', 'error', validate_to_none), - ('extra.', 'raw_response', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ws-discovery', - 'classification.identifier': 'open-ws-discovery', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-xdmcp-service-report/ -scan_xdmcp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'opcode', validate_to_none), - ('extra.', 'reported_hostname', validate_to_none), - ('extra.', 'status', validate_to_none), - ('extra.', 'size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'xdmcp', - 'classification.identifier': 'open-xdmcp', - }, -} - -# http://www.shadowserver.org/wiki/pmwiki.php/Services/Spam-URL -spam_url = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ], - 'optional_fields': [ - ('source.url', 'url', convert_http_host_and_url, True), - ('source.fqdn', 'host', validate_fqdn), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'source', validate_to_none), - ('extra.', 'sender', validate_to_none), - ('extra.', 'subject', validate_to_none), - ('malware.hash.md5', 'md5', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'abusive-content', - 'classification.type': 'spam', - 'classification.identifier': 'spam-url', - }, -} - -special = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('source.reverse_dns', 'hostname'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('malware.name', 'tag'), - ('extra.', 'public_source', validate_to_none), - ('extra.', 'status', validate_to_none), - ('extra.', 'detail', validate_to_none), - ('extra.', 'method', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'special', - }, -} - -mapping = ( - # feed name, file name, function - ('Blocklist', 'blocklist', blocklist), - ('Compromised-Website', 'compromised_website', compromised_website), - ('Device-Identification IPv4', 'device_id', device_id), - ('Device-Identification IPv6', 'device_id6', device_id), - ('DDoS-Participant', 'event4_ddos_participant', event_ddos_participant), - ('Honeypot-Brute-Force-Events', 'event4_honeypot_brute_force', event_honeypot_brute_force), - ('Honeypot-Darknet', 'event4_honeypot_darknet', event_honeypot_darknet), - ('Honeypot-DDoS', 'event4_honeypot_ddos', event_honeypot_ddos), - ('Honeypot-Amplification-DDoS-Events', 'event4_honeypot_ddos_amp', event_honeypot_ddos_amp), - ('Honeypot-DDoS-Target', 'event4_honeypot_ddos_target', event_honeypot_ddos_target), - ('Honeypot-HTTP-Scan', 'event4_honeypot_http_scan', event_honeypot_http_scan), - ('Honeypot-ICS-Scanner', 'event4_honeypot_ics_scan', event_honeypot_ics_scan), - ('IP-Spoofer-Events', 'event4_ip_spoofer', event_ip_spoofer), - ('Microsoft-Sinkhole-Events IPv4', 'event4_microsoft_sinkhole', event_sinkhole), - ('Microsoft-Sinkhole-Events-HTTP IPv4', 'event4_microsoft_sinkhole_http', event_sinkhole_http), - ('Sinkhole-Events IPv4', 'event4_sinkhole', event_sinkhole), - ('Sinkhole-DNS', 'event4_sinkhole_dns', event_sinkhole_dns), - ('Sinkhole-Events-HTTP IPv4', 'event4_sinkhole_http', event_sinkhole_http), - ('Sinkhole-Events-HTTP-Referer IPv4', 'event4_sinkhole_http_referer', event_sinkhole_http_referer), - ('Sinkhole-Events IPv6', 'event6_sinkhole', event_sinkhole), - ('Sinkhole-Events-HTTP IPv6', 'event6_sinkhole_http', event_sinkhole_http), - ('Sinkhole-Events-HTTP-Referer IPv6', 'event6_sinkhole_http_referer', event_sinkhole_http_referer), - ('Malware-URL', 'malware_url', malware_url), - ('Phish-URL', 'phish_url', phish_url), - ('IPv6-Accessible-HTTP-Proxy', 'population6_http_proxy', population_http_proxy), - ('Accessible-HTTP-Proxy', 'population_http_proxy', population_http_proxy), - ('Sandbox-Connections', 'sandbox_conn', sandbox_conn), - ('Sandbox-DNS', 'sandbox_dns', sandbox_dns), - ('Sandbox-URL', 'sandbox_url', sandbox_url), - ('IPv6-Accessible-CWMP', 'scan6_cwmp', scan_cwmp), - ('IPv6-DNS-Open-Resolvers', 'scan6_dns', scan_dns), - ('IPv6-Vulnerable-Exchange', 'scan6_exchange', scan_exchange), - ('IPv6-Accessible-FTP', 'scan6_ftp', scan_ftp), - ('IPv6-Accessible-HTTP', 'scan6_http', scan_http), - ('IPv6-Open-HTTP-Proxy', 'scan6_http_proxy', scan_http_proxy), - ('IPv6-Vulnerable-HTTP', 'scan6_http_vulnerable', scan_http_vulnerable), - ('IPv6-Open-IPP', 'scan6_ipp', scan_ipp), - ('IPv6-Open-LDAP-TCP', 'scan6_ldap_tcp', scan_ldap_tcp), - ('IPv6-Open-MQTT', 'scan6_mqtt', scan_mqtt), - ('IPv6-Open-Anonymous-MQTT', 'scan6_mqtt_anon', scan_mqtt_anon), - ('IPv6-Accessible-MySQL', 'scan6_mysql', scan_mysql), - ('IPv6-NTP-Version', 'scan6_ntp', scan_ntp), - ('IPv6-NTP-Monitor', 'scan6_ntpmonitor', scan_ntpmonitor), - ('IPv6-Accessible-PostgreSQL', 'scan6_postgres', scan_postgres), - ('IPv6-Accessible-RDP', 'scan6_rdp', scan_rdp), - ('IPv6-Accessible-SLP', 'scan6_slp', scan_slp), - ('IPv6-Accessible-SMB', 'scan6_smb', scan_smb), - ('IPv6-Accessible-SMTP', 'scan6_smtp', scan_smtp), - ('IPv6-Vulnerable-SMTP', 'scan6_smtp_vulnerable', scan_smtp_vulnerable), - ('IPv6-Open-SNMP', 'scan6_snmp', scan_snmp), - ('IPv6-Accessible-SSH', 'scan6_ssh', scan_ssh), - ('IPv6-Accessible-SSL', 'scan6_ssl', scan_ssl), - ('SSL-FREAK-Vulnerable-Servers IPv6', 'scan6_ssl_freak', scan_ssl_freak), - ('SSL-POODLE-Vulnerable-Servers IPv6', 'scan6_ssl_poodle', scan_ssl_poodle), - ('IPv6-Accessible-Session-Traversal-Utilities-for-NAT', 'scan6_stun', scan_stun), - ('IPv6-Accessible-Telnet', 'scan6_telnet', scan_telnet), - ('IPv6-Accessible-VNC', 'scan6_vnc', scan_vnc), - ('Accessible-ADB', 'scan_adb', scan_adb), - ('Accessible-AFP', 'scan_afp', scan_afp), - ('Accessible-AMQP', 'scan_amqp', scan_amqp), - ('Accessible-ARD', 'scan_ard', scan_ard), - ('Open-Chargen', 'scan_chargen', scan_chargen), - ('Accessible-Cisco-Smart-Install', 'scan_cisco_smart_install', scan_cisco_smart_install), - ('Accessible-CoAP', 'scan_coap', scan_coap), - ('Accessible-CouchDB', 'scan_couchdb', scan_couchdb), - ('Accessible-CWMP', 'scan_cwmp', scan_cwmp), - ('Open-DB2-Discovery-Service', 'scan_db2', scan_db2), - ('Vulnerable-DDoS-Middlebox', 'scan_ddos_middlebox', scan_ddos_middlebox), - ('DNS-Open-Resolvers', 'scan_dns', scan_dns), - ('Accessible-Docker', 'scan_docker', scan_docker), - ('Accessible-DVR-DHCPDiscover', 'scan_dvr_dhcpdiscover', scan_dvr_dhcpdiscover), - ('Open-Elasticsearch', 'scan_elasticsearch', scan_elasticsearch), - ('Accessible-Erlang-Port-Mapper-Daemon', 'scan_epmd', scan_epmd), - ('Vulnerable-Exchange-Server', 'scan_exchange', scan_exchange), - ('Accessible-FTP', 'scan_ftp', scan_ftp), - ('Accessible-Hadoop', 'scan_hadoop', scan_hadoop), - ('Accessible-HTTP', 'scan_http', scan_http), - ('Open-HTTP-Proxy', 'scan_http_proxy', scan_http_proxy), - ('Vulnerable-HTTP', 'scan_http_vulnerable', scan_http_vulnerable), - ('Accessible-ICS', 'scan_ics', scan_ics), - ('Open-IPMI', 'scan_ipmi', scan_ipmi), - ('Open-IPP', 'scan_ipp', scan_ipp), - ('Vulnerable-ISAKMP', 'scan_isakmp', scan_isakmp), - ('Accessible-Kubernetes-API', 'scan_kubernetes', scan_kubernetes), - ('Open-LDAP-TCP', 'scan_ldap_tcp', scan_ldap_tcp), - ('Open-LDAP', 'scan_ldap_udp', scan_ldap_udp), - ('Open-mDNS', 'scan_mdns', scan_mdns), - ('Open-Memcached', 'scan_memcached', scan_memcached), - ('Open-MongoDB', 'scan_mongodb', scan_mongodb), - ('Open-MQTT', 'scan_mqtt', scan_mqtt), - ('Open-Anonymous-MQTT', 'scan_mqtt_anon', scan_mqtt_anon), - ('Open-MSSQL', 'scan_mssql', scan_mssql), - ('Accessible-MySQL', 'scan_mysql', scan_mysql), - ('Open-NATPMP', 'scan_nat_pmp', scan_nat_pmp), - ('Open-NetBIOS-Nameservice', 'scan_netbios', scan_netbios), - ('Open-Netis', 'scan_netis_router', scan_netis_router), - ('NTP-Version', 'scan_ntp', scan_ntp), - ('NTP-Monitor', 'scan_ntpmonitor', scan_ntpmonitor), - ('Open-Portmapper', 'scan_portmapper', scan_portmapper), - ('Accessible-PostgreSQL', 'scan_postgres', scan_postgres), - ('Open-QOTD', 'scan_qotd', scan_qotd), - ('Accessible-QUIC', 'scan_quic', scan_quic), - ('Accessible-Radmin', 'scan_radmin', scan_radmin), - ('Accessible-RDP', 'scan_rdp', scan_rdp), - ('Accessible-MS-RDPEUDP', 'scan_rdpeudp', scan_rdpeudp), - ('Open-Redis', 'scan_redis', scan_redis), - ('Accessible-Rsync', 'scan_rsync', scan_rsync), - ('Accessible-SIP', 'scan_sip', scan_sip), - ('Accessible-SLP', 'scan_slp', scan_slp), - ('Accessible-SMB', 'scan_smb', scan_smb), - ('Accessible-SMTP', 'scan_smtp', scan_smtp), - ('Vulnerable-SMTP', 'scan_smtp_vulnerable', scan_smtp_vulnerable), - ('Open-SNMP', 'scan_snmp', scan_snmp), - ('Accessible-SOCKS4/5-Proxy', 'scan_socks', scan_socks), - ('Open-SSDP', 'scan_ssdp', scan_ssdp), - ('Accessible-SSH', 'scan_ssh', scan_ssh), - ('Accessible-SSL', 'scan_ssl', scan_ssl), - ('SSL-FREAK-Vulnerable-Servers', 'scan_ssl_freak', scan_ssl_freak), - ('SSL-POODLE-Vulnerable-Servers IPv4', 'scan_ssl_poodle', scan_ssl_poodle), - ('Accessible-Session-Traversal-Utilities-for-NAT', 'scan_stun', scan_stun), - ('SYNful-Knock', 'scan_synfulknock', scan_synfulknock), - ('Accessible-Telnet', 'scan_telnet', scan_telnet), - ('Open-TFTP', 'scan_tftp', scan_tftp), - ('Accessible-Ubiquiti-Discovery-Service', 'scan_ubiquiti', scan_ubiquiti), - ('Accessible-VNC', 'scan_vnc', scan_vnc), - ('Accessible-WS-Discovery-Service', 'scan_ws_discovery', scan_ws_discovery), - ('Open-XDMCP', 'scan_xdmcp', scan_xdmcp), - ('Spam-URL', 'spam_url', spam_url), - ('Special', 'special', special), - ('Accessible-RDPEUDP', 'scan_rdpeudp', scan_rdpeudp), - ('Sinkhole-Events', 'event4_sinkhole', event_sinkhole), - ('Sinkhole-Events-HTTP', 'event4_sinkhole_http', event_sinkhole_http), - ('Sinkhole-Events-HTTP-Referer', 'event4_sinkhole_http_referer', event_sinkhole_http_referer), -) -# END CONFGEN + try: + with open(tmp) as fh: + schema = json.load(fh) + except: + # leave tempfile behind for diagnosis + raise ValueError("Failed to validate %r" % tmp) -feedname_mapping = {feedname: function for feedname, filename, function in mapping} -filename_mapping = {filename: (feedname, function) for feedname, filename, function in mapping} + os.replace(tmp, __config.schema_file) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index 70ba3b4bb6..f14549141a 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -22,6 +22,7 @@ """ import copy import re +import os from intelmq.lib.bot import ParserBot from intelmq.lib.exceptions import InvalidKey, InvalidValue @@ -29,7 +30,13 @@ class ShadowserverParserBot(ParserBot): - """Parse all ShadowServer feeds""" + """ + Parse all ShadowServer feeds + + Parameters: + schema_file (str): Path to the report schema file + + """ recover_line = ParserBot.recover_line_csv_dict _csv_params = {'dialect': 'unix'} @@ -124,10 +131,17 @@ def parse_line(self, row, report): value = raw_value if conv_func is not None and raw_value is not None: - if len(item) == 4 and item[3]: - value = conv_func(raw_value, row) - else: - value = conv_func(raw_value) + try: + if len(item) == 4 and item[3]: + value = config.functions[conv_func](raw_value, row) + else: + value = config.functions[conv_func](raw_value) + except Exception: + """ fail early and often in this case. We want to be able to convert everything """ + self.logger.error('Could not convert shadowkey: %r in feed %r, ' + 'value: %r via conversion function %r.', + shadowkey, self.feedname, raw_value, conv_func) + raise if value is not None: event.add(intelmqkey, value) @@ -153,17 +167,17 @@ def parse_line(self, row, report): value = raw_value if conv_func is not None and raw_value is not None: - if len(item) == 4 and item[3]: - value = conv_func(raw_value, row) - else: - try: - value = conv_func(raw_value) - except Exception: - """ fail early and often in this case. We want to be able to convert everything """ - self.logger.error('Could not convert shadowkey: %r in feed %r, ' - 'value: %r via conversion function %r.', - shadowkey, self.feedname, raw_value, conv_func.__name__) - raise + try: + if len(item) == 4 and item[3]: + value = config.functions[conv_func](raw_value, row) + else: + value = config.functions[conv_func](raw_value) + except Exception: + """ fail early and often in this case. We want to be able to convert everything """ + self.logger.error('Could not convert shadowkey: %r in feed %r, ' + 'value: %r via conversion function %r.', + shadowkey, self.feedname, raw_value, conv_func) + raise if value is not None: if intelmqkey == 'extra.': diff --git a/intelmq/bots/parsers/shadowserver/schema.json.test b/intelmq/bots/parsers/shadowserver/schema.json.test new file mode 100644 index 0000000000..2cfb8bb1d3 --- /dev/null +++ b/intelmq/bots/parsers/shadowserver/schema.json.test @@ -0,0 +1,180 @@ +{ + "test_smb" : { + "constant_fields" : { + "classification.identifier" : "test-smb", + "classification.taxonomy" : "vulnerable", + "classification.type" : "vulnerable-system", + "protocol.application" : "smb", + "protocol.transport" : "tcp" + }, + "feed_name" : "Test-Accessible-SMB", + "file_name" : "test_smb", + "optional_fields" : [ + [ + "extra.", + "smb_implant", + "convert_bool" + ], + [ + "source.reverse_dns", + "hostname" + ], + [ + "extra.", + "tag" + ], + [ + "source.asn", + "asn", + "invalidate_zero" + ], + [ + "source.geolocation.cc", + "geo" + ], + [ + "source.geolocation.region", + "region" + ], + [ + "source.geolocation.city", + "city" + ], + [ + "extra.source.naics", + "naics", + "invalidate_zero" + ], + [ + "extra.source.sic", + "sic", + "invalidate_zero" + ], + [ + "extra.", + "arch", + "validate_to_none" + ], + [ + "extra.", + "key", + "validate_to_none" + ], + [ + "extra.", + "smbv1_support", + "validate_to_none" + ], + [ + "extra.", + "smb_major_number", + "validate_to_none" + ], + [ + "extra.", + "smb_minor_number", + "validate_to_none" + ], + [ + "extra.", + "smb_revision", + "validate_to_none" + ], + [ + "extra.", + "smb_version_string", + "validate_to_none" + ] + ], + "required_fields" : [ + [ + "time.source", + "timestamp", + "add_UTC_to_timestamp" + ], + [ + "source.ip", + "ip", + "validate_ip" + ], + [ + "source.port", + "port", + "convert_int" + ] + ] + }, + "test_telnet" : { + "constant_fields" : { + "classification.identifier" : "test-telnet", + "classification.taxonomy" : "vulnerable", + "classification.type" : "vulnerable-system", + "protocol.application" : "telnet" + }, + "feed_name" : "Test-Accessible-Telnet", + "file_name" : "test_telnet", + "optional_fields" : [ + [ + "protocol.transport", + "protocol" + ], + [ + "source.reverse_dns", + "hostname" + ], + [ + "extra.", + "tag", + "validate_to_none" + ], + [ + "source.asn", + "asn", + "invalidate_zero" + ], + [ + "source.geolocation.cc", + "geo" + ], + [ + "source.geolocation.region", + "region" + ], + [ + "source.geolocation.city", + "city" + ], + [ + "extra.", + "naics", + "invalidate_zero" + ], + [ "extra.", + "sic", + "invalidate_zero" + ], + [ + "extra.", + "banner", + "validate_to_none" + ] + ], + "required_fields" : [ + [ + "time.source", + "timestamp", + "add_UTC_to_timestamp" + ], + [ + "source.ip", + "ip", + "validate_ip" + ], + [ + "source.port", + "port", + "convert_int" + ] + ] + } +} diff --git a/intelmq/bots/parsers/shadowserver/update_schema.py b/intelmq/bots/parsers/shadowserver/update_schema.py new file mode 100644 index 0000000000..040f672593 --- /dev/null +++ b/intelmq/bots/parsers/shadowserver/update_schema.py @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: 2023 The Shadowserver Foundation +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +# -*- coding: utf-8 -*- + +import os +import intelmq.bots.parsers.shadowserver._config as config + +if __name__ == '__main__': # pragma: no cover + exec(open(os.path.join(os.path.dirname(__file__), '../../../version.py')).read()) # defines __version__ + config.update_schema(__version__) From 0a39e0de01db827faa9a603c97b0846effdb0cbb Mon Sep 17 00:00:00 2001 From: elsif2 Date: Wed, 12 Apr 2023 00:01:32 +0000 Subject: [PATCH 04/76] revised tests --- .../bots/parsers/shadowserver/test_broken.py | 12 +- .../bots/parsers/shadowserver/test_mapping.py | 8 +- .../parsers/shadowserver/test_parameters.py | 37 +++--- .../parsers/shadowserver/test_report_smb.py | 124 ++++++++++++++++++ .../shadowserver/test_report_switch.py | 16 +-- .../shadowserver/test_report_telnet.py | 87 ++++++++++++ .../shadowserver/testdata/test_smb.csv | 4 + .../testdata/test_smb.csv.license | 2 + .../shadowserver/testdata/test_telnet.csv | 3 + .../testdata/test_telnet.csv.license | 2 + 10 files changed, 260 insertions(+), 35 deletions(-) create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_report_smb.py create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv.license create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv.license diff --git a/intelmq/tests/bots/parsers/shadowserver/test_broken.py b/intelmq/tests/bots/parsers/shadowserver/test_broken.py index 472dd0b90c..2b803142eb 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_broken.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_broken.py @@ -13,12 +13,12 @@ REPORT1 = {"raw": utils.base64_encode('adasdasdasdasd\nadasdasdafgf'), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_http-test-test.csv", + "extra.file_name": "2019-01-01-test_smb-test-test.csv", } REPORT2 = {"raw": utils.base64_encode('timestamp,ip,port\n2018-08-01T00:00:00+00,127.0.0.1,80'), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ftp-test-test.csv", + "extra.file_name": "2019-01-01-test_telnet-test-test.csv", } REPORT3 = {"raw": utils.base64_encode('adasdasdasdasd\nadasdasdafgf'), "__type": "Report", @@ -48,10 +48,10 @@ def test_broken(self): """ self.input_message = REPORT1 self.run_bot(allowed_error_count=1) - self.assertLogMatches(pattern="Detected report's file name: 'scan_http'.", + self.assertLogMatches(pattern="Detected report's file name: 'test_smb'.", levelname="DEBUG") self.assertLogMatches(pattern="Failed to parse line.") - self.assertLogMatches(pattern="ValueError: Required column 'timestamp' not found in feed 'Accessible-HTTP'. Possible change in data format or misconfiguration.") + self.assertLogMatches(pattern="ValueError: Required column 'timestamp' not found in feed 'Test-Accessible-SMB'. Possible change in data format or misconfiguration.") self.assertLogMatches(pattern=r"Sent 0 events and found 1 problem\(s\)\.", levelname="INFO") @@ -61,9 +61,9 @@ def test_half_broken(self): """ self.input_message = REPORT2 self.run_bot(allowed_warning_count=63) - self.assertLogMatches(pattern="Detected report's file name: 'scan_ftp'.", + self.assertLogMatches(pattern="Detected report's file name: 'test_telnet'.", levelname="DEBUG") - self.assertLogMatches(pattern="Optional key 'jarm' not found in feed 'Accessible-FTP'.", + self.assertLogMatches(pattern="Optional key 'banner' not found in feed 'Test-Accessible-Telnet'.", levelname="WARNING") self.assertLogMatches(pattern=r"Sent 1 events and found 0 problem\(s\)\.", levelname="INFO") diff --git a/intelmq/tests/bots/parsers/shadowserver/test_mapping.py b/intelmq/tests/bots/parsers/shadowserver/test_mapping.py index f58aed66eb..6a2af94475 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_mapping.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_mapping.py @@ -11,22 +11,22 @@ with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_telnet.csv')) as handle: + 'testdata/test_telnet.csv')) as handle: TELNET_FILE = handle.read() EXAMPLE_TELNET = { "raw": utils.base64_encode(TELNET_FILE), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_telnet.csv", + "extra.file_name": "2019-01-01-test_telnet.csv", } with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_vnc.csv')) as handle: + 'testdata/test_smb.csv')) as handle: TELNET_FILE = handle.read() EXAMPLE_VNC = { "raw": utils.base64_encode(TELNET_FILE), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_vnc.csv", + "extra.file_name": "2019-01-01-test_smb.csv", } diff --git a/intelmq/tests/bots/parsers/shadowserver/test_parameters.py b/intelmq/tests/bots/parsers/shadowserver/test_parameters.py index a5ea81f199..677cd0319b 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_parameters.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_parameters.py @@ -12,38 +12,41 @@ from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_dns.csv')) as handle: + 'testdata/test_smb.csv')) as handle: EXAMPLE_FILE = handle.read() EXAMPLE_LINES = EXAMPLE_FILE.splitlines() EXAMPLE_REPORT = {"raw": utils.base64_encode(EXAMPLE_FILE), "__type": "Report", "time.observation": "2018-07-30T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_dns-test-test.csv", + "extra.file_name": "2019-01-01-test_smb-test-test.csv", 'feed.name': 'report feedname', } EVENTS = [{ '__type': 'Event', 'feed.name': 'report feedname', - "classification.identifier": "dns-open-resolver", + "classification.identifier": 'test-smb', "classification.taxonomy": "vulnerable", "classification.type": "vulnerable-system", - "extra.dns_version": "dnsmasq-2.66", - "extra.min_amplification": 4.619, - "extra.tag": "openresolver", - "protocol.application": "dns", - "protocol.transport": "udp", + "extra.smb_implant": False, + "extra.smb_major_number": '2', + "extra.smb_minor_number": '1', + "extra.smb_version_string": 'SMB 2.1', + "extra.smbv1_support": 'N', + "extra.tag": "smb", + "protocol.application": "smb", + "protocol.transport": "tcp", 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - "source.asn": 25255, - "source.geolocation.cc": "AT", - "source.geolocation.city": "VIENNA", - "source.geolocation.region": "WIEN", - "source.ip": "198.51.100.179", - "source.port": 53, - "source.reverse_dns": "198-51-100-189.example.net", + "source.asn": 64512, + "source.geolocation.cc": "ZZ", + "source.geolocation.city": "City", + "source.geolocation.region": "Region", + "source.ip": "192.168.0.1", + "source.port": 445, + "source.reverse_dns": "node01.example.com", "time.observation": "2018-07-30T00:00:00+00:00", - "time.source": "2018-04-14T00:14:34+00:00" + "time.source": "2010-02-10T00:00:00+00:00" }, ] @@ -70,7 +73,7 @@ def test_overwrite_feed_name(self): self.run_bot(prepare=False) for i, EVENT in enumerate(EVENTS): event = EVENT.copy() - event['feed.name'] = 'DNS-Open-Resolvers' + event['feed.name'] = 'Test-Accessible-SMB' self.assertMessageEqual(i, event) diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py b/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py new file mode 100644 index 0000000000..c7eefdf0a9 --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py @@ -0,0 +1,124 @@ +# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +# -*- coding: utf-8 -*- + +import os +import unittest + +import intelmq.lib.test as test +import intelmq.lib.utils as utils +from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot + +with open(os.path.join(os.path.dirname(__file__), + 'testdata/test_smb.csv')) as handle: + EXAMPLE_FILE = handle.read() +EXAMPLE_LINES = EXAMPLE_FILE.splitlines() + +EXAMPLE_REPORT = {'feed.name': 'Test-Accessible-SMB', + "raw": utils.base64_encode(EXAMPLE_FILE), + "__type": "Report", + "time.observation": "2015-01-01T00:00:00+00:00", + "extra.file_name": "2019-01-01-test_smb-test-geo.csv", + } +EVENTS = [ +{ + '__type' : 'Event', + 'classification.identifier' : 'test-smb', + 'classification.taxonomy' : 'vulnerable', + 'classification.type' : 'vulnerable-system', + 'extra.smb_implant' : False, + 'extra.smb_major_number' : '2', + 'extra.smb_minor_number' : '1', + 'extra.smb_version_string' : 'SMB 2.1', + 'extra.smbv1_support' : 'N', + 'extra.tag' : 'smb', + 'feed.name' : 'Test-Accessible-SMB', + 'protocol.application' : 'smb', + 'protocol.transport' : 'tcp', + 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), + 'source.asn' : 64512, + 'source.geolocation.cc' : 'ZZ', + 'source.geolocation.city' : 'City', + 'source.geolocation.region' : 'Region', + 'source.ip' : '192.168.0.1', + 'source.port' : 445, + 'source.reverse_dns' : 'node01.example.com', + 'time.observation' : '2015-01-01T00:00:00+00:00', + 'time.source' : '2010-02-10T00:00:00+00:00' +}, + +{ + '__type' : 'Event', + 'classification.identifier' : 'test-smb', + 'classification.taxonomy' : 'vulnerable', + 'classification.type' : 'vulnerable-system', + 'extra.smb_implant' : False, + 'extra.smb_major_number' : '2', + 'extra.smb_minor_number' : '1', + 'extra.smb_version_string' : 'SMB 2.1', + 'extra.smbv1_support' : 'N', + 'extra.tag' : 'smb', + 'feed.name' : 'Test-Accessible-SMB', + 'protocol.application' : 'smb', + 'protocol.transport' : 'tcp', + 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), + 'source.asn' : 64512, + 'source.geolocation.cc' : 'ZZ', + 'source.geolocation.city' : 'City', + 'source.geolocation.region' : 'Region', + 'source.ip' : '192.168.0.2', + 'source.port' : 445, + 'source.reverse_dns' : 'node02.example.com', + 'time.observation' : '2015-01-01T00:00:00+00:00', + 'time.source' : '2010-02-10T00:00:01+00:00' +}, + +{ + '__type' : 'Event', + 'classification.identifier' : 'test-smb', + 'classification.taxonomy' : 'vulnerable', + 'classification.type' : 'vulnerable-system', + 'extra.smb_implant' : False, + 'extra.smb_major_number' : '2', + 'extra.smb_minor_number' : '1', + 'extra.smb_version_string' : 'SMB 2.1', + 'extra.smbv1_support' : 'N', + 'extra.tag' : 'smb', + 'feed.name' : 'Test-Accessible-SMB', + 'protocol.application' : 'smb', + 'protocol.transport' : 'tcp', + 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), + 'source.asn' : 64512, + 'source.geolocation.cc' : 'ZZ', + 'source.geolocation.city' : 'City', + 'source.geolocation.region' : 'Region', + 'source.ip' : '192.168.0.3', + 'source.port' : 445, + 'source.reverse_dns' : 'node03.example.com', + 'time.observation' : '2015-01-01T00:00:00+00:00', + 'time.source' : '2010-02-10T00:00:02+00:00' +} + ] + + +class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): + """ + A TestCase for a ShadowserverParserBot. + """ + + @classmethod + def set_bot(cls): + cls.bot_reference = ShadowserverParserBot + cls.default_input_message = EXAMPLE_REPORT + + def test_event(self): + """ Test if correct Event has been produced. """ + self.run_bot() + for i, EVENT in enumerate(EVENTS): + self.assertMessageEqual(i, EVENT) + + +if __name__ == '__main__': # pragma: no cover + unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py b/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py index 0a34a69f0a..570d612fb4 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py @@ -12,24 +12,24 @@ from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ftp.csv')) as handle: + 'testdata/test_smb.csv')) as handle: EXAMPLE_LINES = handle.read().splitlines()[:2] -FIRST_REPORT = {'feed.name': 'Accessible FTP', +FIRST_REPORT = {'feed.name': 'Test-Accessible-SMB', "raw": utils.base64_encode('\n'.join(EXAMPLE_LINES)), "__type": "Report", "time.observation": "2019-03-25T00:00:00+00:00", - "extra.file_name": "2019-03-25-scan_ftp-test-test.csv", + "extra.file_name": "2019-03-25-test_smb-test-test.csv", } -with open(os.path.join(os.path.dirname(__file__), 'testdata/blocklist.csv')) as handle: +with open(os.path.join(os.path.dirname(__file__), 'testdata/test_telnet.csv')) as handle: EXAMPLE_LINES = handle.read().splitlines()[:2] SECOND_REPORT = { - 'feed.name': 'Blocklist', + 'feed.name': 'Test-Accessible-Telnet', "raw": utils.base64_encode('\n'.join(EXAMPLE_LINES)), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-blocklist-test-geo.csv", + "extra.file_name": "2019-01-01-test_telnet-test-geo.csv", } @@ -48,9 +48,9 @@ def test_event(self): """ Test if the parser correctly detects and handles different report types. """ self.input_message = [FIRST_REPORT, SECOND_REPORT] self.run_bot(iterations=2) - self.assertLogMatches("Detected report's file name: 'scan_ftp'", + self.assertLogMatches("Detected report's file name: 'test_smb'", levelname='DEBUG') - self.assertLogMatches("Detected report's file name: 'blocklist'", + self.assertLogMatches("Detected report's file name: 'test_telnet'", levelname='DEBUG') diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py b/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py new file mode 100644 index 0000000000..6d539ac4a7 --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py @@ -0,0 +1,87 @@ +# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +# -*- coding: utf-8 -*- + +import os +import unittest + +import intelmq.lib.test as test +import intelmq.lib.utils as utils +from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot + +with open(os.path.join(os.path.dirname(__file__), + 'testdata/test_telnet.csv')) as handle: + EXAMPLE_FILE = handle.read() +EXAMPLE_LINES = EXAMPLE_FILE.splitlines() + +EXAMPLE_REPORT = {'feed.name': 'Test-Accessible-Telnet', + "raw": utils.base64_encode(EXAMPLE_FILE), + "__type": "Report", + "time.observation": "2015-01-01T00:00:00+00:00", + "extra.file_name": "2019-01-01-test_telnet-test-geo.csv", + } +EVENTS = [{'__type': 'Event', + 'feed.name': 'Test-Accessible-Telnet', + "classification.identifier": "test-telnet", + "classification.taxonomy": "vulnerable", + "classification.type": "vulnerable-system", + "extra.banner": "|MikroTik v6.5|Login:", + "extra.tag": "telnet-alt", + "protocol.application": "telnet", + "protocol.transport": "tcp", + 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], + EXAMPLE_LINES[1]])), + "source.asn": 20255, + "source.geolocation.cc": "AA", + "source.geolocation.city": "LOCATION", + "source.geolocation.region": "LOCATION", + "source.ip": "198.123.245.145", + "source.port": 5678, + "source.reverse_dns": "example.local", + "time.observation": "2015-01-01T00:00:00+00:00", + "time.source": "2019-09-04T12:27:34+00:00" + }, + {'__type': 'Event', + 'feed.name': 'Test-Accessible-Telnet', + "classification.identifier": "test-telnet", + "classification.taxonomy": "vulnerable", + "classification.type": "vulnerable-system", + "extra.banner": "|MikroTik v6.45.3 (stable)|Login:", + "extra.tag": "telnet-alt", + "protocol.application": "telnet", + "protocol.transport": "tcp", + 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], + EXAMPLE_LINES[2]])), + "source.asn": 20255, + "source.geolocation.cc": "AA", + "source.geolocation.city": "LOCATION", + "source.geolocation.region": "LOCATION", + "source.ip": "198.123.245.145", + "source.port": 5678, + "source.reverse_dns": "example.local", + "time.observation": "2015-01-01T00:00:00+00:00", + "time.source": "2019-09-04T12:27:40+00:00" + }] + + +class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): + """ + A TestCase for a ShadowserverParserBot. + """ + + @classmethod + def set_bot(cls): + cls.bot_reference = ShadowserverParserBot + cls.default_input_message = EXAMPLE_REPORT + + def test_event(self): + """ Test if correct Event has been produced. """ + self.run_bot() + for i, EVENT in enumerate(EVENTS): + self.assertMessageEqual(i, EVENT) + + +if __name__ == '__main__': # pragma: no cover + unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv new file mode 100644 index 0000000000..fc7fe2fff6 --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv @@ -0,0 +1,4 @@ +"timestamp","ip","port","hostname","tag","asn","geo","region","city","naics","sic","smb_implant","arch","key","smbv1_support","smb_major_number","smb_minor_number","smb_revision","smb_version_string" +"2010-02-10 00:00:00",192.168.0.1,445,node01.example.com,smb,64512,ZZ,Region,City,0,0,N,,,N,2,1,0,"SMB 2.1" +"2010-02-10 00:00:01",192.168.0.2,445,node02.example.com,smb,64512,ZZ,Region,City,0,0,N,,,N,2,1,0,"SMB 2.1" +"2010-02-10 00:00:02",192.168.0.3,445,node03.example.com,smb,64512,ZZ,Region,City,0,0,N,,,N,2,1,0,"SMB 2.1" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv.license new file mode 100644 index 0000000000..f512a890e4 --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv.license @@ -0,0 +1,2 @@ +SPDX-FileCopyrightText: 2022 The Shadowserver Foundation +SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv new file mode 100644 index 0000000000..3309e9a3d8 --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv @@ -0,0 +1,3 @@ +"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","banner" +"2019-09-04 12:27:34","198.123.245.145","tcp",5678,"example.local","telnet-alt",20255,"AA","LOCATION","LOCATION",0,0,"|MikroTik v6.5|Login:" +"2019-09-04 12:27:40","198.123.245.145","tcp",5678,"example.local","telnet-alt",20255,"AA","LOCATION","LOCATION",0,0,"|MikroTik v6.45.3 (stable)|Login:" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv.license new file mode 100644 index 0000000000..942a94035d --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv.license @@ -0,0 +1,2 @@ +SPDX-FileCopyrightText: 2019 Guillermo Rodriguez +SPDX-License-Identifier: AGPL-3.0-or-later From 94b22fb677173f968bd57f02ed6a056f6c2cb5c7 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Mon, 8 May 2023 15:05:12 +0000 Subject: [PATCH 05/76] Updated to reset report type on reload #2361 --- intelmq/bots/parsers/shadowserver/README.md | 2 +- intelmq/bots/parsers/shadowserver/_config.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md index 297930861b..bb6216b9a7 100644 --- a/intelmq/bots/parsers/shadowserver/README.md +++ b/intelmq/bots/parsers/shadowserver/README.md @@ -11,6 +11,6 @@ The report configuration is now stored in a _schema.json_ file downloaded from h For environments that have internet connectivity the `update_schema.py` script should be setup as a cron job to obtain the latest revision. -For air-gapped systems automation will be required to download and copy the _schema.json_ file into this directory +For air-gapped systems automation will be required to download and copy the _schema.json_ file into this directory. The parser will automatically reload the configuration when the file changes. diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index a7b80b7a6c..29382d2782 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -272,15 +272,14 @@ def scan_exchange_identifier(field): def reload (): """ reload the configuration if it has changed """ mtime = 0.0 + schema_file = __config.schema_file - if (os.path.isfile(__config.schema_file)): + if os.path.isfile(__config.schema_file): mtime = os.path.getmtime(__config.schema_file) if __config.schema_mtime == mtime: return - schema_file = __config.schema_file else: # load a test schema if one has not been downloaded yet - schema_file = __config.schema_file schema_file += '.test' __config.feedname_mapping.clear() From b2f9bc371ed35a88c3179e0d6ed002c43362368d Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 23 May 2023 01:12:47 +0000 Subject: [PATCH 06/76] Added schema download on startup and additional logging --- intelmq/bots/parsers/shadowserver/_config.py | 33 +++++++++++++------ intelmq/bots/parsers/shadowserver/parser.py | 1 + .../parsers/shadowserver/update_schema.py | 3 +- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index 29382d2782..f766be3221 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -106,6 +106,8 @@ def get_feed_by_filename(given_filename: str) -> Optional[Tuple[str, Dict[str, A reload() return __config.filename_mapping.get(given_filename, None) +def set_logger(logger): + __config.logger = logger def add_UTC_to_timestamp(value: str) -> str: return value + ' UTC' @@ -272,29 +274,38 @@ def scan_exchange_identifier(field): def reload (): """ reload the configuration if it has changed """ mtime = 0.0 - schema_file = __config.schema_file if os.path.isfile(__config.schema_file): mtime = os.path.getmtime(__config.schema_file) if __config.schema_mtime == mtime: return else: - # load a test schema if one has not been downloaded yet - schema_file += '.test' + __config.logger.info("The schema file does not exist.") + + if __config.schema_mtime == 0.0 and mtime == 0.0 and not os.environ.get('INTELMQ_SKIP_INTERNET'): + __config.logger.info("Attempting to download schema.") + update_schema() __config.feedname_mapping.clear() __config.filename_mapping.clear() - with open(schema_file) as fh: - schema = json.load(fh) - for report in schema: - __config.feedname_mapping[schema[report]['feed_name']] = (schema[report]['feed_name'], schema[report]) - __config.filename_mapping[schema[report]['file_name']] = (schema[report]['feed_name'], schema[report]) + for schema_file in [ __config.schema_file, ".".join([__config.schema_file, 'test']) ]: + if os.path.isfile(schema_file): + with open(schema_file) as fh: + schema = json.load(fh) + for report in schema: + if report == "_meta": + __config.logger.info("Loading schema %s." % schema[report]['date_created']) + for msg in schema[report]['change_log']: + __config.logger.info(msg) + else: + __config.feedname_mapping[schema[report]['feed_name']] = (schema[report]['feed_name'], schema[report]) + __config.filename_mapping[schema[report]['file_name']] = (schema[report]['feed_name'], schema[report]) __config.schema_mtime = mtime -def update_schema (version): +def update_schema (): """ download the latest configuration """ (th, tmp) = tempfile.mkstemp() - url = 'https://interchange.shadowserver.org/intelmq/'+version + url = 'https://interchange.shadowserver.org/intelmq/v1' try: urllib.request.urlretrieve(url, tmp) except: @@ -307,4 +318,6 @@ def update_schema (version): # leave tempfile behind for diagnosis raise ValueError("Failed to validate %r" % tmp) + if os.path.exists(__config.schema_file): + os.replace(__config.schema_file, ".".join([__config.schema_file, 'bak'])) os.replace(tmp, __config.schema_file) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index f14549141a..2f20262bfa 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -47,6 +47,7 @@ class ShadowserverParserBot(ParserBot): overwrite = False def init(self): + config.set_logger(self.logger) if self.feedname is not None: self._sparser_config = config.get_feed_by_feedname(self.feedname) if self._sparser_config: diff --git a/intelmq/bots/parsers/shadowserver/update_schema.py b/intelmq/bots/parsers/shadowserver/update_schema.py index 040f672593..a7975147ed 100644 --- a/intelmq/bots/parsers/shadowserver/update_schema.py +++ b/intelmq/bots/parsers/shadowserver/update_schema.py @@ -8,5 +8,4 @@ import intelmq.bots.parsers.shadowserver._config as config if __name__ == '__main__': # pragma: no cover - exec(open(os.path.join(os.path.dirname(__file__), '../../../version.py')).read()) # defines __version__ - config.update_schema(__version__) + config.update_schema() From d5cf063756d0a087d17531cde546d920d7768703 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 23 May 2023 23:32:53 +0000 Subject: [PATCH 07/76] Added version support to the schema update function. --- intelmq/bots/parsers/shadowserver/README.md | 6 ++-- intelmq/bots/parsers/shadowserver/_config.py | 32 +++++++++++++++++--- intelmq/bots/parsers/shadowserver/parser.py | 4 +++ 3 files changed, 34 insertions(+), 8 deletions(-) diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md index bb6216b9a7..c757020e94 100644 --- a/intelmq/bots/parsers/shadowserver/README.md +++ b/intelmq/bots/parsers/shadowserver/README.md @@ -7,10 +7,10 @@ This module is maintained by [The Shadowserver Foundation](https://www.shadowser Please contact intelmq@shadowserver.org with any issues or concerns. -The report configuration is now stored in a _schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/_version_. +The report configuration is now stored in a _schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/v1. -For environments that have internet connectivity the `update_schema.py` script should be setup as a cron job to obtain the latest revision. +For environments that have internet connectivity the `update_schema.py` script can be called from a cron job to obtain the latest revision. -For air-gapped systems automation will be required to download and copy the _schema.json_ file into this directory. +For air-gapped systems automation will be required to download and copy the _schema.json_ file into this directory. The parser will automatically reload the configuration when the file changes. diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index f766be3221..bb67db525a 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -97,6 +97,11 @@ class __Container: __config.feedname_mapping = {} __config.filename_mapping = {} +def set_logger(logger): + """ Sets the logger instance. """ + __config.logger = logger + + def get_feed_by_feedname(given_feedname: str) -> Optional[Dict[str, Any]]: reload() return __config.feedname_mapping.get(given_feedname, None) @@ -106,8 +111,6 @@ def get_feed_by_filename(given_filename: str) -> Optional[Tuple[str, Dict[str, A reload() return __config.filename_mapping.get(given_filename, None) -def set_logger(logger): - __config.logger = logger def add_UTC_to_timestamp(value: str) -> str: return value + ' UTC' @@ -304,20 +307,39 @@ def reload (): def update_schema (): """ download the latest configuration """ - (th, tmp) = tempfile.mkstemp() + if os.environ.get('INTELMQ_SKIP_INTERNET'): + return None + + (th, tmp) = tempfile.mkstemp(dir=os.path.dirname(__file__)) url = 'https://interchange.shadowserver.org/intelmq/v1' try: urllib.request.urlretrieve(url, tmp) except: raise ValueError("Failed to download %r" % url) + new_version = '' + old_version = '' + try: with open(tmp) as fh: schema = json.load(fh) + new_version = schema['_meta']['date_created'] except: # leave tempfile behind for diagnosis raise ValueError("Failed to validate %r" % tmp) if os.path.exists(__config.schema_file): - os.replace(__config.schema_file, ".".join([__config.schema_file, 'bak'])) - os.replace(tmp, __config.schema_file) + old_version = '' + try: + with open(__config.schema_file) as fh: + schema = json.load(fh) + old_version = schema['_meta']['date_created'] + if new_version != old_version: + os.replace(__config.schema_file, ".".join([__config.schema_file, 'bak'])) + except: + pass + + if new_version != old_version: + os.replace(tmp, __config.schema_file) + else: + os.unlink(tmp) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index 2f20262bfa..71489e2ec1 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -48,6 +48,10 @@ class ShadowserverParserBot(ParserBot): def init(self): config.set_logger(self.logger) + try: + config.update_schema() + except Exception as e: + logger.warning(f"Schema update failed: {e}.") if self.feedname is not None: self._sparser_config = config.get_feed_by_feedname(self.feedname) if self._sparser_config: From 1e6ea8982b085b8e0e4d2224c4f574803097be63 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Sun, 28 May 2023 23:13:54 +0000 Subject: [PATCH 08/76] Documentation and style updates. --- CHANGELOG.md | 6 + .../shadowserver/collector_reports_api.py | 2 +- intelmq/bots/parsers/shadowserver/README.md | 39 ++++- intelmq/bots/parsers/shadowserver/_config.py | 52 +++--- intelmq/bots/parsers/shadowserver/parser.py | 2 +- .../bots/parsers/shadowserver/test_broken.py | 4 +- .../bots/parsers/shadowserver/test_mapping.py | 1 - .../parsers/shadowserver/test_report_smb.py | 151 +++++++++--------- .../shadowserver/test_report_switch.py | 10 +- .../shadowserver/test_report_telnet.py | 4 +- 10 files changed, 154 insertions(+), 117 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea160a7c5c..6b54eb84d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,15 +62,21 @@ CHANGELOG ### Bots #### Collectors +<<<<<<< HEAD - `intelmq.bots.collector.rt`: - restrict `python-rt` to be below version 3.0 due to introduced breaking changes, - added support for `Subject NOT LIKE` queries, - added support for multiple values in ticket subject queries. - `intelmq.bots.collectors.rsync`: Support for optional private key, relative time parsing for the source path, extra rsync parameters and strict host key checking (PR#2241 by Mateo Durante). +======= +- `intelmq.bots.collectors.shadowserver.collector_reports_api`: + - The 'json' option is no longer supported as the 'csv' option provides better performance. Please see intelmq/bots/parsers/shadowserver/README.md for a sample configuration. (PR#2372) +>>>>>>> Documentation and style updates. #### Parsers - `intelmq.bots.parsers.shadowserver._config`: - Reset detected `feedname` at shutdown to re-detect the feedname on reloads (PR#2361 by @elsif2, fixes #2360). + - Switch to dynamic configuration to decouple report schema changes from IntelMQ releases. Please see intelmq/bots/parsers/shadowserver/README.md for details. (PR#2372) - `intelmq.bots.parsers.shadowserver._config`: - Added 'IPv6-Vulnerable-Exchange' alias and 'Accessible-WS-Discovery-Service' report. (PR#2338) - Removed unused `p0f_genre` and `p0f_detail` from the 'DNS-Open-Resolvers' report. (PR#2338) diff --git a/intelmq/bots/collectors/shadowserver/collector_reports_api.py b/intelmq/bots/collectors/shadowserver/collector_reports_api.py index dc8bd6b420..5e7117bd23 100644 --- a/intelmq/bots/collectors/shadowserver/collector_reports_api.py +++ b/intelmq/bots/collectors/shadowserver/collector_reports_api.py @@ -34,7 +34,7 @@ class ShadowServerAPICollectorBot(CollectorBot, HttpMixin, CacheMixin): A list of strings or a comma-separated list of the mailing lists you want to process. types (list): A list of strings or a string of comma-separated values with the names of reporttypes you want to process. If you leave this empty, all the available reports will be downloaded and processed (i.e. 'scan', 'drones', 'intel', 'sandbox_connection', 'sinkhole_combined'). - file_format (str): File format to download ('csv'). The 'json' option is not longer supported. + file_format (str): File format to download ('csv'). The 'json' option is no longer supported. """ country = None diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md index c757020e94..ae38dcb8cc 100644 --- a/intelmq/bots/parsers/shadowserver/README.md +++ b/intelmq/bots/parsers/shadowserver/README.md @@ -7,10 +7,45 @@ This module is maintained by [The Shadowserver Foundation](https://www.shadowser Please contact intelmq@shadowserver.org with any issues or concerns. -The report configuration is now stored in a _schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/v1. +The report configuration is now stored in a _schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/v1/schema. -For environments that have internet connectivity the `update_schema.py` script can be called from a cron job to obtain the latest revision. +For environments that have internet connectivity the `update_schema.py` script should be called from a cron job to obtain the latest revision. +The parser will attempt to download a schema update on startup unless INTELMQ_SKIP_INTERNET is set. For air-gapped systems automation will be required to download and copy the _schema.json_ file into this directory. The parser will automatically reload the configuration when the file changes. + + +## Sample configuration: + +``` +shadowserver-collector: + description: Our bot responsible for getting reports from Shadowserver + enabled: true + group: Collector + module: intelmq.bots.collectors.shadowserver.collector_reports_api + name: Shadowserver_Collector + parameters: + destination_queues: + _default: [shadowserver-parser-queue] + file_format: csv + api_key: "$API_KEY_received_from_the_shadowserver_foundation" + secret: "$SECRET_received_from_the_shadowserver_foundation" + run_mode: continuous +``` + +``` +shadowserver-parser: + bot_id: shadowserver-parser + name: Shadowserver Parser + enabled: true + group: Parser + groupname: parsers + module: intelmq.bots.parsers.shadowserver.parser + parameters: + destination_queues: + _default: [file-output-queue] + run_mode: continuous +``` + diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index bb67db525a..5219fdb344 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -88,15 +88,18 @@ import intelmq.lib.harmonization as harmonization + class __Container: pass + __config = __Container() __config.schema_file = os.path.join(os.path.dirname(__file__), 'schema.json') __config.schema_mtime = 0.0 __config.feedname_mapping = {} __config.filename_mapping = {} + def set_logger(logger): """ Sets the logger instance. """ __config.logger = logger @@ -254,27 +257,28 @@ def scan_exchange_identifier(field): return 'exchange-server-webshell' return 'vulnerable-exchange-server' + functions = { - 'add_UTC_to_timestamp': add_UTC_to_timestamp, - 'convert_bool': convert_bool, - 'validate_to_none': validate_to_none, - 'convert_int': convert_int, - 'convert_float': convert_float, - 'convert_http_host_and_url': convert_http_host_and_url, - 'invalidate_zero': invalidate_zero, - 'validate_ip': validate_ip, - 'validate_network': validate_network, - 'validate_fqdn': validate_fqdn, - 'convert_date': convert_date, - 'convert_date_utc': convert_date_utc, - 'force_base64': force_base64, - 'scan_exchange_taxonomy': scan_exchange_taxonomy, - 'scan_exchange_type': scan_exchange_type, - 'scan_exchange_identifier': scan_exchange_identifier, - } - - -def reload (): + 'add_UTC_to_timestamp': add_UTC_to_timestamp, + 'convert_bool': convert_bool, + 'validate_to_none': validate_to_none, + 'convert_int': convert_int, + 'convert_float': convert_float, + 'convert_http_host_and_url': convert_http_host_and_url, + 'invalidate_zero': invalidate_zero, + 'validate_ip': validate_ip, + 'validate_network': validate_network, + 'validate_fqdn': validate_fqdn, + 'convert_date': convert_date, + 'convert_date_utc': convert_date_utc, + 'force_base64': force_base64, + 'scan_exchange_taxonomy': scan_exchange_taxonomy, + 'scan_exchange_type': scan_exchange_type, + 'scan_exchange_identifier': scan_exchange_identifier, +} + + +def reload(): """ reload the configuration if it has changed """ mtime = 0.0 @@ -291,7 +295,7 @@ def reload (): __config.feedname_mapping.clear() __config.filename_mapping.clear() - for schema_file in [ __config.schema_file, ".".join([__config.schema_file, 'test']) ]: + for schema_file in [__config.schema_file, ".".join([__config.schema_file, 'test'])]: if os.path.isfile(schema_file): with open(schema_file) as fh: schema = json.load(fh) @@ -305,13 +309,14 @@ def reload (): __config.filename_mapping[schema[report]['file_name']] = (schema[report]['feed_name'], schema[report]) __config.schema_mtime = mtime -def update_schema (): + +def update_schema(): """ download the latest configuration """ if os.environ.get('INTELMQ_SKIP_INTERNET'): return None (th, tmp) = tempfile.mkstemp(dir=os.path.dirname(__file__)) - url = 'https://interchange.shadowserver.org/intelmq/v1' + url = 'https://interchange.shadowserver.org/intelmq/v1/schema' try: urllib.request.urlretrieve(url, tmp) except: @@ -329,7 +334,6 @@ def update_schema (): raise ValueError("Failed to validate %r" % tmp) if os.path.exists(__config.schema_file): - old_version = '' try: with open(__config.schema_file) as fh: schema = json.load(fh) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index 71489e2ec1..668a815341 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -51,7 +51,7 @@ def init(self): try: config.update_schema() except Exception as e: - logger.warning(f"Schema update failed: {e}.") + self.logger.warning("Schema update failed: %s." % e) if self.feedname is not None: self._sparser_config = config.get_feed_by_feedname(self.feedname) if self._sparser_config: diff --git a/intelmq/tests/bots/parsers/shadowserver/test_broken.py b/intelmq/tests/bots/parsers/shadowserver/test_broken.py index 2b803142eb..3797f03cd5 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_broken.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_broken.py @@ -24,12 +24,12 @@ "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", "extra.file_name": "2019-01-01-some_string-test-test.csv", -} + } REPORT4 = {"raw": utils.base64_encode('adasdasdasdasd\nadasdasdafgf'), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", "extra.file_name": "2020.wrong-filename.csv", -} + } class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): diff --git a/intelmq/tests/bots/parsers/shadowserver/test_mapping.py b/intelmq/tests/bots/parsers/shadowserver/test_mapping.py index 6a2af94475..d296dfdc26 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_mapping.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_mapping.py @@ -52,6 +52,5 @@ def test_changed_feed(self): self.run_bot(iterations=2) - if __name__ == '__main__': # pragma: no cover unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py b/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py index c7eefdf0a9..93d592d15c 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py @@ -22,85 +22,78 @@ "time.observation": "2015-01-01T00:00:00+00:00", "extra.file_name": "2019-01-01-test_smb-test-geo.csv", } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'test-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Test-Accessible-SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 445, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'test-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Test-Accessible-SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 445, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'test-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Test-Accessible-SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 445, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} - ] +EVENTS = [{'__type': 'Event', + 'classification.identifier': 'test-smb', + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + 'extra.smb_implant': False, + 'extra.smb_major_number': '2', + 'extra.smb_minor_number': '1', + 'extra.smb_version_string': 'SMB 2.1', + 'extra.smbv1_support': 'N', + 'extra.tag': 'smb', + 'feed.name': 'Test-Accessible-SMB', + 'protocol.application': 'smb', + 'protocol.transport': 'tcp', + 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), + 'source.asn': 64512, + 'source.geolocation.cc': 'ZZ', + 'source.geolocation.city': 'City', + 'source.geolocation.region': 'Region', + 'source.ip': '192.168.0.1', + 'source.port': 445, + 'source.reverse_dns': 'node01.example.com', + 'time.observation': '2015-01-01T00:00:00+00:00', + 'time.source': '2010-02-10T00:00:00+00:00' + }, + {'__type': 'Event', + 'classification.identifier': 'test-smb', + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + 'extra.smb_implant': False, + 'extra.smb_major_number': '2', + 'extra.smb_minor_number': '1', + 'extra.smb_version_string': 'SMB 2.1', + 'extra.smbv1_support': 'N', + 'extra.tag': 'smb', + 'feed.name': 'Test-Accessible-SMB', + 'protocol.application': 'smb', + 'protocol.transport': 'tcp', + 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), + 'source.asn': 64512, + 'source.geolocation.cc': 'ZZ', + 'source.geolocation.city': 'City', + 'source.geolocation.region': 'Region', + 'source.ip': '192.168.0.2', + 'source.port': 445, + 'source.reverse_dns': 'node02.example.com', + 'time.observation': '2015-01-01T00:00:00+00:00', + 'time.source': '2010-02-10T00:00:01+00:00' + }, + {'__type': 'Event', + 'classification.identifier': 'test-smb', + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + 'extra.smb_implant': False, + 'extra.smb_major_number': '2', + 'extra.smb_minor_number': '1', + 'extra.smb_version_string': 'SMB 2.1', + 'extra.smbv1_support': 'N', + 'extra.tag': 'smb', + 'feed.name': 'Test-Accessible-SMB', + 'protocol.application': 'smb', + 'protocol.transport': 'tcp', + 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), + 'source.asn': 64512, + 'source.geolocation.cc': 'ZZ', + 'source.geolocation.city': 'City', + 'source.geolocation.region': 'Region', + 'source.ip': '192.168.0.3', + 'source.port': 445, + 'source.reverse_dns': 'node03.example.com', + 'time.observation': '2015-01-01T00:00:00+00:00', + 'time.source': '2010-02-10T00:00:02+00:00' + }] class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py b/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py index 570d612fb4..a9be8a0a13 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py @@ -16,11 +16,11 @@ EXAMPLE_LINES = handle.read().splitlines()[:2] FIRST_REPORT = {'feed.name': 'Test-Accessible-SMB', - "raw": utils.base64_encode('\n'.join(EXAMPLE_LINES)), - "__type": "Report", - "time.observation": "2019-03-25T00:00:00+00:00", - "extra.file_name": "2019-03-25-test_smb-test-test.csv", - } + "raw": utils.base64_encode('\n'.join(EXAMPLE_LINES)), + "__type": "Report", + "time.observation": "2019-03-25T00:00:00+00:00", + "extra.file_name": "2019-03-25-test_smb-test-test.csv", + } with open(os.path.join(os.path.dirname(__file__), 'testdata/test_telnet.csv')) as handle: EXAMPLE_LINES = handle.read().splitlines()[:2] diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py b/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py index 6d539ac4a7..df9cf25dca 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py @@ -42,7 +42,7 @@ "source.reverse_dns": "example.local", "time.observation": "2015-01-01T00:00:00+00:00", "time.source": "2019-09-04T12:27:34+00:00" - }, + }, {'__type': 'Event', 'feed.name': 'Test-Accessible-Telnet', "classification.identifier": "test-telnet", @@ -63,7 +63,7 @@ "source.reverse_dns": "example.local", "time.observation": "2015-01-01T00:00:00+00:00", "time.source": "2019-09-04T12:27:40+00:00" - }] + }] class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): From fc3f5b0aa1685109f62a7addc69699a92676e935 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 30 May 2023 16:05:26 +0000 Subject: [PATCH 09/76] Added schema.json.test.license. --- intelmq/bots/parsers/shadowserver/schema.json.test.license | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 intelmq/bots/parsers/shadowserver/schema.json.test.license diff --git a/intelmq/bots/parsers/shadowserver/schema.json.test.license b/intelmq/bots/parsers/shadowserver/schema.json.test.license new file mode 100644 index 0000000000..9f58c89ef0 --- /dev/null +++ b/intelmq/bots/parsers/shadowserver/schema.json.test.license @@ -0,0 +1,2 @@ +SPDX-FileCopyrightText: 2023 The Shadowserver Foundation +SPDX-License-Identifier: AGPL-3.0-or-later From b996e0e5ef93f9acd283609d0b5fd9f196d44438 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Thu, 27 Jul 2023 20:19:25 +0000 Subject: [PATCH 10/76] Updates in response to feedback. --- .../shadowserver/collector_reports_api.py | 9 +++- intelmq/bots/parsers/shadowserver/README.md | 21 ++++++-- intelmq/bots/parsers/shadowserver/_config.py | 53 +++++++++++++------ intelmq/bots/parsers/shadowserver/parser.py | 45 +++++++++++++--- .../parsers/shadowserver/update_schema.py | 11 ---- .../shadowserver/test_download_schema.py | 28 ++++++++++ 6 files changed, 130 insertions(+), 37 deletions(-) delete mode 100644 intelmq/bots/parsers/shadowserver/update_schema.py create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_download_schema.py diff --git a/intelmq/bots/collectors/shadowserver/collector_reports_api.py b/intelmq/bots/collectors/shadowserver/collector_reports_api.py index 5e7117bd23..05bffa898e 100644 --- a/intelmq/bots/collectors/shadowserver/collector_reports_api.py +++ b/intelmq/bots/collectors/shadowserver/collector_reports_api.py @@ -68,12 +68,19 @@ def init(self): if self.file_format is not None: if not (self.file_format == 'csv'): - raise ValueError('Invalid file_format') + raise ValueError("Invalid file_format '%s'. Must be 'csv'." % self.file_format) else: self.file_format = 'csv' self.preamble = f'{{ "apikey": "{self.api_key}" ' + def check(parameters: dict): + for key in parameters: + if key == 'file_format' and parameters[key] != 'csv': + return [["error", "Invalid file_format '%s'. Must be 'csv'." % parameters[key]]] + elif key == 'country': + return [["warning", "Deprecated parameter 'country' found. Please use 'reports' instead. The backwards-compatibility will be removed in IntelMQ version 4.0.0."]] + def _headers(self, data): return {'HMAC2': hmac.new(self.secret.encode(), data.encode('utf-8'), digestmod=hashlib.sha256).hexdigest()} diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md index ae38dcb8cc..cd750d00b3 100644 --- a/intelmq/bots/parsers/shadowserver/README.md +++ b/intelmq/bots/parsers/shadowserver/README.md @@ -7,16 +7,28 @@ This module is maintained by [The Shadowserver Foundation](https://www.shadowser Please contact intelmq@shadowserver.org with any issues or concerns. -The report configuration is now stored in a _schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/v1/schema. +The report configuration is now stored in a _shadowserver-schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/v1/schema. -For environments that have internet connectivity the `update_schema.py` script should be called from a cron job to obtain the latest revision. -The parser will attempt to download a schema update on startup unless INTELMQ_SKIP_INTERNET is set. +The parser will attempt to download a schema update on startup when the *auto_update* option is enabled. -For air-gapped systems automation will be required to download and copy the _schema.json_ file into this directory. +Schema downloads can also be scheduled as a cron job: + +``` +02 01 * * * intelmq.bots.parsers.shadowserver.parser --update-schema +``` + +For air-gapped systems automation will be required to download and copy the file to VAR_STATE_PATH/shadowserver-schema.json. The parser will automatically reload the configuration when the file changes. +## Schema contract + +Once set the `classification.identifier`, `classification.taxonomy`, and `classification.type` fields will remain static. + +Once set report fields will not be deleted. + + ## Sample configuration: ``` @@ -46,6 +58,7 @@ shadowserver-parser: parameters: destination_queues: _default: [file-output-queue] + auto_update: true run_mode: continuous ``` diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index 5219fdb344..afe3a6b11f 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -82,11 +82,12 @@ import base64 import binascii import json -import urllib.request import tempfile from typing import Optional, Dict, Tuple, Any import intelmq.lib.harmonization as harmonization +from intelmq.lib.utils import create_request_session +from intelmq import VAR_STATE_PATH class __Container: @@ -94,8 +95,10 @@ class __Container: __config = __Container() -__config.schema_file = os.path.join(os.path.dirname(__file__), 'schema.json') +__config.schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json') +__config.schema_base = os.path.join(os.path.dirname(__file__), 'schema.json.test') __config.schema_mtime = 0.0 +__config.auto_update = False __config.feedname_mapping = {} __config.filename_mapping = {} @@ -105,13 +108,16 @@ def set_logger(logger): __config.logger = logger +def enable_auto_update(enable): + """ Enable automatic schema update. """ + __config.auto_update = enable + + def get_feed_by_feedname(given_feedname: str) -> Optional[Dict[str, Any]]: - reload() return __config.feedname_mapping.get(given_feedname, None) def get_feed_by_filename(given_filename: str) -> Optional[Tuple[str, Dict[str, Any]]]: - reload() return __config.filename_mapping.get(given_filename, None) @@ -289,19 +295,18 @@ def reload(): else: __config.logger.info("The schema file does not exist.") - if __config.schema_mtime == 0.0 and mtime == 0.0 and not os.environ.get('INTELMQ_SKIP_INTERNET'): - __config.logger.info("Attempting to download schema.") + if __config.schema_mtime == 0.0 and mtime == 0.0 and __config.auto_update: update_schema() __config.feedname_mapping.clear() __config.filename_mapping.clear() - for schema_file in [__config.schema_file, ".".join([__config.schema_file, 'test'])]: + for schema_file in [__config.schema_file, __config.schema_base]: if os.path.isfile(schema_file): with open(schema_file) as fh: schema = json.load(fh) for report in schema: if report == "_meta": - __config.logger.info("Loading schema %s." % schema[report]['date_created']) + __config.logger.info("Loading schema %r." % schema[report]['date_created']) for msg in schema[report]['change_log']: __config.logger.info(msg) else: @@ -313,37 +318,55 @@ def reload(): def update_schema(): """ download the latest configuration """ if os.environ.get('INTELMQ_SKIP_INTERNET'): - return None + return False - (th, tmp) = tempfile.mkstemp(dir=os.path.dirname(__file__)) + # download the schema to a temp file + (th, tmp) = tempfile.mkstemp(dir=VAR_STATE_PATH) url = 'https://interchange.shadowserver.org/intelmq/v1/schema' + __config.logger.info("Attempting to download schema from %r" % url) + __config.logger.debug("Using temp file %r for the download." % tmp) try: - urllib.request.urlretrieve(url, tmp) + with create_request_session() as session: + with session.get(url, stream=True) as r: + r.raise_for_status() + with open(tmp, 'wb') as f: + for chunk in r.iter_content(chunk_size=8192): + f.write(chunk) except: - raise ValueError("Failed to download %r" % url) + __config.logger.error("Failed to download %r" % url) + return False + __config.logger.info("Download successful.") new_version = '' old_version = '' try: + # validate the downloaded file with open(tmp) as fh: schema = json.load(fh) new_version = schema['_meta']['date_created'] except: # leave tempfile behind for diagnosis - raise ValueError("Failed to validate %r" % tmp) + __config.logger.error("Failed to validate %r" % tmp) + return False if os.path.exists(__config.schema_file): + # compare the new version against the old; rename the existing file try: with open(__config.schema_file) as fh: schema = json.load(fh) old_version = schema['_meta']['date_created'] if new_version != old_version: os.replace(__config.schema_file, ".".join([__config.schema_file, 'bak'])) - except: - pass + except Exception as e: + __config.logger.error("Unable to replace schema file: %s" % str(e)) + return False if new_version != old_version: os.replace(tmp, __config.schema_file) + __config.logger.info("New schema version is %r." % new_version) + return True else: os.unlink(tmp) + + return False diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index 668a815341..2e383a004e 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -26,6 +26,8 @@ from intelmq.lib.bot import ParserBot from intelmq.lib.exceptions import InvalidKey, InvalidValue +from intelmq.bin.intelmqctl import IntelMQController +import intelmq.lib.utils as utils import intelmq.bots.parsers.shadowserver._config as config @@ -34,8 +36,7 @@ class ShadowserverParserBot(ParserBot): Parse all ShadowServer feeds Parameters: - schema_file (str): Path to the report schema file - + auto_update (boolean): Enable automatic schema download """ recover_line = ParserBot.recover_line_csv_dict @@ -45,13 +46,15 @@ class ShadowserverParserBot(ParserBot): feedname = None _mode = None overwrite = False + auto_update = False def init(self): config.set_logger(self.logger) - try: - config.update_schema() - except Exception as e: - self.logger.warning("Schema update failed: %s." % e) + if self.auto_update: + config.enable_auto_update(True) + self.logger.debug("Feature 'auto_update' is enabled.") + config.reload() + if self.feedname is not None: self._sparser_config = config.get_feed_by_feedname(self.feedname) if self._sparser_config: @@ -228,5 +231,35 @@ def parse_line(self, row, report): def shutdown(self): self.feedname = None + @classmethod + def _create_argparser(cls): + argparser = super()._create_argparser() + argparser.add_argument("--update-schema", action='store_true', help='downloads latest report schema') + argparser.add_argument("--verbose", action='store_true', help='be verbose') + return argparser + + @classmethod + def run(cls, parsed_args=None): + if not parsed_args: + parsed_args = cls._create_argparser().parse_args() + if parsed_args.update_schema: + logger = utils.log(__name__, log_path=None) + if parsed_args.verbose: + logger.setLevel('INFO') + else: + logger.setLevel('ERROR') + config.set_logger(logger) + if config.update_schema(): + runtime_conf = utils.get_bots_settings() + try: + ctl = IntelMQController() + for bot in runtime_conf: + if runtime_conf[bot]["module"] == __name__ and runtime_conf[bot]['parameters'].get('auto_update', True): + ctl.bot_reload(bot) + except Exception as e: + logger.error("Failed to signal bot: %r" % str(e)) + else: + super().run(parsed_args=parsed_args) + BOT = ShadowserverParserBot diff --git a/intelmq/bots/parsers/shadowserver/update_schema.py b/intelmq/bots/parsers/shadowserver/update_schema.py deleted file mode 100644 index a7975147ed..0000000000 --- a/intelmq/bots/parsers/shadowserver/update_schema.py +++ /dev/null @@ -1,11 +0,0 @@ -# SPDX-FileCopyrightText: 2023 The Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import intelmq.bots.parsers.shadowserver._config as config - -if __name__ == '__main__': # pragma: no cover - config.update_schema() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py new file mode 100644 index 0000000000..e685876826 --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: 2023 The Shadowserver Foundation +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +# -*- coding: utf-8 -*- +""" +Created on Thu Jul 27 19:44:44 2023 + +""" + +import unittest +import os +import logging +from intelmq import VAR_STATE_PATH +import intelmq.bots.parsers.shadowserver._config as config +import intelmq.lib.utils as utils +import intelmq.lib.test as test + +@test.skip_internet() +class TestShadowserverSchemaDownload(unittest.TestCase): + + def test_download(self): + schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json') + config.set_logger(utils.log('test-bot', log_path=None)) + if os.path.exists(schema_file): + os.unlink(schema_file) + self.assertEqual(True, config.update_schema()) + self.assertEqual(True, os.path.exists(schema_file)) From 661a96471a991cc3dac8faf4a553874f1164d4d6 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Fri, 28 Jul 2023 14:17:41 +0000 Subject: [PATCH 11/76] Removed file_format parameter --- .../shadowserver/collector_reports_api.py | 20 ++++--------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/intelmq/bots/collectors/shadowserver/collector_reports_api.py b/intelmq/bots/collectors/shadowserver/collector_reports_api.py index 05bffa898e..66169d96f1 100644 --- a/intelmq/bots/collectors/shadowserver/collector_reports_api.py +++ b/intelmq/bots/collectors/shadowserver/collector_reports_api.py @@ -34,7 +34,6 @@ class ShadowServerAPICollectorBot(CollectorBot, HttpMixin, CacheMixin): A list of strings or a comma-separated list of the mailing lists you want to process. types (list): A list of strings or a string of comma-separated values with the names of reporttypes you want to process. If you leave this empty, all the available reports will be downloaded and processed (i.e. 'scan', 'drones', 'intel', 'sandbox_connection', 'sinkhole_combined'). - file_format (str): File format to download ('csv'). The 'json' option is no longer supported. """ country = None @@ -42,7 +41,6 @@ class ShadowServerAPICollectorBot(CollectorBot, HttpMixin, CacheMixin): secret = None types = None reports = None - file_format = None rate_limit: int = 86400 redis_cache_db: int = 12 redis_cache_host: str = "127.0.0.1" # TODO: type could be ipadress @@ -66,18 +64,12 @@ def init(self): self.logger.warn("Deprecated parameter 'country' found. Please use 'reports' instead. The backwards-compatibility will be removed in IntelMQ version 4.0.0.") self._report_list.append(self.country) - if self.file_format is not None: - if not (self.file_format == 'csv'): - raise ValueError("Invalid file_format '%s'. Must be 'csv'." % self.file_format) - else: - self.file_format = 'csv' - self.preamble = f'{{ "apikey": "{self.api_key}" ' def check(parameters: dict): for key in parameters: - if key == 'file_format' and parameters[key] != 'csv': - return [["error", "Invalid file_format '%s'. Must be 'csv'." % parameters[key]]] + if key == 'file_format': + return [["error", "The file_format parameter is no longer supported. All reports are CSV."]] elif key == 'country': return [["warning", "Deprecated parameter 'country' found. Please use 'reports' instead. The backwards-compatibility will be removed in IntelMQ version 4.0.0."]] @@ -129,11 +121,7 @@ def _report_download(self, reportid: str): data = self.preamble data += f',"id": "{reportid}"}}' self.logger.debug('Downloading report with data: %s.', data) - - if (self.file_format == 'json'): - response = self.http_session().post(APIROOT + 'reports/download', data=data, headers=self._headers(data)) - else: - response = self.http_session().get(DLROOT + reportid) + response = self.http_session().get(DLROOT + reportid) response.raise_for_status() return response.text @@ -150,7 +138,7 @@ def process(self): for item in reportslist: filename = item['file'] - filename_fixed = FILENAME_PATTERN.sub('.' + self.file_format, filename, count=1) + filename_fixed = FILENAME_PATTERN.sub('.csv', filename, count=1) if self.cache_get(filename): self.logger.debug('Processed file %r (fixed: %r) already.', filename, filename_fixed) continue From a045bee263ee0c2b447b8191503d88e3829e5a9f Mon Sep 17 00:00:00 2001 From: elsif2 Date: Thu, 24 Aug 2023 16:04:21 +0000 Subject: [PATCH 12/76] Minor changes based on feedback 2023-08-24 --- CHANGELOG.md | 2 - intelmq/bots/parsers/shadowserver/README.md | 2 + intelmq/bots/parsers/shadowserver/_config.py | 49 ++++++++++--------- intelmq/bots/parsers/shadowserver/parser.py | 6 ++- .../bots/parsers/shadowserver/test_broken.py | 5 ++ .../bots/parsers/shadowserver/test_mapping.py | 1 + .../parsers/shadowserver/test_parameters.py | 3 +- .../parsers/shadowserver/test_report_smb.py | 1 + .../shadowserver/test_report_switch.py | 1 + .../shadowserver/test_report_telnet.py | 1 + 10 files changed, 45 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b54eb84d1..b7daa0be04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -68,10 +68,8 @@ CHANGELOG - added support for `Subject NOT LIKE` queries, - added support for multiple values in ticket subject queries. - `intelmq.bots.collectors.rsync`: Support for optional private key, relative time parsing for the source path, extra rsync parameters and strict host key checking (PR#2241 by Mateo Durante). -======= - `intelmq.bots.collectors.shadowserver.collector_reports_api`: - The 'json' option is no longer supported as the 'csv' option provides better performance. Please see intelmq/bots/parsers/shadowserver/README.md for a sample configuration. (PR#2372) ->>>>>>> Documentation and style updates. #### Parsers - `intelmq.bots.parsers.shadowserver._config`: diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md index cd750d00b3..4969acb6d0 100644 --- a/intelmq/bots/parsers/shadowserver/README.md +++ b/intelmq/bots/parsers/shadowserver/README.md @@ -28,6 +28,8 @@ Once set the `classification.identifier`, `classification.taxonomy`, and `classi Once set report fields will not be deleted. +The schema revision history is maintained at https://github.com/The-Shadowserver-Foundation/report_schema/. + ## Sample configuration: diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index afe3a6b11f..4bfadb9d98 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -95,8 +95,10 @@ class __Container: __config = __Container() +__config.schema_url = 'https://interchange.shadowserver.org/intelmq/v1/schema' __config.schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json') __config.schema_base = os.path.join(os.path.dirname(__file__), 'schema.json.test') +__config.schema_active = __config.schema_file __config.schema_mtime = 0.0 __config.auto_update = False __config.feedname_mapping = {} @@ -108,6 +110,13 @@ def set_logger(logger): __config.logger = logger +def enable_test_mode(enable): + """ Set which schema to load. """ + if enable: + __config.schema_active = __config.schema_base + else: + __config.schema_active = __config.schema_file + def enable_auto_update(enable): """ Enable automatic schema update. """ __config.auto_update = enable @@ -300,40 +309,36 @@ def reload(): __config.feedname_mapping.clear() __config.filename_mapping.clear() - for schema_file in [__config.schema_file, __config.schema_base]: - if os.path.isfile(schema_file): - with open(schema_file) as fh: - schema = json.load(fh) - for report in schema: - if report == "_meta": - __config.logger.info("Loading schema %r." % schema[report]['date_created']) - for msg in schema[report]['change_log']: - __config.logger.info(msg) - else: - __config.feedname_mapping[schema[report]['feed_name']] = (schema[report]['feed_name'], schema[report]) - __config.filename_mapping[schema[report]['file_name']] = (schema[report]['feed_name'], schema[report]) + if os.path.isfile(__config.schema_active): + with open(__config.schema_active) as fh: + schema = json.load(fh) + for report in schema: + if report == "_meta": + __config.logger.info("Loading schema %r.", schema[report]['date_created']) + for msg in schema[report]['change_log']: + __config.logger.info(msg) + else: + __config.feedname_mapping[schema[report]['feed_name']] = (schema[report]['feed_name'], schema[report]) + __config.filename_mapping[schema[report]['file_name']] = (schema[report]['feed_name'], schema[report]) __config.schema_mtime = mtime def update_schema(): """ download the latest configuration """ - if os.environ.get('INTELMQ_SKIP_INTERNET'): - return False # download the schema to a temp file (th, tmp) = tempfile.mkstemp(dir=VAR_STATE_PATH) - url = 'https://interchange.shadowserver.org/intelmq/v1/schema' - __config.logger.info("Attempting to download schema from %r" % url) - __config.logger.debug("Using temp file %r for the download." % tmp) + __config.logger.info("Attempting to download schema from %r", __config.schema_url) + __config.logger.debug("Using temp file %r for the download.", tmp) try: with create_request_session() as session: - with session.get(url, stream=True) as r: + with session.get(__config.schema_url, stream=True) as r: r.raise_for_status() with open(tmp, 'wb') as f: for chunk in r.iter_content(chunk_size=8192): f.write(chunk) except: - __config.logger.error("Failed to download %r" % url) + __config.logger.error("Failed to download %r", __config.schema_url) return False __config.logger.info("Download successful.") @@ -347,7 +352,7 @@ def update_schema(): new_version = schema['_meta']['date_created'] except: # leave tempfile behind for diagnosis - __config.logger.error("Failed to validate %r" % tmp) + __config.logger.error("Failed to validate %r", tmp) return False if os.path.exists(__config.schema_file): @@ -359,12 +364,12 @@ def update_schema(): if new_version != old_version: os.replace(__config.schema_file, ".".join([__config.schema_file, 'bak'])) except Exception as e: - __config.logger.error("Unable to replace schema file: %s" % str(e)) + __config.logger.error("Unable to replace schema file: %s", str(e)) return False if new_version != old_version: os.replace(tmp, __config.schema_file) - __config.logger.info("New schema version is %r." % new_version) + __config.logger.info("New schema version is %r.", new_version) return True else: os.unlink(tmp) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index 2e383a004e..fd9fa6b2cf 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -37,6 +37,7 @@ class ShadowserverParserBot(ParserBot): Parameters: auto_update (boolean): Enable automatic schema download + test_mode (boolean): Use test schema """ recover_line = ParserBot.recover_line_csv_dict @@ -47,9 +48,12 @@ class ShadowserverParserBot(ParserBot): _mode = None overwrite = False auto_update = False + test_mode = False def init(self): config.set_logger(self.logger) + if self.test_mode: + config.enable_test_mode(True) if self.auto_update: config.enable_auto_update(True) self.logger.debug("Feature 'auto_update' is enabled.") @@ -254,7 +258,7 @@ def run(cls, parsed_args=None): try: ctl = IntelMQController() for bot in runtime_conf: - if runtime_conf[bot]["module"] == __name__ and runtime_conf[bot]['parameters'].get('auto_update', True): + if runtime_conf[bot]["module"] == __name__: ctl.bot_reload(bot) except Exception as e: logger.error("Failed to signal bot: %r" % str(e)) diff --git a/intelmq/tests/bots/parsers/shadowserver/test_broken.py b/intelmq/tests/bots/parsers/shadowserver/test_broken.py index 3797f03cd5..54a85e7802 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_broken.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_broken.py @@ -46,6 +46,7 @@ def test_broken(self): """ Test a report which does not have valid fields """ + self.prepare_bot(parameters={'test_mode': True}) self.input_message = REPORT1 self.run_bot(allowed_error_count=1) self.assertLogMatches(pattern="Detected report's file name: 'test_smb'.", @@ -59,6 +60,7 @@ def test_half_broken(self): """ Test a report which does not have an optional field. """ + self.prepare_bot(parameters={'test_mode': True}) self.input_message = REPORT2 self.run_bot(allowed_warning_count=63) self.assertLogMatches(pattern="Detected report's file name: 'test_telnet'.", @@ -72,6 +74,7 @@ def test_no_config(self): """ Test a report which does not have a valid extra.file_name """ + self.prepare_bot(parameters={'test_mode': True}) self.input_message = REPORT3 self.run_bot(allowed_error_count=1) self.assertLogMatches(pattern="ValueError: Could not get a config for 'some_string', check the documentation.") @@ -80,6 +83,7 @@ def test_invalid_filename(self): """ Test a report which does not have a valid extra.file_name """ + self.prepare_bot(parameters={'test_mode': True}) self.input_message = REPORT4 self.run_bot(allowed_error_count=1) self.assertLogMatches(pattern="ValueError: Report's 'extra.file_name' '2020.wrong-filename.csv' is not valid.") @@ -89,6 +93,7 @@ def test_no_report_name(self): Test a report without file_name and no given feedname as parameter. Error message should be verbose. """ + self.prepare_bot(parameters={'test_mode': True}) self.run_bot(allowed_error_count=1) self.assertLogMatches(pattern="ValueError: No feedname given as parameter and the " "processed report has no 'extra.file_name'. " diff --git a/intelmq/tests/bots/parsers/shadowserver/test_mapping.py b/intelmq/tests/bots/parsers/shadowserver/test_mapping.py index d296dfdc26..b764de8274 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_mapping.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_mapping.py @@ -48,6 +48,7 @@ def test_changed_feed(self): Tests if the parser correctly re-detects the feed for the second received report #1493 """ + self.prepare_bot(parameters={'test_mode': True}) self.input_message = (EXAMPLE_TELNET, EXAMPLE_VNC) self.run_bot(iterations=2) diff --git a/intelmq/tests/bots/parsers/shadowserver/test_parameters.py b/intelmq/tests/bots/parsers/shadowserver/test_parameters.py index 677cd0319b..45a4a87354 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_parameters.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_parameters.py @@ -63,13 +63,14 @@ def set_bot(cls): def test_default(self): """ Test if feed name is not overwritten has been produced. """ + self.prepare_bot(parameters={'test_mode': True}) self.run_bot() for i, EVENT in enumerate(EVENTS): self.assertMessageEqual(i, EVENT) def test_overwrite_feed_name(self): """ Test if feed name is overwritten if asked to do so. """ - self.prepare_bot(parameters={'overwrite': True}) + self.prepare_bot(parameters={'test_mode': True, 'overwrite': True}) self.run_bot(prepare=False) for i, EVENT in enumerate(EVENTS): event = EVENT.copy() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py b/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py index 93d592d15c..aa6940061b 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py @@ -108,6 +108,7 @@ def set_bot(cls): def test_event(self): """ Test if correct Event has been produced. """ + self.prepare_bot(parameters={'test_mode': True}) self.run_bot() for i, EVENT in enumerate(EVENTS): self.assertMessageEqual(i, EVENT) diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py b/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py index a9be8a0a13..488f5a51a1 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py @@ -46,6 +46,7 @@ def set_bot(cls): def test_event(self): """ Test if the parser correctly detects and handles different report types. """ + self.prepare_bot(parameters={'test_mode': True}) self.input_message = [FIRST_REPORT, SECOND_REPORT] self.run_bot(iterations=2) self.assertLogMatches("Detected report's file name: 'test_smb'", diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py b/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py index df9cf25dca..b2499c589d 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py @@ -78,6 +78,7 @@ def set_bot(cls): def test_event(self): """ Test if correct Event has been produced. """ + self.prepare_bot(parameters={'test_mode': True}) self.run_bot() for i, EVENT in enumerate(EVENTS): self.assertMessageEqual(i, EVENT) From 0660a893e4a520973b8dfd6e27b93240574bb007 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Thu, 24 Aug 2023 16:26:59 +0000 Subject: [PATCH 13/76] Added VAR_STATE_PATH check. --- intelmq/bots/parsers/shadowserver/_config.py | 1 + .../parsers/shadowserver/test_download_schema.py | 13 +++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index 4bfadb9d98..6ffffdae86 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -117,6 +117,7 @@ def enable_test_mode(enable): else: __config.schema_active = __config.schema_file + def enable_auto_update(enable): """ Enable automatic schema update. """ __config.auto_update = enable diff --git a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py index e685876826..f9512ca98c 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py @@ -20,9 +20,10 @@ class TestShadowserverSchemaDownload(unittest.TestCase): def test_download(self): - schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json') - config.set_logger(utils.log('test-bot', log_path=None)) - if os.path.exists(schema_file): - os.unlink(schema_file) - self.assertEqual(True, config.update_schema()) - self.assertEqual(True, os.path.exists(schema_file)) + if os.path.isdir(VAR_STATE_PATH): + schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json') + config.set_logger(utils.log('test-bot', log_path=None)) + if os.path.exists(schema_file): + os.unlink(schema_file) + self.assertEqual(True, config.update_schema()) + self.assertEqual(True, os.path.exists(schema_file)) From 33370cf1eeb4fefef24ca8f7c2ea34dc02c97b42 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Fri, 25 Aug 2023 15:37:51 +0000 Subject: [PATCH 14/76] Changes based on feedback 2023-08-25. --- CHANGELOG.md | 6 +- docs/user/bots.rst | 171 ++++++------------ intelmq/bots/parsers/shadowserver/README.md | 57 ------ intelmq/bots/parsers/shadowserver/_config.py | 10 +- .../shadowserver/test_download_schema.py | 8 +- 5 files changed, 72 insertions(+), 180 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b7daa0be04..0e9ede890b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,20 +62,18 @@ CHANGELOG ### Bots #### Collectors -<<<<<<< HEAD - `intelmq.bots.collector.rt`: - restrict `python-rt` to be below version 3.0 due to introduced breaking changes, - added support for `Subject NOT LIKE` queries, - added support for multiple values in ticket subject queries. - `intelmq.bots.collectors.rsync`: Support for optional private key, relative time parsing for the source path, extra rsync parameters and strict host key checking (PR#2241 by Mateo Durante). - `intelmq.bots.collectors.shadowserver.collector_reports_api`: - - The 'json' option is no longer supported as the 'csv' option provides better performance. Please see intelmq/bots/parsers/shadowserver/README.md for a sample configuration. (PR#2372) + - The 'json' option is no longer supported as the 'csv' option provides better performance. #### Parsers - `intelmq.bots.parsers.shadowserver._config`: - Reset detected `feedname` at shutdown to re-detect the feedname on reloads (PR#2361 by @elsif2, fixes #2360). - - Switch to dynamic configuration to decouple report schema changes from IntelMQ releases. Please see intelmq/bots/parsers/shadowserver/README.md for details. (PR#2372) -- `intelmq.bots.parsers.shadowserver._config`: + - Switch to dynamic configuration to decouple report schema changes from IntelMQ releases. - Added 'IPv6-Vulnerable-Exchange' alias and 'Accessible-WS-Discovery-Service' report. (PR#2338) - Removed unused `p0f_genre` and `p0f_detail` from the 'DNS-Open-Resolvers' report. (PR#2338) - Added 'Accessible-SIP' report. (PR#2348) diff --git a/docs/user/bots.rst b/docs/user/bots.rst index 8e8f36396b..3da99af1e1 100644 --- a/docs/user/bots.rst +++ b/docs/user/bots.rst @@ -673,6 +673,23 @@ The resulting reports contain the following special field: * `extra.file_name`: The name of the downloaded file, with fixed filename extension. The API returns file names with the extension `.csv`, although the files are JSON, not CSV. Therefore, for clarity and better error detection in the parser, the file name in `extra.file_name` uses `.json` as extension. +**Sample configuration** + +.. code-block:: yaml + + shadowserver-collector: + description: Our bot responsible for getting reports from Shadowserver + enabled: true + group: Collector + module: intelmq.bots.collectors.shadowserver.collector_reports_api + name: Shadowserver_Collector + parameters: + destination_queues: + _default: [shadowserver-parser-queue] + file_format: csv + api_key: "$API_KEY_received_from_the_shadowserver_foundation" + secret: "$SECRET_received_from_the_shadowserver_foundation" + run_mode: continuous .. _intelmq.bots.collectors.shodan.collector_stream: @@ -1554,17 +1571,15 @@ This does not affect URLs which already include the scheme. .. _intelmq.bots.parsers.shadowserver.parser: -.. _intelmq.bots.parsers.shadowserver.parser_json: Shadowserver ^^^^^^^^^^^^ -There are two Shadowserver parsers, one for data in ``CSV`` format (``intelmq.bots.parsers.shadowserver.parser``) and one for data in ``JSON`` format (``intelmq.bots.parsers.shadowserver.parser_json``). -The latter was added in IntelMQ 2.3 and is meant to be used together with the Shadowserver API collector. +The Shadowserver parser operates on ``CSV`` formatted data. **Information** -* `name:` `intelmq.bots.parsers.shadowserver.parser` (for CSV data) or `intelmq.bots.parsers.shadowserver.parser_json` (for JSON data) +* `name:` `intelmq.bots.parsers.shadowserver.parser` * `public:` yes * `description:` Parses different reports from Shadowserver. @@ -1600,107 +1615,45 @@ A list of possible feeds can be found in the table below in the column "feed nam **Supported reports** -These are the supported feed name and their corresponding file name for automatic detection: - - ======================================= ========================= - feed name file name - ======================================= ========================= - Accessible-ADB `scan_adb` - Accessible-AFP `scan_afp` - Accessible-AMQP `scan_amqp` - Accessible-ARD `scan_ard` - Accessible-Cisco-Smart-Install `cisco_smart_install` - Accessible-CoAP `scan_coap` - Accessible-CWMP `scan_cwmp` - Accessible-MS-RDPEUDP `scan_msrdpeudp` - Accessible-FTP `scan_ftp` - Accessible-Hadoop `scan_hadoop` - Accessible-HTTP `scan_http` - Accessible-Radmin `scan_radmin` - Accessible-RDP `scan_rdp` - Accessible-Rsync `scan_rsync` - Accessible-SMB `scan_smb` - Accessible-Telnet `scan_telnet` - Accessible-Ubiquiti-Discovery-Service `scan_ubiquiti` - Accessible-VNC `scan_vnc` - Blacklisted-IP (deprecated) `blacklist` - Blocklist `blocklist` - Compromised-Website `compromised_website` - Device-Identification IPv4 / IPv6 `device_id`/`device_id6` - DNS-Open-Resolvers `scan_dns` - Honeypot-Amplification-DDoS-Events `event4_honeypot_ddos_amp` - Honeypot-Brute-Force-Events `event4_honeypot_brute_force` - Honeypot-Darknet `event4_honeypot_darknet` - Honeypot-HTTP-Scan `event4_honeypot_http_scan` - HTTP-Scanners `hp_http_scan` - ICS-Scanners `hp_ics_scan` - IP-Spoofer-Events `event4_ip_spoofer` - Microsoft-Sinkhole-Events IPv4 `event4_microsoft_sinkhole` - Microsoft-Sinkhole-Events-HTTP IPv4 `event4_microsoft_sinkhole_http` - NTP-Monitor `scan_ntpmonitor` - NTP-Version `scan_ntp` - Open-Chargen `scan_chargen` - Open-DB2-Discovery-Service `scan_db2` - Open-Elasticsearch `scan_elasticsearch` - Open-IPMI `scan_ipmi` - Open-IPP `scan_ipp` - Open-LDAP `scan_ldap` - Open-LDAP-TCP `scan_ldap_tcp` - Open-mDNS `scan_mdns` - Open-Memcached `scan_memcached` - Open-MongoDB `scan_mongodb` - Open-MQTT `scan_mqtt` - Open-MSSQL `scan_mssql` - Open-NATPMP `scan_nat_pmp` - Open-NetBIOS-Nameservice `scan_netbios` - Open-Netis `netis_router` - Open-Portmapper `scan_portmapper` - Open-QOTD `scan_qotd` - Open-Redis `scan_redis` - Open-SNMP `scan_snmp` - Open-SSDP `scan_ssdp` - Open-TFTP `scan_tftp` - Open-XDMCP `scan_xdmcp` - Outdated-DNSSEC-Key `outdated_dnssec_key` - Outdated-DNSSEC-Key-IPv6 `outdated_dnssec_key_v6` - Sandbox-URL `cwsandbox_url` - Sinkhole-DNS `sinkhole_dns` - Sinkhole-Events `event4_sinkhole`/`event6_sinkhole` - Sinkhole-Events IPv4 `event4_sinkhole` - Sinkhole-Events IPv6 `event6_sinkhole` - Sinkhole-HTTP-Events `event4_sinkhole_http`/`event6_sinkhole_http` - Sinkhole-HTTP-Events IPv4 `event4_sinkhole_http` - Sinkhole-HTTP-Events IPv6 `event6_sinkhole_http` - Sinkhole-Events-HTTP-Referer `event4_sinkhole_http_referer`/`event6_sinkhole_http_referer` - Sinkhole-Events-HTTP-Referer IPv4 `event4_sinkhole_http_referer` - Sinkhole-Events-HTTP-Referer IPv6 `event6_sinkhole_http_referer` - Spam-URL `spam_url` - SSL-FREAK-Vulnerable-Servers `scan_ssl_freak` - SSL-POODLE-Vulnerable-Servers `scan_ssl_poodle`/`scan6_ssl_poodle` - Vulnerable-Exchange-Server `*` `scan_exchange` - Vulnerable-ISAKMP `scan_isakmp` - Vulnerable-HTTP `scan_http` - Vulnerable-SMTP `scan_smtp_vulnerable` - ======================================= ========================= - -`*` This report can also contain data on active webshells (column `tag` is `exchange;webshell`), and are therefore not only vulnerable but also actively infected. - -In addition, the following legacy reports are supported: - - =========================== =================================================== ======================== - feed name successor feed name file name - =========================== =================================================== ======================== - Amplification-DDoS-Victim Honeypot-Amplification-DDoS-Events ``ddos_amplification`` - CAIDA-IP-Spoofer IP-Spoofer-Events ``caida_ip_spoofer`` - Darknet Honeypot-Darknet ``darknet`` - Drone Sinkhole-Events ``botnet_drone`` - Drone-Brute-Force Honeypot-Brute-Force-Events, Sinkhole-HTTP-Events ``drone_brute_force`` - Microsoft-Sinkhole Sinkhole-HTTP-Events ``microsoft_sinkhole`` - Sinkhole-HTTP-Drone Sinkhole-HTTP-Events ``sinkhole_http_drone`` - IPv6-Sinkhole-HTTP-Drone Sinkhole-HTTP-Events ``sinkhole6_http`` - =========================== =================================================== ======================== - -More information on these legacy reports can be found in `Changes in Sinkhole and Honeypot Report Types and Formats `_. +The report configuration is stored in a `shadowserver-schema.json` file downloaded from https://interchange.shadowserver.org/intelmq/v1/schema. + +The parser will attempt to download a schema update on startup when the *auto_update* option is enabled. + +Schema downloads can also be scheduled as a cron job: + +.. code-block:: bash + + 02 01 * * * intelmq.bots.parsers.shadowserver.parser --update-schema + + +For air-gapped systems automation will be required to download and copy the file to VAR_STATE_PATH/shadowserver-schema.json. + +The parser will automatically reload the configuration when the file changes. + +**Schema contract** + +Once set in the schema, the `classification.identifier`, `classification.taxonomy`, and `classification.type` fields will remain static for a specific report. + +Report fields will not be removed from a report. + +The schema revision history is maintained at https://github.com/The-Shadowserver-Foundation/report_schema/. + +**Sample configuration** + +.. code-block:: yaml + + shadowserver-parser: + bot_id: shadowserver-parser + name: Shadowserver Parser + enabled: true + group: Parser + groupname: parsers + module: intelmq.bots.parsers.shadowserver.parser + parameters: + destination_queues: + _default: [file-output-queue] + auto_update: true + run_mode: continuous **Development** @@ -1712,14 +1665,6 @@ The parser consists of two files: Both files are required for the parser to work properly. -**Add new Feedformats** - -Add a new feed format and conversions if required to the file -``_config.py``. Don't forget to update the ``mapping`` dict. -It is required to look up the correct configuration. - -Look at the documentation in the bot's ``_config.py`` file for more information. - .. _intelmq.bots.parsers.shodan.parser: diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md index 4969acb6d0..eb0ddfb4a7 100644 --- a/intelmq/bots/parsers/shadowserver/README.md +++ b/intelmq/bots/parsers/shadowserver/README.md @@ -7,60 +7,3 @@ This module is maintained by [The Shadowserver Foundation](https://www.shadowser Please contact intelmq@shadowserver.org with any issues or concerns. -The report configuration is now stored in a _shadowserver-schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/v1/schema. - -The parser will attempt to download a schema update on startup when the *auto_update* option is enabled. - -Schema downloads can also be scheduled as a cron job: - -``` -02 01 * * * intelmq.bots.parsers.shadowserver.parser --update-schema -``` - -For air-gapped systems automation will be required to download and copy the file to VAR_STATE_PATH/shadowserver-schema.json. - -The parser will automatically reload the configuration when the file changes. - - -## Schema contract - -Once set the `classification.identifier`, `classification.taxonomy`, and `classification.type` fields will remain static. - -Once set report fields will not be deleted. - -The schema revision history is maintained at https://github.com/The-Shadowserver-Foundation/report_schema/. - - -## Sample configuration: - -``` -shadowserver-collector: - description: Our bot responsible for getting reports from Shadowserver - enabled: true - group: Collector - module: intelmq.bots.collectors.shadowserver.collector_reports_api - name: Shadowserver_Collector - parameters: - destination_queues: - _default: [shadowserver-parser-queue] - file_format: csv - api_key: "$API_KEY_received_from_the_shadowserver_foundation" - secret: "$SECRET_received_from_the_shadowserver_foundation" - run_mode: continuous -``` - -``` -shadowserver-parser: - bot_id: shadowserver-parser - name: Shadowserver Parser - enabled: true - group: Parser - groupname: parsers - module: intelmq.bots.parsers.shadowserver.parser - parameters: - destination_queues: - _default: [file-output-queue] - auto_update: true - run_mode: continuous -``` - diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index 6ffffdae86..279093dfe3 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -95,6 +95,7 @@ class __Container: __config = __Container() +__config.var_state_path = VAR_STATE_PATH __config.schema_url = 'https://interchange.shadowserver.org/intelmq/v1/schema' __config.schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json') __config.schema_base = os.path.join(os.path.dirname(__file__), 'schema.json.test') @@ -328,7 +329,7 @@ def update_schema(): """ download the latest configuration """ # download the schema to a temp file - (th, tmp) = tempfile.mkstemp(dir=VAR_STATE_PATH) + (th, tmp) = tempfile.mkstemp(dir=__config.var_state_path) __config.logger.info("Attempting to download schema from %r", __config.schema_url) __config.logger.debug("Using temp file %r for the download.", tmp) try: @@ -376,3 +377,10 @@ def update_schema(): os.unlink(tmp) return False + + +def prepare_update_schema_test(path): + """ Reconfigure internal settings to perform a schema update test. """ + __config.var_state_path = path + __config.schema_file = os.path.join(path, 'shadowserver-schema.json') + return __config.schema_file diff --git a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py index f9512ca98c..5246e6bb67 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py @@ -10,8 +10,8 @@ import unittest import os +import tempfile import logging -from intelmq import VAR_STATE_PATH import intelmq.bots.parsers.shadowserver._config as config import intelmq.lib.utils as utils import intelmq.lib.test as test @@ -20,10 +20,8 @@ class TestShadowserverSchemaDownload(unittest.TestCase): def test_download(self): - if os.path.isdir(VAR_STATE_PATH): - schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json') + with tempfile.TemporaryDirectory() as tmp_dir: + schema_file = config.prepare_update_schema_test(tmp_dir) config.set_logger(utils.log('test-bot', log_path=None)) - if os.path.exists(schema_file): - os.unlink(schema_file) self.assertEqual(True, config.update_schema()) self.assertEqual(True, os.path.exists(schema_file)) From bd76ab71368485bfcba545a8005f442bc90e6ce2 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Fri, 25 Aug 2023 15:51:38 +0000 Subject: [PATCH 15/76] Added INTELMQ_SKIP_INTERNET check --- .../bots/parsers/shadowserver/test_download_schema.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py index 5246e6bb67..203a3c0b12 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py @@ -20,8 +20,9 @@ class TestShadowserverSchemaDownload(unittest.TestCase): def test_download(self): - with tempfile.TemporaryDirectory() as tmp_dir: - schema_file = config.prepare_update_schema_test(tmp_dir) - config.set_logger(utils.log('test-bot', log_path=None)) - self.assertEqual(True, config.update_schema()) - self.assertEqual(True, os.path.exists(schema_file)) + if not os.environ.get('INTELMQ_SKIP_INTERNET'): + with tempfile.TemporaryDirectory() as tmp_dir: + schema_file = config.prepare_update_schema_test(tmp_dir) + config.set_logger(utils.log('test-bot', log_path=None)) + self.assertEqual(True, config.update_schema()) + self.assertEqual(True, os.path.exists(schema_file)) From 6e5e110f3baae88c9782e7c214a8cb1a6cdcbf51 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Fri, 25 Aug 2023 16:11:21 +0000 Subject: [PATCH 16/76] Added debug logging for CI test. --- intelmq/bots/parsers/shadowserver/_config.py | 3 ++- .../tests/bots/parsers/shadowserver/test_download_schema.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index 279093dfe3..d573d12c61 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -339,8 +339,9 @@ def update_schema(): with open(tmp, 'wb') as f: for chunk in r.iter_content(chunk_size=8192): f.write(chunk) - except: + except Exception as e: __config.logger.error("Failed to download %r", __config.schema_url) + __config.logger.debug(str(e)) return False __config.logger.info("Download successful.") diff --git a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py index 203a3c0b12..abcd0ca2a4 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py @@ -23,6 +23,6 @@ def test_download(self): if not os.environ.get('INTELMQ_SKIP_INTERNET'): with tempfile.TemporaryDirectory() as tmp_dir: schema_file = config.prepare_update_schema_test(tmp_dir) - config.set_logger(utils.log('test-bot', log_path=None)) + config.set_logger(utils.log('test-bot', log_path=None, log_level=logging.DEBUG)) self.assertEqual(True, config.update_schema()) self.assertEqual(True, os.path.exists(schema_file)) From 01dcd5ee8d0ee6c99ab293526aaac2a43f2b2b22 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Fri, 25 Aug 2023 18:47:54 +0000 Subject: [PATCH 17/76] Refactored test_download_schema to utilize mocking. --- intelmq/bots/parsers/shadowserver/parser.py | 6 ++++ .../shadowserver/test_download_schema.py | 30 ++++++++++++------- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index fd9fa6b2cf..48cbba901a 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -23,6 +23,7 @@ import copy import re import os +import tempfile from intelmq.lib.bot import ParserBot from intelmq.lib.exceptions import InvalidKey, InvalidValue @@ -265,5 +266,10 @@ def run(cls, parsed_args=None): else: super().run(parsed_args=parsed_args) + def test_update_schema(cls): + with tempfile.TemporaryDirectory() as tmp_dir: + schema_file = config.prepare_update_schema_test(tmp_dir) + return config.update_schema() + BOT = ShadowserverParserBot diff --git a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py index abcd0ca2a4..abf27a5bd4 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py @@ -8,21 +8,29 @@ """ -import unittest -import os -import tempfile import logging -import intelmq.bots.parsers.shadowserver._config as config +import unittest +import unittest.mock as mock +from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot import intelmq.lib.utils as utils import intelmq.lib.test as test + @test.skip_internet() -class TestShadowserverSchemaDownload(unittest.TestCase): +class TestShadowserverSchemaDownload(test.BotTestCase, unittest.TestCase): + + @classmethod + def set_bot(cls): + cls.bot_reference = ShadowserverParserBot + cls.sysconfig = {"logging_level": "DEBUG"} def test_download(self): - if not os.environ.get('INTELMQ_SKIP_INTERNET'): - with tempfile.TemporaryDirectory() as tmp_dir: - schema_file = config.prepare_update_schema_test(tmp_dir) - config.set_logger(utils.log('test-bot', log_path=None, log_level=logging.DEBUG)) - self.assertEqual(True, config.update_schema()) - self.assertEqual(True, os.path.exists(schema_file)) + self.prepare_bot(prepare_source_queue=False, parameters={'test_mode': True}) + result = False + with mock.patch('intelmq.lib.utils.load_configuration', new=self.mocked_config): + with mock.patch('intelmq.lib.utils.log', self.get_mocked_logger(self.logger)): + self.log_stream.truncate(0) + result = self.bot.test_update_schema() + self.bot.stop(exitcode=0) + print(self.log_stream.getvalue()) + self.assertEqual(True, result) From 9314c84c19ef06cdfad508c64d3398785d82fff8 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Mon, 28 Aug 2023 14:18:22 +0000 Subject: [PATCH 18/76] Added docstring for test_update_schema(). --- intelmq/bots/parsers/shadowserver/parser.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index 48cbba901a..4485a26020 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -267,6 +267,13 @@ def run(cls, parsed_args=None): super().run(parsed_args=parsed_args) def test_update_schema(cls): + """ + Test schema download to a temporary directory. + + This is necessary as the request session requires mocking in order to function. + + Returns True on success. + """ with tempfile.TemporaryDirectory() as tmp_dir: schema_file = config.prepare_update_schema_test(tmp_dir) return config.update_schema() From 2f11b2a6667393c25eacd2584d14ade09065eb15 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 29 Aug 2023 14:09:33 +0000 Subject: [PATCH 19/76] Removed logging output. --- intelmq/tests/bots/parsers/shadowserver/test_download_schema.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py index abf27a5bd4..84922bf176 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py @@ -29,8 +29,6 @@ def test_download(self): result = False with mock.patch('intelmq.lib.utils.load_configuration', new=self.mocked_config): with mock.patch('intelmq.lib.utils.log', self.get_mocked_logger(self.logger)): - self.log_stream.truncate(0) result = self.bot.test_update_schema() self.bot.stop(exitcode=0) - print(self.log_stream.getvalue()) self.assertEqual(True, result) From 46f2ca775df9591b293c07e2a1a049d49264d25a Mon Sep 17 00:00:00 2001 From: elsif2 Date: Thu, 31 Aug 2023 20:52:17 +0000 Subject: [PATCH 20/76] Removed the assertion regarding report fields. --- docs/user/bots.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/user/bots.rst b/docs/user/bots.rst index 3da99af1e1..8d3c7555d0 100644 --- a/docs/user/bots.rst +++ b/docs/user/bots.rst @@ -1634,8 +1634,6 @@ The parser will automatically reload the configuration when the file changes. Once set in the schema, the `classification.identifier`, `classification.taxonomy`, and `classification.type` fields will remain static for a specific report. -Report fields will not be removed from a report. - The schema revision history is maintained at https://github.com/The-Shadowserver-Foundation/report_schema/. **Sample configuration** From d0311e0058b7e8c7125fed5ff50e9abe454afc64 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 11 Apr 2023 23:53:24 +0000 Subject: [PATCH 21/76] remove obsolete tests and data --- .../shadowserver/scan_rdpeudp.csv.license | 2 - .../parsers/shadowserver/test_blocklist.py | 103 ------- .../shadowserver/test_compromised_website.py | 88 ------ .../parsers/shadowserver/test_device_id.py | 116 -------- .../test_event4_ddos_participant.py | 131 --------- .../test_event4_honeypot_darknet.py | 106 ------- .../shadowserver/test_event4_honeypot_ddos.py | 148 ---------- .../test_event4_honeypot_ddos_target.py | 150 ---------- .../test_event4_honeypot_http_scan.py | 109 -------- .../shadowserver/test_event4_ip_spoofer.py | 182 ------------ .../test_event4_microsoft_sinkhole.py | 135 --------- .../test_event4_microsoft_sinkhole_http.py | 202 -------------- .../shadowserver/test_event4_sinkhole.py | 73 ----- .../shadowserver/test_event4_sinkhole_dns.py | 127 --------- .../shadowserver/test_event4_sinkhole_http.py | 189 ------------- .../test_event4_sinkhole_http_referer.py | 213 --------------- .../shadowserver/test_event6_sinkhole_http.py | 146 ---------- .../shadowserver/test_honeypot_brute_force.py | 72 ----- .../shadowserver/test_honeypot_ddos_amp.py | 91 ------ .../parsers/shadowserver/test_malware_url.py | 107 -------- .../parsers/shadowserver/test_phish_url.py | 106 ------- .../test_population_http_proxy.py | 130 --------- .../parsers/shadowserver/test_sandbox_conn.py | 99 ------- .../parsers/shadowserver/test_sandbox_dns.py | 95 ------- .../parsers/shadowserver/test_sandbox_url.py | 104 ------- .../parsers/shadowserver/test_scan_adb.py | 98 ------- .../parsers/shadowserver/test_scan_afp.py | 106 ------- .../parsers/shadowserver/test_scan_amqp.py | 144 ---------- .../parsers/shadowserver/test_scan_ard.py | 111 -------- .../parsers/shadowserver/test_scan_chargen.py | 110 -------- .../test_scan_cisco_smart_install.py | 82 ------ .../parsers/shadowserver/test_scan_coap.py | 121 -------- .../parsers/shadowserver/test_scan_couchdb.py | 128 --------- .../parsers/shadowserver/test_scan_cwmp.py | 103 ------- .../parsers/shadowserver/test_scan_db2.py | 91 ------ .../shadowserver/test_scan_ddos_middlebox.py | 119 -------- .../parsers/shadowserver/test_scan_dns.py | 91 ------ .../parsers/shadowserver/test_scan_docker.py | 159 ----------- .../test_scan_dvr_dhcpdiscover.py | 178 ------------ .../shadowserver/test_scan_elasticsearch.py | 126 --------- .../shadowserver/test_scan_exchange.py | 149 ---------- .../parsers/shadowserver/test_scan_ftp.py | 120 -------- .../parsers/shadowserver/test_scan_hadoop.py | 94 ------- .../parsers/shadowserver/test_scan_http.py | 100 ------- .../shadowserver/test_scan_http_proxy.py | 118 -------- .../shadowserver/test_scan_http_vulnerable.py | 125 --------- .../parsers/shadowserver/test_scan_ics.py | 125 --------- .../parsers/shadowserver/test_scan_ipmi.py | 106 ------- .../parsers/shadowserver/test_scan_ipp.py | 79 ------ .../parsers/shadowserver/test_scan_isakmp.py | 105 ------- .../shadowserver/test_scan_kubernetes.py | 214 --------------- .../shadowserver/test_scan_ldap_tcp.py | 154 ----------- .../shadowserver/test_scan_ldap_udp.py | 162 ----------- .../parsers/shadowserver/test_scan_mdns.py | 127 --------- .../shadowserver/test_scan_memcached.py | 130 --------- .../parsers/shadowserver/test_scan_mongodb.py | 103 ------- .../parsers/shadowserver/test_scan_mqtt.py | 89 ------ .../shadowserver/test_scan_mqtt_anon.py | 173 ------------ .../parsers/shadowserver/test_scan_mssql.py | 123 --------- .../parsers/shadowserver/test_scan_mysql.py | 258 ------------------ .../parsers/shadowserver/test_scan_nat_pmp.py | 116 -------- .../parsers/shadowserver/test_scan_netbios.py | 121 -------- .../shadowserver/test_scan_netis_router.py | 107 -------- .../parsers/shadowserver/test_scan_ntp.py | 161 ----------- .../shadowserver/test_scan_ntpmonitor.py | 108 -------- .../shadowserver/test_scan_portmapper.py | 120 -------- .../shadowserver/test_scan_postgres.py | 199 -------------- .../parsers/shadowserver/test_scan_qotd.py | 119 -------- .../parsers/shadowserver/test_scan_quic.py | 118 -------- .../parsers/shadowserver/test_scan_radmin.py | 236 ---------------- .../parsers/shadowserver/test_scan_rdp.py | 117 -------- .../parsers/shadowserver/test_scan_rdpeudp.py | 109 -------- .../parsers/shadowserver/test_scan_redis.py | 107 -------- .../parsers/shadowserver/test_scan_rsync.py | 116 -------- .../parsers/shadowserver/test_scan_sip.py | 124 --------- .../parsers/shadowserver/test_scan_slp.py | 137 ---------- .../parsers/shadowserver/test_scan_smb.py | 124 --------- .../shadowserver/test_scan_smb_json.py | 123 --------- .../shadowserver/test_scan_smtp_vulnerable.py | 92 ------- .../parsers/shadowserver/test_scan_snmp.py | 120 -------- .../parsers/shadowserver/test_scan_socks.py | 107 -------- .../parsers/shadowserver/test_scan_ssdp.py | 136 --------- .../parsers/shadowserver/test_scan_ssh.py | 182 ------------ .../parsers/shadowserver/test_scan_ssl.py | 218 --------------- .../shadowserver/test_scan_ssl_freak.py | 136 --------- .../shadowserver/test_scan_ssl_poodle.py | 91 ------ .../parsers/shadowserver/test_scan_stun.py | 146 ---------- .../shadowserver/test_scan_synfulknock.py | 117 -------- .../parsers/shadowserver/test_scan_telnet.py | 87 ------ .../parsers/shadowserver/test_scan_tftp.py | 121 -------- .../shadowserver/test_scan_ubiquiti.py | 124 --------- .../parsers/shadowserver/test_scan_vnc.py | 86 ------ .../shadowserver/test_scan_ws_discovery.py | 119 -------- .../parsers/shadowserver/test_scan_xdmcp.py | 117 -------- .../bots/parsers/shadowserver/test_special.py | 106 ------- .../parsers/shadowserver/test_testdata.py | 81 ------ .../shadowserver/testdata/blocklist.csv | 4 - .../testdata/blocklist.csv.license | 2 - .../testdata/botnet_drone.csv.license | 2 - .../testdata/caida_ip_spoofer.csv.license | 2 - .../testdata/compromised_website.csv | 4 - .../testdata/compromised_website.csv.license | 2 - .../shadowserver/testdata/darknet.csv.license | 2 - .../testdata/ddos_amplification.csv.license | 2 - .../shadowserver/testdata/device_id.csv | 4 - .../testdata/device_id.csv.license | 2 - .../testdata/drone_brute_force.csv.license | 2 - .../testdata/event4_ddos_participant.csv | 4 - .../event4_ddos_participant.csv.license | 2 - .../testdata/event4_honeypot_brute_force.csv | 7 - .../event4_honeypot_brute_force.csv.license | 2 - .../testdata/event4_honeypot_darknet.csv | 9 - .../event4_honeypot_darknet.csv.license | 2 - .../testdata/event4_honeypot_ddos.csv | 4 - .../testdata/event4_honeypot_ddos.csv.license | 2 - .../testdata/event4_honeypot_ddos_amp.csv | 6 - .../event4_honeypot_ddos_amp.csv.license | 2 - .../testdata/event4_honeypot_ddos_target.csv | 4 - .../event4_honeypot_ddos_target.csv.license | 2 - .../testdata/event4_honeypot_http_scan.csv | 3 - .../event4_honeypot_http_scan.csv.license | 2 - .../testdata/event4_ip_spoofer.csv | 7 - .../testdata/event4_ip_spoofer.csv.license | 2 - .../testdata/event4_microsoft_sinkhole.csv | 7 - .../event4_microsoft_sinkhole.csv.license | 2 - .../event4_microsoft_sinkhole_http.csv | 6 - ...event4_microsoft_sinkhole_http.csv.license | 2 - .../shadowserver/testdata/event4_sinkhole.csv | 4 - .../testdata/event4_sinkhole.csv.license | 2 - .../testdata/event4_sinkhole_dns.csv | 4 - .../testdata/event4_sinkhole_dns.csv.license | 2 - .../testdata/event4_sinkhole_http.csv | 6 - .../testdata/event4_sinkhole_http.csv.license | 2 - .../testdata/event4_sinkhole_http_referer.csv | 6 - .../event4_sinkhole_http_referer.csv.license | 2 - .../testdata/event6_sinkhole_http.csv | 4 - .../testdata/event6_sinkhole_http.csv.license | 2 - .../testdata/hp_http_scan.csv.license | 2 - .../testdata/hp_ics_scan.csv.license | 2 - .../shadowserver/testdata/malware_url.csv | 4 - .../testdata/malware_url.csv.license | 2 - .../testdata/outdated_dnssec_key.csv.license | 2 - .../shadowserver/testdata/phish_url.csv | 4 - .../testdata/phish_url.csv.license | 2 - .../testdata/population_http_proxy.csv | 4 - .../population_http_proxy.csv.license | 2 - .../shadowserver/testdata/sandbox_conn.csv | 4 - .../testdata/sandbox_conn.csv.license | 2 - .../shadowserver/testdata/sandbox_dns.csv | 4 - .../testdata/sandbox_dns.csv.license | 2 - .../shadowserver/testdata/sandbox_url.csv | 4 - .../testdata/sandbox_url.csv.license | 2 - .../shadowserver/testdata/scan_adb.csv | 3 - .../testdata/scan_adb.csv.license | 2 - .../shadowserver/testdata/scan_afp.csv | 3 - .../testdata/scan_afp.csv.license | 2 - .../shadowserver/testdata/scan_amqp.csv | 4 - .../testdata/scan_amqp.csv.license | 2 - .../shadowserver/testdata/scan_ard.csv | 4 - .../testdata/scan_ard.csv.license | 2 - .../shadowserver/testdata/scan_chargen.csv | 4 - .../testdata/scan_chargen.csv.license | 2 - .../testdata/scan_cisco_smart_install.csv | 3 - .../scan_cisco_smart_install.csv.license | 2 - .../shadowserver/testdata/scan_coap.csv | 4 - .../testdata/scan_coap.csv.license | 2 - .../shadowserver/testdata/scan_couchdb.csv | 4 - .../testdata/scan_couchdb.csv.license | 2 - .../shadowserver/testdata/scan_cwmp.csv | 3 - .../testdata/scan_cwmp.csv.license | 2 - .../shadowserver/testdata/scan_db2.csv | 3 - .../testdata/scan_db2.csv.license | 2 - .../testdata/scan_ddos_middlebox.csv | 4 - .../testdata/scan_ddos_middlebox.csv.license | 2 - .../shadowserver/testdata/scan_dns.csv | 101 ------- .../testdata/scan_dns.csv.license | 2 - .../shadowserver/testdata/scan_docker.csv | 4 - .../testdata/scan_docker.csv.license | 2 - .../testdata/scan_dvr_dhcpdiscover.csv | 4 - .../scan_dvr_dhcpdiscover.csv.license | 2 - .../testdata/scan_elasticsearch.csv | 4 - .../testdata/scan_elasticsearch.csv.license | 2 - .../shadowserver/testdata/scan_exchange.csv | 8 - .../testdata/scan_exchange.csv.license | 2 - .../shadowserver/testdata/scan_ftp.csv | 3 - .../testdata/scan_ftp.csv.license | 2 - .../shadowserver/testdata/scan_hadoop.csv | 3 - .../testdata/scan_hadoop.csv.license | 2 - .../shadowserver/testdata/scan_http.csv | 3 - .../testdata/scan_http.csv.license | 2 - .../shadowserver/testdata/scan_http_proxy.csv | 4 - .../testdata/scan_http_proxy.csv.license | 2 - .../testdata/scan_http_vulnerable.csv | 4 - .../testdata/scan_http_vulnerable.csv.license | 2 - .../shadowserver/testdata/scan_ics.csv | 4 - .../testdata/scan_ics.csv.license | 2 - .../shadowserver/testdata/scan_ipmi.csv | 96 ------- .../testdata/scan_ipmi.csv.license | 2 - .../shadowserver/testdata/scan_ipp.csv | 2 - .../testdata/scan_ipp.csv.license | 2 - .../shadowserver/testdata/scan_isakmp.csv | 3 - .../testdata/scan_isakmp.csv.license | 2 - .../shadowserver/testdata/scan_kubernetes.csv | 4 - .../testdata/scan_kubernetes.csv.license | 2 - .../shadowserver/testdata/scan_ldap_tcp.csv | 4 - .../testdata/scan_ldap_tcp.csv.license | 2 - .../shadowserver/testdata/scan_ldap_udp.csv | 4 - .../testdata/scan_ldap_udp.csv.license | 2 - .../shadowserver/testdata/scan_mdns.csv | 4 - .../testdata/scan_mdns.csv.license | 2 - .../shadowserver/testdata/scan_memcached.csv | 4 - .../testdata/scan_memcached.csv.license | 2 - .../shadowserver/testdata/scan_mongodb.csv | 11 - .../testdata/scan_mongodb.csv.license | 2 - .../shadowserver/testdata/scan_mqtt.csv | 2 - .../testdata/scan_mqtt.csv.license | 2 - .../shadowserver/testdata/scan_mqtt_anon.csv | 4 - .../testdata/scan_mqtt_anon.csv.license | 2 - .../shadowserver/testdata/scan_mssql.csv | 4 - .../testdata/scan_mssql.csv.license | 2 - .../shadowserver/testdata/scan_mysql.csv | 4 - .../testdata/scan_mysql.csv.license | 2 - .../shadowserver/testdata/scan_nat_pmp.csv | 4 - .../testdata/scan_nat_pmp.csv.license | 2 - .../shadowserver/testdata/scan_netbios.csv | 4 - .../testdata/scan_netbios.csv.license | 2 - .../testdata/scan_netis_router.csv | 4 - .../testdata/scan_netis_router.csv.license | 2 - .../shadowserver/testdata/scan_ntp.csv | 4 - .../testdata/scan_ntp.csv.license | 2 - .../shadowserver/testdata/scan_ntpmonitor.csv | 4 - .../testdata/scan_ntpmonitor.csv.license | 2 - .../shadowserver/testdata/scan_portmapper.csv | 4 - .../testdata/scan_portmapper.csv.license | 2 - .../shadowserver/testdata/scan_postgres.csv | 4 - .../testdata/scan_postgres.csv.license | 2 - .../shadowserver/testdata/scan_qotd.csv | 4 - .../testdata/scan_qotd.csv.license | 2 - .../shadowserver/testdata/scan_quic.csv | 4 - .../testdata/scan_quic.csv.license | 2 - .../shadowserver/testdata/scan_radmin.csv | 10 - .../testdata/scan_radmin.csv.license | 2 - .../shadowserver/testdata/scan_rdp.csv | 3 - .../testdata/scan_rdp.csv.license | 2 - .../shadowserver/testdata/scan_rdpeudp.csv | 4 - .../testdata/scan_rdpeudp.csv.license | 2 - .../shadowserver/testdata/scan_redis.csv | 94 ------- .../testdata/scan_redis.csv.license | 2 - .../shadowserver/testdata/scan_rsync.csv | 4 - .../testdata/scan_rsync.csv.license | 2 - .../shadowserver/testdata/scan_sip.csv | 4 - .../testdata/scan_sip.csv.license | 2 - .../shadowserver/testdata/scan_slp.csv | 4 - .../testdata/scan_slp.csv.license | 2 - .../shadowserver/testdata/scan_smb.csv | 4 - .../testdata/scan_smb.csv.license | 2 - .../testdata/scan_smtp_vulnerable.csv | 3 - .../testdata/scan_smtp_vulnerable.csv.license | 2 - .../shadowserver/testdata/scan_snmp.csv | 4 - .../testdata/scan_snmp.csv.license | 2 - .../shadowserver/testdata/scan_socks.csv | 4 - .../testdata/scan_socks.csv.license | 2 - .../shadowserver/testdata/scan_ssdp.csv | 4 - .../testdata/scan_ssdp.csv.license | 2 - .../shadowserver/testdata/scan_ssh.csv | 4 - .../testdata/scan_ssh.csv.license | 2 - .../shadowserver/testdata/scan_ssl.csv | 4 - .../testdata/scan_ssl.csv.license | 2 - .../shadowserver/testdata/scan_ssl_freak.csv | 46 ---- .../testdata/scan_ssl_freak.csv.license | 2 - .../shadowserver/testdata/scan_ssl_poodle.csv | 32 --- .../testdata/scan_ssl_poodle.csv.license | 2 - .../shadowserver/testdata/scan_stun.csv | 4 - .../testdata/scan_stun.csv.license | 2 - .../testdata/scan_synfulknock.csv | 4 - .../testdata/scan_synfulknock.csv.license | 2 - .../shadowserver/testdata/scan_telnet.csv | 3 - .../testdata/scan_telnet.csv.license | 2 - .../shadowserver/testdata/scan_tftp.csv | 4 - .../testdata/scan_tftp.csv.license | 2 - .../shadowserver/testdata/scan_ubiquiti.csv | 4 - .../testdata/scan_ubiquiti.csv.license | 2 - .../shadowserver/testdata/scan_vnc.csv | 3 - .../testdata/scan_vnc.csv.license | 2 - .../testdata/scan_ws_discovery.csv | 4 - .../testdata/scan_ws_discovery.csv.license | 2 - .../shadowserver/testdata/scan_xdmcp.csv | 4 - .../testdata/scan_xdmcp.csv.license | 2 - .../testdata/sinkhole_http_drone.csv.license | 2 - .../parsers/shadowserver/testdata/special.csv | 4 - .../shadowserver/testdata/special.csv.license | 2 - 291 files changed, 12939 deletions(-) delete mode 100644 intelmq/tests/bots/parsers/shadowserver/scan_rdpeudp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_blocklist.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_compromised_website.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_device_id.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_ddos_participant.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_darknet.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos_target.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_http_scan.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_ip_spoofer.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole_http.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_dns.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http_referer.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event6_sinkhole_http.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_honeypot_brute_force.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_honeypot_ddos_amp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_malware_url.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_phish_url.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_population_http_proxy.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_sandbox_conn.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_sandbox_dns.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_sandbox_url.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_adb.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_afp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_amqp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ard.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_chargen.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_cisco_smart_install.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_coap.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_couchdb.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_cwmp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_db2.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ddos_middlebox.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_dns.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_docker.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_dvr_dhcpdiscover.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_elasticsearch.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_exchange.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ftp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_hadoop.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_http.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_http_proxy.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_http_vulnerable.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ics.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ipmi.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ipp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_isakmp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_kubernetes.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_tcp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_udp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_mdns.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_memcached.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_mongodb.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt_anon.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_mssql.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_mysql.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_nat_pmp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_netbios.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_netis_router.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ntp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ntpmonitor.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_portmapper.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_postgres.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_qotd.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_quic.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_radmin.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_rdp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_rdpeudp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_redis.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_rsync.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_sip.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_slp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_smb.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_smb_json.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_smtp_vulnerable.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_snmp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_socks.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ssdp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ssh.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ssl.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_freak.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_poodle.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_stun.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_synfulknock.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_telnet.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_tftp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ubiquiti.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_vnc.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ws_discovery.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_xdmcp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_special.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_testdata.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/botnet_drone.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/caida_ip_spoofer.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/darknet.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/ddos_amplification.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/drone_brute_force.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/hp_http_scan.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/hp_ics_scan.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/outdated_dnssec_key.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sinkhole_http_drone.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/special.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/special.csv.license diff --git a/intelmq/tests/bots/parsers/shadowserver/scan_rdpeudp.csv.license b/intelmq/tests/bots/parsers/shadowserver/scan_rdpeudp.csv.license deleted file mode 100644 index 043ed079f1..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/scan_rdpeudp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Sebastian Waldbauer -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/test_blocklist.py b/intelmq/tests/bots/parsers/shadowserver/test_blocklist.py deleted file mode 100644 index 48509eea0e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_blocklist.py +++ /dev/null @@ -1,103 +0,0 @@ -# SPDX-FileCopyrightText: 2020 Thomas Hungenberg -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/blocklist.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = { - 'feed.name': 'Block Listed IP Addresses', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-blocklist-test-geo.csv", -} -EVENTS = [{ - '__type': 'Event', - 'feed.name': 'Block Listed IP Addresses', - "classification.identifier": "blacklisted-ip", - "classification.taxonomy": "other", - "classification.type": "blacklist", - "extra.naics": 517311, - "extra.reason": "Malicious Host AA", - "extra.source": "Alien Vault", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 5678, - "source.geolocation.cc": "XX", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.134", - "source.reverse_dns": "host.local", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T07:00:19+00:00" -}, -{ - '__type': 'Event', - 'feed.name': 'Block Listed IP Addresses', - "classification.identifier": "blacklisted-ip", - "classification.taxonomy": "other", - "classification.type": "blacklist", - "extra.naics": 517311, - "extra.reason": "Malicious Host AA", - "extra.source": "Alien Vault", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 5678, - "source.geolocation.cc": "XX", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.171", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T07:00:19+00:00" -}, -{ - '__type': 'Event', - 'feed.name': 'Block Listed IP Addresses', - "classification.identifier": "blacklisted-ip", - "classification.taxonomy": "other", - "classification.type": "blacklist", - "extra.naics": 517311, - "extra.reason": "Malicious Host AA", - "extra.source": "Alien Vault", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - "source.asn": 5678, - "source.geolocation.cc": "XX", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.network": "198.123.245.0/24", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T07:00:19+00:00" -},] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_compromised_website.py b/intelmq/tests/bots/parsers/shadowserver/test_compromised_website.py deleted file mode 100644 index 53c5b247b1..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_compromised_website.py +++ /dev/null @@ -1,88 +0,0 @@ -# SPDX-FileCopyrightText: 2017 Sebastian Wagner -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/compromised_website.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "ShadowServer Compromised Website", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-compromised_website-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'ShadowServer Compromised Website', - 'classification.taxonomy': 'intrusions', - 'classification.type': 'system-compromise', - 'classification.identifier': 'compromised-website', - 'extra.server': 'Microsoft-IIS/7.5', - 'extra.system': 'WINNT', - 'extra.detected_since': '2015-05-09 05:51:12', - 'protocol.application': 'http', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 64496, - 'source.geolocation.cc': 'AT', - 'source.geolocation.city': 'VIENNA', - 'source.geolocation.region': 'WIEN', - 'source.ip': '203.0.113.1', - 'source.port': 80, - 'source.url': 'http://example.com/header.php', - 'source.fqdn': 'example.com', - 'source.reverse_dns': 'example.com', - 'malware.name': 'hacked-webserver-stealrat-t1', - 'event_description.text': 'spam', - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2017-01-16T00:43:48+00:00'}, - {'__type': 'Event', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'classification.identifier': 'compromised-website', - 'classification.taxonomy': 'intrusions', - 'classification.type': 'system-compromise', - 'event_description.text': 'phishing', - 'feed.name': 'ShadowServer Compromised Website', - 'malware.name': 'phishing', - 'protocol.application': 'http', - 'source.asn': 64496, - 'source.fqdn': 'example.com', - 'source.geolocation.cc': 'AT', - 'source.geolocation.city': 'GRAZ', - 'source.geolocation.region': 'STEIERMARK', - 'source.ip': '203.0.113.1', - 'source.port': 80, - 'source.url': 'http://example.com/', - 'time.source': '2018-04-09T15:43:41+00:00'}, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_device_id.py b/intelmq/tests/bots/parsers/shadowserver/test_device_id.py deleted file mode 100644 index e8954e03c1..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_device_id.py +++ /dev/null @@ -1,116 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/device_id.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Device ID', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-device_id-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'device-id', - 'classification.taxonomy' : 'other', - 'classification.type' : 'undetermined', - 'extra.device_model' : 'FortiGate', - 'extra.device_type' : 'firewall', - 'extra.device_vendor' : 'Fortinet', - 'extra.naics' : 517311, - 'feed.name' : 'Device ID', - 'extra.tag' : 'ssl,vpn', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 2116, - 'source.geolocation.cc' : 'NO', - 'source.geolocation.city' : 'TROMVIK', - 'source.geolocation.region' : 'TROMS OG FINNMARK', - 'source.ip' : '88.84.0.0', - 'source.port' : 10443, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:42+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'device-id', - 'classification.taxonomy' : 'other', - 'classification.type' : 'undetermined', - 'extra.device_model' : 'FortiGate', - 'extra.device_type' : 'firewall', - 'extra.device_vendor' : 'Fortinet', - 'feed.name' : 'Device ID', - 'extra.tag' : 'ssl,vpn', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 27843, - 'source.geolocation.cc' : 'PE', - 'source.geolocation.city' : 'LIMA', - 'source.geolocation.region' : 'METROPOLITANA DE LIMA', - 'source.ip' : '170.231.0.0', - 'source.port' : 10443, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:42+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'device-id', - 'classification.taxonomy' : 'other', - 'classification.type' : 'undetermined', - 'extra.device_model' : 'FortiGate', - 'extra.device_type' : 'firewall', - 'extra.device_vendor' : 'Fortinet', - 'extra.naics' : 517311, - 'feed.name' : 'Device ID', - 'extra.tag' : 'ssl,vpn', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 4181, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'MILWAUKEE', - 'source.geolocation.region' : 'WISCONSIN', - 'source.ip' : '96.60.0.0', - 'source.port' : 10443, - 'source.reverse_dns' : '96-60-66-218.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:42+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_ddos_participant.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_ddos_participant.py deleted file mode 100644 index badc53a736..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_ddos_participant.py +++ /dev/null @@ -1,131 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_ddos_participant.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'DDoS Participant', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-event4_ddos_participant-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'ddos-participant', - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'destination.asn': 65534, - 'destination.geolocation.cc': 'ZZ', - 'destination.geolocation.city': 'City', - 'destination.geolocation.region': 'Region', - 'destination.ip': '172.16.0.1', - 'destination.port': 443, - 'destination.reverse_dns': 'node01.example.net', - 'extra.application': 'https', - 'extra.domain': 'www.example.com', - 'extra.http_method': 'GET', - 'extra.http_path': '/??=GovpfOoaWYlk', - 'feed.name': 'DDoS Participant', - 'malware.name': 'ddos-participant', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 38055, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'ddos-participant', - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'destination.asn': 65534, - 'destination.geolocation.cc': 'ZZ', - 'destination.geolocation.city': 'City', - 'destination.geolocation.region': 'Region', - 'destination.ip': '172.16.0.2', - 'destination.port': 53, - 'destination.reverse_dns': 'node02.example.net', - 'extra.application': 'dns', - 'feed.name': 'DDoS Participant', - 'malware.name': 'ddos-participant', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 53, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'ddos-participant', - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'destination.asn': 65534, - 'destination.geolocation.cc': 'ZZ', - 'destination.geolocation.city': 'City', - 'destination.geolocation.region': 'Region', - 'destination.ip': '172.16.0.3', - 'destination.port': 53, - 'destination.reverse_dns': 'node03.example.net', - 'extra.application': 'dns', - 'extra.device_model': 'Exchange', - 'extra.device_type': 'email', - 'extra.device_vendor': 'Microsoft', - 'feed.name': 'DDoS Participant', - 'malware.name': 'ddos-participant', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 53, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_darknet.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_darknet.py deleted file mode 100644 index 1d020f4737..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_darknet.py +++ /dev/null @@ -1,106 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_honeypot_darknet.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "ShadowServer Darknet", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_honeypot_darknet.csv", - } -EVENTS = [{'__type': 'Event', - 'classification.identifier': 'mirai', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'destination.port': 23, - 'extra.source.naics': 518210, - 'extra.tag': 'mirai', - 'protocol.transport': 'tcp', - 'feed.name': 'ShadowServer Darknet', - 'malware.name': 'mirai', - 'source.asn': 9829, - 'source.geolocation.cc': 'IN', - 'source.geolocation.city': 'CHENGANNUR', - 'source.geolocation.region': 'KERALA', - 'source.ip': '61.3.1.2', - 'source.port': 4717, - 'time.source': '2021-03-07T00:00:00+00:00', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - }, - {'__type': 'Event', - 'classification.identifier': 'mirai', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'destination.port': 23, - 'protocol.transport': 'tcp', - 'extra.source.naics': 517311, - 'extra.tag': 'mirai', - 'feed.name': 'ShadowServer Darknet', - 'malware.name': 'mirai', - 'source.asn': 4766, - 'source.geolocation.cc': 'KR', - 'source.geolocation.city': 'PYEONGCHANG-EUP', - 'source.geolocation.region': 'GANGWON-DO', - 'source.ip': '211.218.3.4', - 'source.port': 4405, - 'time.source': '2021-03-07T00:00:00+00:00', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - }, - {'__type': 'Event', - 'classification.identifier': 'mirai', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'extra.tag': 'mirai', - 'destination.port': 23, - 'protocol.transport': 'tcp', - 'feed.name': 'ShadowServer Darknet', - 'malware.name': 'mirai', - 'source.asn': 266915, - 'source.geolocation.cc': 'BR', - 'source.geolocation.city': 'VITORIA DA CONQUISTA', - 'source.geolocation.region': 'BAHIA', - 'source.ip': '45.225.5.6', - 'source.port': 59777, - 'source.reverse_dns': 'static-45-225-x-x.example.net', - 'time.source': '2021-03-07T00:00:00+00:00', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - }, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos.py deleted file mode 100644 index c62a610faf..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos.py +++ /dev/null @@ -1,148 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_honeypot_ddos.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Honeypot DDoS Events', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-event4_honeypot_ddos-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'honeypot-ddos', - 'classification.taxonomy' : 'availability', - 'classification.type' : 'ddos', - 'destination.asn' : 65534, - 'destination.geolocation.cc' : 'ZZ', - 'destination.geolocation.city' : 'City', - 'destination.geolocation.region' : 'Region', - 'destination.ip' : '172.16.0.1', - 'destination.port' : 88, - 'destination.reverse_dns' : 'node01.example.net', - 'extra.application' : 'mirai', - 'extra.attack' : 'atk10', - 'extra.dst_netmask' : '32', - 'extra.dst_network' : '121.12.110.28/32', - 'extra.duration' : 30, - 'extra.family' : 'mirai', - 'extra.packet_length' : 1440, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'mirai', - 'feed.name' : 'Honeypot DDoS Events', - 'malware.name' : 'ddos', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 61234, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'honeypot-ddos', - 'classification.taxonomy' : 'availability', - 'classification.type' : 'ddos', - 'destination.asn' : 65534, - 'destination.geolocation.cc' : 'ZZ', - 'destination.geolocation.city' : 'City', - 'destination.geolocation.region' : 'Region', - 'destination.ip' : '172.16.0.2', - 'destination.port' : 80, - 'destination.reverse_dns' : 'node02.example.net', - 'extra.application' : 'mirai', - 'extra.attack' : 'atk10', - 'extra.dst_netmask' : '32', - 'extra.dst_network' : '180.97.183.94/32', - 'extra.duration' : 30, - 'extra.family' : 'mirai', - 'extra.packet_length' : 1440, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'mirai', - 'feed.name' : 'Honeypot DDoS Events', - 'malware.name' : 'ddos', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 61234, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'honeypot-ddos', - 'classification.taxonomy' : 'availability', - 'classification.type' : 'ddos', - 'destination.asn' : 65534, - 'destination.geolocation.cc' : 'ZZ', - 'destination.geolocation.city' : 'City', - 'destination.geolocation.region' : 'Region', - 'destination.ip' : '172.16.0.3', - 'destination.reverse_dns' : 'node03.example.net', - 'extra.application' : 'mirai', - 'extra.attack' : 'atk7', - 'extra.destination.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.dst_netmask' : '32', - 'extra.dst_network' : '104.237.138.135/32', - 'extra.duration' : 10, - 'extra.family' : 'mirai', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'mirai', - 'feed.name' : 'Honeypot DDoS Events', - 'malware.name' : 'ddos', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 6379, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos_target.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos_target.py deleted file mode 100644 index f379d1c882..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos_target.py +++ /dev/null @@ -1,150 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_honeypot_ddos_target.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Honeypot DDoS Target Events', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-event4_honeypot_ddos_target-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'honeypot-ddos-target', - 'classification.taxonomy' : 'availability', - 'classification.type' : 'ddos', - 'destination.asn' : 65534, - 'destination.geolocation.cc' : 'ZZ', - 'destination.geolocation.city' : 'City', - 'destination.geolocation.region' : 'Region', - 'destination.ip' : '172.16.0.1', - 'destination.port' : 80, - 'destination.reverse_dns' : 'node01.example.net', - 'extra.application' : 'mirai', - 'extra.attack' : 'atk0', - 'extra.dst_netmask' : '32', - 'extra.dst_network' : '115.238.198.85/32', - 'extra.duration' : 30, - 'extra.family' : 'mirai', - 'extra.packet_length' : 1440, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'mirai', - 'feed.name' : 'Honeypot DDoS Target Events', - 'malware.name' : 'ddos', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 61234, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'honeypot-ddos-target', - 'classification.taxonomy' : 'availability', - 'classification.type' : 'ddos', - 'destination.asn' : 65534, - 'destination.geolocation.cc' : 'ZZ', - 'destination.geolocation.city' : 'City', - 'destination.geolocation.region' : 'Region', - 'destination.ip' : '172.16.0.2', - 'destination.port' : 43437, - 'destination.reverse_dns' : 'node02.example.net', - 'extra.application' : 'mirai', - 'extra.attack' : 'atk0', - 'extra.destination.sector' : 'Information', - 'extra.dst_netmask' : '32', - 'extra.dst_network' : '52.184.50.250/32', - 'extra.duration' : 30, - 'extra.family' : 'mirai', - 'extra.packet_length' : 1440, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'mirai', - 'feed.name' : 'Honeypot DDoS Target Events', - 'malware.name' : 'ddos', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 61234, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'honeypot-ddos-target', - 'classification.taxonomy' : 'availability', - 'classification.type' : 'ddos', - 'destination.asn' : 65534, - 'destination.geolocation.cc' : 'ZZ', - 'destination.geolocation.city' : 'City', - 'destination.geolocation.region' : 'Region', - 'destination.ip' : '172.16.0.3', - 'destination.port' : 80, - 'destination.reverse_dns' : 'node03.example.net', - 'extra.application' : 'mirai', - 'extra.attack' : 'atk10', - 'extra.dst_netmask' : '32', - 'extra.dst_network' : '211.99.102.216/32', - 'extra.duration' : 30, - 'extra.family' : 'mirai', - 'extra.packet_length' : 1440, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'mirai', - 'feed.name' : 'Honeypot DDoS Target Events', - 'malware.name' : 'ddos', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 61234, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_http_scan.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_http_scan.py deleted file mode 100644 index bcf268ba7d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_http_scan.py +++ /dev/null @@ -1,109 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Mikk Margus Möll -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_honeypot_http_scan.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Honeypot-HTTP-Scan', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2021-08-01T12:00:00+00:00", - "extra.file_name": "2021-08-01-event4_honeypot_http_scan.csv", - } - -EVENTS = [{'__type': 'Event', - 'feed.name': 'Honeypot-HTTP-Scan', - 'classification.identifier': 'honeypot-http-scan', - 'classification.taxonomy': 'information-gathering', - 'classification.type': 'scanner', - 'destination.asn': 5678, - 'destination.geolocation.cc': 'UK', - 'destination.geolocation.city': 'MAIDENHEAD', - 'destination.geolocation.region': 'WINDSOR AND MAIDENHEAD', - 'destination.ip': '109.87.65.43', - 'destination.port': 80, - 'extra.http_url': '/js/ueditor/wwwroot/way-board.cgi', - 'extra.destination.naics': 518210, - 'protocol.transport': 'tcp', - 'protocol.application': 'http', - 'extra.public_source': 'CAPRICA-EU', - 'extra.request_raw': 'R0VUIC9qcy91ZWRpdG9yL3d3d3Jvb3Qvd2F5LWJvYXJkLmNnaSBIVFRQLzEuMHJuQWNjZXB0OiB0ZXh0L2h0bWwsYXBwbGljYXRpb24veGh0bWwreG1sLGFwcGxpY2F0aW9uL3htbDtxPTAuOSwqLyo7cT0wLjhybkFjY2VwdC1FbmNvZGluZzogZ3ppcCwgZGVmbGF0ZXJuQWNjZXB0LUxhbmd1YWdlOiBlbi1VUyxlbjtxPTAuNXJuQ29ubmVjdGlvbjogY2xvc2VybkRudDogMXJuSG9zdDogMTA5Ljg3LjY1LjQzcm5PcmlnaW46IGh0dHA6Ly8xMDkuODcuNjUuNDNyblJlZmVyZXI6IGh0dHA6Ly8xMDkuODcuNjUuNDMvcm5Vc2VyLUFnZW50OiBNb3ppbGxhLzUuMCAoV2luZG93cyBOVCA2LjE7IFdPVzY0KSBBcHBsZVdlYktpdC81MzcuMzYgKEtIVE1MLCBsaWtlIEdlY2tvKSBDaHJvbWUvNTMuMC4yNzg1LjEwNCBTYWZhcmkvNTM3LjM2IENvcmUvMS41My4zMDg0LjQwMCBRUUJyb3dzZXIvOS42LjExMzQ2LjQwMA==', - 'extra.source.naics': 518210, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.version': '3.1.3-dev', - 'malware.name': 'http-scan', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 1234, - 'source.geolocation.cc': 'EE', - 'source.geolocation.city': 'TALLINN', - 'source.geolocation.region': 'HARJUMAA', - 'source.ip': '191.23.45.67', - 'source.port': 36455, - 'source.reverse_dns': '191-23-45-67-host.example.com', - 'time.observation': '2021-08-01T12:00:00+00:00', - 'time.source': '2021-08-01T00:24:08+00:00'}, - {'__type': 'Event', - 'feed.name': 'Honeypot-HTTP-Scan', - 'classification.identifier': 'honeypot-http-scan', - 'classification.taxonomy': 'information-gathering', - 'classification.type': 'scanner', - 'destination.asn': 23456, - 'destination.geolocation.cc': 'UA', - 'destination.geolocation.city': 'KHARKIV', - 'destination.geolocation.region': "KHARKIVS'KA OBLAST'", - 'destination.ip': '82.41.20.10', - 'destination.port': 8080, - 'extra.http_url': '/', - 'extra.method': 'GET', - 'protocol.transport': 'tcp', - 'protocol.application': 'http', - 'extra.public_source': 'CAPRICA-EU', - 'extra.request_raw': 'R0VUIC8gSFRUUC8xLjENCkhvc3Q6IDgyLjQxLjIwLjEwOjgwODANCkFjY2VwdDogdGV4dC9odG1sLGFwcGxpY2F0aW9uL3hodG1sK3htbCxhcHBsaWNhdGlvbi94bWw7cT0wLjksKi8qO3E9MC44DQpBY2NlcHQtRW5jb2Rpbmc6IGRlZmxhdGUsIGd6aXAsIGlkZW50aXR5DQpBY2NlcHQtTGFuZ3VhZ2U6IGVuLVVTO3E9MC42LGVuO3E9MC40DQpVc2VyLUFnZW50OiBNb3ppbGxhLzUuMCAoV2luZG93cyBOVCA1LjE7IHJ2OjkuMC4xKSBHZWNrby8yMDEwMDEwMSBGaXJlZm94LzkuMC4xDQoNCg==', - 'extra.url_scheme': 'http', - 'extra.user_agent': 'Mozilla/5.0 (Windows NT 5.1; rv:9.0.1) Gecko/20100101 Firefox/9.0.1', - 'malware.name': 'http-scan', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn': 12345, - 'source.geolocation.cc': 'EE', - 'source.geolocation.city': 'TALLINN', - 'source.geolocation.region': 'HARJUMAA', - 'source.ip': '45.67.89.123', - 'source.port': 58610, - 'time.observation': '2021-08-01T12:00:00+00:00', - 'time.source': '2021-08-01T05:21:59+00:00'}, -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_ip_spoofer.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_ip_spoofer.py deleted file mode 100644 index d21fb10c5b..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_ip_spoofer.py +++ /dev/null @@ -1,182 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), "testdata/event4_ip_spoofer.csv")) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = { - "feed.name": "CAIDA", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2020-08-19T00:00:00+00:00", - "extra.file_name": "2020-08-19-event4_ip_spoofer.csv", -} - -EVENTS = [ - { - "__type": "Event", - "feed.name": "CAIDA", - "time.source": "2021-03-28T00:42:59+00:00", - "source.ip": "98.191.250.0", - - "source.asn": 22898, - - "source.geolocation.cc": "US", - "source.geolocation.region": "OKLAHOMA", - "source.geolocation.city": "OKLAHOMA CITY", - "source.network": "98.191.250.0/24", - "source.reverse_dns": 'ip-98.191.250.0.atlinkservices.com', - "extra.routedspoof": "received", - "extra.session": '1112907', - "extra.nat": True, - "extra.public_source": "caida", - "extra.source.naics": 517311, - "extra.version": 'ipv4', - "protocol.transport": 'tcp', - "extra.infection": 'ip-spoofer', - - "classification.identifier": "ip-spoofer", - "classification.taxonomy": "fraud", - "classification.type": "masquerade", - - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[1]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "CAIDA", - "time.source": "2021-03-28T01:36:22+00:00", - "source.ip": "191.7.16.0", - - "source.asn": 262485, - - "source.geolocation.cc": "BR", - "source.geolocation.region": "RIO DE JANEIRO", - "source.geolocation.city": "NOVA IGUACU", - "source.network": "191.7.16.0/24", - "extra.routedspoof": "received", - "extra.session": '1112914', - "extra.nat": False, - "extra.public_source": "caida", - "extra.version": 'ipv4', - "protocol.transport": 'tcp', - "extra.infection": 'ip-spoofer', - - "classification.identifier": "ip-spoofer", - "classification.taxonomy": "fraud", - "classification.type": "masquerade", - - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[2]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "CAIDA", - "time.source": "2021-03-28T02:10:58+00:00", - "source.ip": "202.53.160.0", - - "source.asn": 23923, - - "source.geolocation.cc": "BD", - "source.geolocation.region": "DHAKA", - "source.geolocation.city": "DHAKA", - "source.network": "202.53.160.0/24", - "extra.routedspoof": "received", - "extra.session": '1112931', - "extra.nat": True, - "extra.public_source": "caida", - "extra.version": 'ipv4', - "protocol.transport": 'tcp', - "extra.infection": 'ip-spoofer', - - "classification.identifier": "ip-spoofer", - "classification.taxonomy": "fraud", - "classification.type": "masquerade", - - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[3]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "CAIDA", - "time.source": "2021-03-28T03:41:51+00:00", - "source.ip": "87.121.75.0", - - "source.asn": 134697, - - "source.geolocation.cc": "AU", - "source.geolocation.region": "QUEENSLAND", - "source.geolocation.city": "BRISBANE", - "source.network": "87.121.75.0/24", - "extra.routedspoof": "received", - "extra.session": '1112953', - "extra.nat": True, - "extra.public_source": "caida", - "extra.version": 'ipv4', - "protocol.transport": 'tcp', - "extra.infection": 'ip-spoofer', - - "classification.identifier": "ip-spoofer", - "classification.taxonomy": "fraud", - "classification.type": "masquerade", - - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[4]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "CAIDA", - "time.source": "2021-03-28T06:07:17+00:00", - "source.ip": "189.201.194.0", - - "source.asn": 262944, - - "source.network": "189.201.194.0/24", - "source.geolocation.cc": 'MX', - "source.geolocation.city": 'SALTILLO', - "source.geolocation.region": 'COAHUILA', - "source.reverse_dns": 'ip-189-201-194-0.slw.spectro.mx', - "extra.routedspoof": "received", - "extra.session": '1113015', - "extra.nat": True, - "extra.public_source": "caida", - "extra.version": 'ipv4', - "protocol.transport": 'tcp', - "extra.infection": 'ip-spoofer', - - "classification.identifier": "ip-spoofer", - "classification.taxonomy": "fraud", - "classification.type": "masquerade", - - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[5]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, -] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == "__main__": - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole.py deleted file mode 100644 index f008fd18e1..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole.py +++ /dev/null @@ -1,135 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_microsoft_sinkhole.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "ShadowServer Microsoft Sinkhole", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_microsoft_sinkhole.csv", - } -EVENTS = [{'__type': 'Event', - 'classification.identifier': 'b68-zeroaccess-2-32bit', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.geolocation.cc': 'HK', - 'destination.geolocation.city': 'HONG KONG', - 'destination.geolocation.region': 'HONG KONG', - 'destination.ip': '168.63.134.179', - 'destination.port': 16464, - 'extra.destination.naics': 334111, - 'extra.tag': 'b68-zeroaccess-2-32bit', - 'extra.infection': 'b68-zeroaccess-2-32bit', - 'extra.destination.sector': 'Information', - 'extra.public_source': 'MSDCU', - 'extra.source.naics': 517311, - 'extra.tag': 'b68-zeroaccess-2-32bit', - 'feed.name': 'ShadowServer Microsoft Sinkhole', - 'malware.name': 'zeroaccess', - 'protocol.transport': 'tcp', - 'source.asn': 7303, - 'source.geolocation.cc': 'AR', - 'source.geolocation.city': 'CASEROS', - 'source.geolocation.region': 'BUENOS AIRES', - 'source.ip': '190.229.1.2', - 'source.port': 52955, - 'time.source': '2021-06-07T00:00:00+00:00', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - }, - {'__type': 'Event', - 'classification.identifier': 'b68-zeroaccess-2-32bit', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.geolocation.cc': 'IE', - 'destination.geolocation.city': 'DUBLIN', - 'destination.geolocation.region': 'DUBLIN', - 'destination.ip': '52.169.3.4', - 'destination.port': 16464, - 'extra.destination.naics': 334111, - 'extra.tag': 'b68-zeroaccess-2-32bit', - 'extra.infection': 'b68-zeroaccess-2-32bit', - 'extra.destination.sector': 'Information', - 'extra.public_source': 'MSDCU', - 'extra.source.naics': 517311, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'feed.name': 'ShadowServer Microsoft Sinkhole', - 'malware.name': 'zeroaccess', - 'protocol.transport': 'tcp', - 'source.asn': 5769, - 'source.geolocation.cc': 'CA', - 'source.geolocation.city': 'LAVAL', - 'source.geolocation.region': 'QUEBEC', - 'source.ip': '96.20.3.4', - 'source.port': 16464, - 'time.source': '2021-06-07T00:00:00+00:00', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - }, - {'__type': 'Event', - 'classification.identifier': 'b68-zeroaccess-2-32bit', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.geolocation.cc': 'HK', - 'destination.geolocation.city': 'HONG KONG', - 'destination.geolocation.region': 'HONG KONG', - 'destination.ip': '168.63.134.179', - 'destination.port': 16464, - 'extra.tag': 'b68-zeroaccess-2-32bit', - 'extra.infection': 'b68-zeroaccess-2-32bit', - 'extra.destination.naics': 334111, - 'extra.destination.sector': 'Information', - 'extra.public_source': 'MSDCU', - 'extra.source.naics': 517311, - 'feed.name': 'ShadowServer Microsoft Sinkhole', - 'malware.name': 'zeroaccess', - 'protocol.transport': 'tcp', - 'source.asn': 8151, - 'source.geolocation.cc': 'MX', - 'source.geolocation.city': 'MEXICO CITY', - 'source.geolocation.region': "CIUDAD DE MEXICO", - 'source.ip': '187.222.5.6', - 'source.port': 55049, - 'time.source': '2021-06-07T00:00:00+00:00', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - }, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole_http.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole_http.py deleted file mode 100644 index 2f8c3d8e2e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole_http.py +++ /dev/null @@ -1,202 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_microsoft_sinkhole_http.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'HTTP Microsoft Sinkhole IPv4', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_microsoft_sinkhole_http.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'HTTP Microsoft Sinkhole IPv4', - 'classification.identifier': 'necurs', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'ASHBURN', - 'destination.geolocation.region': 'VIRGINIA', - 'destination.ip': '40.121.206.97', - 'destination.port': 80, - 'destination.url': 'http://40.121.206.97/locator.php', - 'extra.destination.naics': 334111, - 'extra.destination.sector': 'Information', - 'extra.public_source': 'MSDCU', - 'extra.infection': 'necurs', - 'extra.tag': 'necurs', - 'protocol.application': 'http', - 'malware.name': 'necurs', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 8386, - 'source.geolocation.cc': 'TR', - 'source.geolocation.city': 'KEPEZ', - 'source.geolocation.region': 'ANTALYA', - 'source.ip': '31.206.1.2', - 'source.port': 49245, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-06-07T00:00:00+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Microsoft Sinkhole IPv4', - 'classification.identifier': 'caphaw', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.fqdn': '3fo8jrthz3y.rgk.cc', - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'REDMOND', - 'destination.geolocation.region': 'WASHINGTON', - 'destination.ip': '204.95.99.204', - 'destination.port': 443, - 'destination.url': 'http://3fo8jrthz3y.rgk.cc/index.php', - 'protocol.application': 'http', - 'extra.infection': 'caphaw', - 'extra.tag': 'caphaw', - 'extra.destination.naics': 334111, - 'extra.destination.sector': 'Information', - 'extra.http_agent': 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.0.6103)', - 'extra.http_referer': 'null', - 'extra.public_source': 'MSDCU', - 'extra.source.naics': 517312, - 'malware.name': 'caphaw', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn': 28573, - 'source.geolocation.cc': 'BR', - 'source.geolocation.city': 'SAO PAULO', - 'source.geolocation.region': 'SAO PAULO', - 'source.ip': '177.140.3.4', - 'source.port': 35919, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-06-07T00:00:00+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Microsoft Sinkhole IPv4', - 'classification.identifier': 'necurs', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'ASHBURN', - 'destination.geolocation.region': 'VIRGINIA', - 'destination.ip': '40.121.206.97', - 'destination.port': 80, - 'destination.url': 'http://40.121.206.97/locator.php', - 'protocol.application': 'http', - 'extra.tag': 'necurs', - 'extra.infection': 'necurs', - 'extra.destination.naics': 334111, - 'extra.destination.sector': 'Information', - 'extra.public_source': 'MSDCU', - 'extra.source.naics': 517311, - 'malware.name': 'necurs', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn': 132199, - 'source.geolocation.cc': 'PH', - 'source.geolocation.city': 'MANDAUE', - 'source.geolocation.region': 'CEBU', - 'source.ip': '180.190.5.6', - 'source.port': 49264, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-06-07T00:00:01+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Microsoft Sinkhole IPv4', - 'classification.identifier': 'necurs', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.geolocation.cc': 'US', - 'destination.ip': '40.121.206.97', - 'destination.geolocation.city': 'ASHBURN', - 'destination.geolocation.region': 'VIRGINIA', - 'destination.port': 80, - 'destination.url': 'http://40.121.206.97/news/stream.php', - 'extra.destination.naics': 334111, - 'extra.destination.sector': 'Information', - 'extra.public_source': 'MSDCU', - 'malware.name': 'necurs', - 'extra.tag': 'necurs', - 'extra.infection': 'necurs', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[4]])), - 'source.asn': 37129, - 'source.geolocation.cc': 'KE', - 'source.geolocation.city': 'NAIROBI', - 'source.geolocation.region': 'NAIROBI CITY', - 'source.ip': '197.157.7.8', - 'source.port': 55307, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-06-07T00:00:01+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Microsoft Sinkhole IPv4', - 'classification.identifier': 'necurs', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'ASHBURN', - 'destination.geolocation.region': 'VIRGINIA', - 'destination.ip': '40.121.206.97', - 'destination.port': 80, - 'destination.url': 'http://40.121.206.97/locator.php', - 'extra.destination.naics': 334111, - 'extra.destination.sector': 'Information', - 'extra.public_source': 'MSDCU', - 'extra.source.naics': 517311, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'malware.name': 'necurs', - 'protocol.application': 'http', - 'extra.tag': 'necurs', - 'extra.infection': 'necurs', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[5]])), - 'source.asn': 812, - 'source.geolocation.cc': 'CA', - 'source.geolocation.city': 'OTTAWA', - 'source.geolocation.region': 'ONTARIO', - 'source.ip': '174.114.9.10', - 'source.port': 59000, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-06-07T00:00:01+00:00'}] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole.py deleted file mode 100644 index 2bb8aa6980..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole.py +++ /dev/null @@ -1,73 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_sinkhole.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "ShadowServer Sinkhole", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_sinkhole.csv", - } -EVENTS = [{'__type': 'Event', - 'classification.identifier': 'victorygate.b', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 28753, - 'destination.geolocation.cc': 'DE', - 'destination.geolocation.city': 'FRANKFURT AM MAIN', - 'destination.geolocation.region': 'HESSEN', - 'destination.ip': '178.162.1.2', - 'destination.port': 4455, - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.public_source': 'eset', - 'feed.name': 'ShadowServer Sinkhole', - 'malware.name': 'victorygate.b', - 'extra.infection': 'victorygate.b', - 'protocol.transport': 'tcp', - 'source.asn': 12252, - 'source.geolocation.cc': 'PE', - 'source.geolocation.city': 'LIMA', - 'source.geolocation.region': 'METROPOLITANA DE LIMA', - 'source.ip': '190.113.1.2', - 'source.port': 17409, - 'time.source': '2021-03-04T00:00:00+00:00', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - }, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_dns.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_dns.py deleted file mode 100644 index cf3bdb1623..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_dns.py +++ /dev/null @@ -1,127 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/event4_sinkhole_dns.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "Sinkhole DNS", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_sinkhole_dns-test-geo.csv", - } - -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'sinkholedns', - 'extra.tag' : 'msexchange', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.count' : 1, - 'extra.infection' : 'calypso', - 'extra.dns_query' : 'YolkIsh.COM', - 'extra.dns_query_type' : 'A', - 'extra.naics' : 518210, - 'extra.sector' : 'Communications, Service Provider, and Hosting Service', - 'feed.name' : 'Sinkhole DNS', - 'malware.name' : 'calypso', - 'protocol.application' : 'dns', - 'protocol.transport' : 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 8220, - 'source.geolocation.cc' : 'DE', - 'source.geolocation.city' : 'FRANKFURT AM MAIN', - 'source.geolocation.region' : 'HESSEN', - 'source.ip' : '217.110.0.0', - 'source.port' : 29614, - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2022-01-06T00:00:02+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'sinkholedns', - 'extra.tag' : 'rat', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.count' : 1, - 'extra.infection' : 'orcus', - 'extra.dns_query' : 'verble.rocks', - 'extra.dns_query_type' : 'A', - 'extra.naics' : 518210, - 'feed.name' : 'Sinkhole DNS', - 'malware.name' : 'orcus', - 'protocol.application' : 'dns', - 'protocol.transport' : 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 40934, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'ASHBURN', - 'source.geolocation.region' : 'VIRGINIA', - 'source.ip' : '209.66.0.0', - 'source.port' : 46189, - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2022-01-06T00:00:02+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'sinkholedns', - 'extra.tag' : 'msexchange', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.count' : 1, - 'extra.infection' : 'calypso', - 'extra.dns_query' : 'RAwFuNS.COM', - 'extra.dns_query_type' : 'A', - 'extra.naics' : 518210, - 'extra.sector' : 'Communications, Service Provider, and Hosting Service', - 'feed.name' : 'Sinkhole DNS', - 'malware.name' : 'calypso', - 'protocol.application' : 'dns', - 'protocol.transport' : 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 8220, - 'source.geolocation.cc' : 'DE', - 'source.geolocation.city' : 'FRANKFURT AM MAIN', - 'source.geolocation.region' : 'HESSEN', - 'source.ip' : '217.110.0.0', - 'source.port' : 3590, - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2022-01-06T00:00:02+00:00' -} -] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http.py deleted file mode 100644 index 60cd6b6efb..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http.py +++ /dev/null @@ -1,189 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_sinkhole_http.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'HTTP Sinkhole IPv4', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_sinkhole_http.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'HTTP Sinkhole IPv4', - 'classification.identifier': 'avalanche-andromeda', - 'extra.tag': 'avalanche-andromeda', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 6939, - 'destination.fqdn': 'differentia.ru', - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'FREMONT', - 'destination.geolocation.region': 'CALIFORNIA', - 'destination.ip': '184.105.1.2', - 'destination.port': 80, - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting Service', - 'malware.name': 'andromeda', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 134707, - 'source.geolocation.cc': 'PH', - 'source.geolocation.city': 'DEL PILAR', - 'source.geolocation.region': 'NUEVA ECIJA', - 'source.ip': '103.196.1.2', - 'source.port': 60902, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-03-04T00:00:00+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Sinkhole IPv4', - 'classification.identifier': 'avalanche-andromeda', - 'extra.tag': 'avalanche-andromeda', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 6939, - 'destination.fqdn': 'differentia.ru', - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'FREMONT', - 'destination.geolocation.region': 'CALIFORNIA', - 'destination.ip': '184.105.3.4', - 'destination.port': 80, - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting Service', - 'malware.name': 'andromeda', - 'extra.source.naics': 517311, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn': 8708, - 'source.geolocation.cc': 'RO', - 'source.geolocation.city': 'CONSTANTA', - 'source.geolocation.region': 'CONSTANTA', - 'source.ip': '5.14.3.4', - 'source.port': 55002, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-03-04T00:00:00+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Sinkhole IPv4', - 'classification.identifier': 'avalanche-andromeda', - 'extra.tag': 'avalanche-andromeda', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 6939, - 'destination.fqdn': 'disorderstatus.ru', - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'FREMONT', - 'destination.geolocation.region': 'CALIFORNIA', - 'destination.ip': '184.105.5.6', - 'destination.port': 80, - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.source.naics': 517311, - 'malware.name': 'andromeda', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn': 9299, - 'source.geolocation.cc': 'PH', - 'source.geolocation.city': 'CEBU', - 'source.geolocation.region': 'CEBU', - 'source.ip': '49.145.5.6', - 'source.port': 31350, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-03-04T00:00:00+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Sinkhole IPv4', - 'classification.identifier': 'avalanche-andromeda', - 'extra.tag': 'avalanche-andromeda', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 6939, - 'destination.fqdn': 'differentia.ru', - 'destination.geolocation.cc': 'US', - 'destination.ip': '184.105.7.8', - 'destination.geolocation.city': 'FREMONT', - 'destination.geolocation.region': 'CALIFORNIA', - 'destination.port': 80, - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.source.naics': 517311, - 'malware.name': 'andromeda', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[4]])), - 'source.asn': 8048, - 'source.geolocation.cc': 'VE', - 'source.geolocation.city': 'VALENCIA', - 'source.geolocation.region': 'CARABOBO', - 'source.ip': '200.44.7.8', - 'source.port': 28063, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-03-04T00:00:00+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Sinkhole IPv4', - 'classification.identifier': 'avalanche-andromeda', - 'extra.tag': 'avalanche-andromeda', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 6939, - 'destination.fqdn': 'differentia.ru', - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'FREMONT', - 'destination.geolocation.region': 'CALIFORNIA', - 'destination.ip': '184.105.9.10', - 'destination.port': 80, - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting Service', - 'malware.name': 'andromeda', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[5]])), - 'source.asn': 17072, - 'source.geolocation.cc': 'MX', - 'source.geolocation.city': 'JUAREZ', - 'source.geolocation.region': 'CHIHUAHUA', - 'source.ip': '187.189.9.10', - 'source.port': 45335, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-03-04T00:00:00+00:00'}] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http_referer.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http_referer.py deleted file mode 100644 index b1ccacd311..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http_referer.py +++ /dev/null @@ -1,213 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Mikk Margus Möll -# -# SPDX-License-Identifier: AGPL-3.0-or-later -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_sinkhole_http_referer.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Sinkhole-Events-HTTP-Referer IPv4', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2021-03-05T00:00:00+00:00", - "extra.file_name": "2021-03-04-event4_sinkhole_http_referer.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Sinkhole-Events-HTTP-Referer IPv4', - 'classification.identifier': 'sinkhole-http-referer', - 'extra.tag': 'kovter', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'destination.asn': 60781, - 'destination.fqdn': '12106.mobapptrack.com', - 'destination.geolocation.cc': 'NL', - 'destination.geolocation.city': 'AMSTERDAM', - 'destination.geolocation.region': 'NOORD-HOLLAND', - 'destination.ip': '85.17.31.82', - 'destination.port': 80, - 'destination.url': 'http://12106.mobapptrack.com/favicon.ico', - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.event_id': '1614816002', - 'malware.name': 'kovter', - 'extra.http_referer': 'http://12106.mobapptrack.com/click/redirect?feed_id=12106&sub_id=7&q=8A5491983C8FBE7743E2D2C36E45EBC4-18307118D2626C9BD756B3F09D14BB910E381EE4', - 'extra.http_referer_asn': 28753, - 'extra.http_referer_city': 'FRANKFURT AM MAIN', - 'extra.http_referer_geo': 'DE', - 'extra.http_referer_hostname': '12106.mobapptrack.com', - 'extra.http_referer_ip': '178.162.203.211', - 'extra.http_referer_naics': 518210, - 'extra.http_referer_port': 80, - 'extra.http_referer_region': 'HESSEN', - 'extra.http_referer_sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-03-04T00:00:02+00:00'}, - {'__type': 'Event', - 'classification.identifier': 'sinkhole-http-referer', - 'extra.tag': 'sunburst', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'destination.asn': 28753, - 'destination.fqdn': 'freescanonline.com', - 'destination.geolocation.cc': 'DE', - 'destination.geolocation.city': 'FRANKFURT AM MAIN', - 'destination.geolocation.region': 'HESSEN', - 'destination.port': 80, - 'destination.url': 'http://freescanonline.com/animalally.com', - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'destination.ip': '178.162.1.2', - 'extra.event_id': '1614816011', - 'malware.name': 'sunburst', - 'extra.http_referer': 'http://x.noizm.com/jump.php?u=http://freescanonline.com/animalally.com', - 'extra.http_referer_asn': 9370, - 'extra.http_referer_city': 'OSAKA', - 'extra.http_referer_geo': 'JP', - 'extra.http_referer_hostname': 'x.noizm.com', - 'extra.http_referer_naics': 518210, - 'extra.http_referer_port': 80, - 'extra.http_referer_ip': '59.106.1.2', - 'extra.http_referer_region': 'OSAKA', - 'extra.http_referer_sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'protocol.transport': 'tcp', - 'feed.name': 'Sinkhole-Events-HTTP-Referer IPv4', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'time.source': '2021-03-04T00:00:11+00:00'}, - {'__type': 'Event', - 'classification.identifier': 'sinkhole-http-referer', - 'extra.tag': 'kovter', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'destination.asn': 28753, - 'destination.fqdn': 'rxrtb.bid', - 'destination.geolocation.cc': 'DE', - 'destination.geolocation.city': 'FRANKFURT AM MAIN', - 'destination.geolocation.region': 'HESSEN', - 'destination.port': 80, - 'destination.url': 'http://rxrtb.bid/getjs?r=0.6393021999392658', - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'destination.ip': '178.162.1.2', - 'extra.event_id': '1614816012', - 'malware.name': 'kovter', - 'extra.http_referer': 'http://x.blogspot.com/', - 'extra.http_referer_ip': '142.250.3.4', - 'extra.http_referer_asn': 15169, - 'extra.http_referer_city': 'MOUNTAIN VIEW', - 'extra.http_referer_geo': 'US', - 'extra.http_referer_hostname': 'x.blogspot.com', - 'extra.http_referer_naics': 519130, - 'extra.http_referer_port': 80, - 'extra.http_referer_region': 'CALIFORNIA', - 'extra.http_referer_sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'protocol.transport': 'tcp', - 'feed.name': 'Sinkhole-Events-HTTP-Referer IPv4', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'time.source': '2021-03-04T00:00:12+00:00'}, - {'__type': 'Event', - 'classification.identifier': 'sinkhole-http-referer', - 'extra.tag': 'sunburst', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'destination.asn': 60781, - 'destination.fqdn': 'freescanonline.com', - 'destination.geolocation.cc': 'NL', - 'destination.geolocation.city': 'AMSTERDAM', - 'destination.geolocation.region': 'NOORD-HOLLAND', - 'destination.ip': '5.79.71.225', - 'destination.port': 80, - 'destination.url': 'http://freescanonline.com/personalationmall.com', - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'extra.event_id': '1614816013', - 'malware.name': 'sunburst', - 'extra.http_referer': 'http://www.example.com/teams/default.asp?u=EKL&t=c&s=lacrosse&p=remote&url=http://freescanonline.com/personalationmall.com', - 'extra.http_referer_asn': 14618, - 'extra.http_referer_city': 'ASHBURN', - 'extra.http_referer_geo': 'US', - 'extra.http_referer_hostname': 'www.example.com', - 'extra.http_referer_ip': '34.232.5.6', - 'extra.http_referer_naics': 454110, - 'extra.http_referer_port': 80, - 'extra.http_referer_region': 'VIRGINIA', - 'extra.http_referer_sector': 'Retail Trade', - 'protocol.transport': 'tcp', - 'feed.name': 'Sinkhole-Events-HTTP-Referer IPv4', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[4]])), - 'time.source': '2021-03-04T00:00:13+00:00'}, - {'__type': 'Event', - 'classification.identifier': 'sinkhole-http-referer', - 'extra.tag': 'sunburst', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'destination.asn': 60781, - 'destination.fqdn': 'freescanonline.com', - 'destination.geolocation.cc': 'NL', - 'destination.geolocation.city': 'AMSTERDAM', - 'destination.geolocation.region': 'NOORD-HOLLAND', - 'destination.port': 80, - 'destination.url': 'http://freescanonline.com/raftcomply.com', - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'destination.ip': '5.79.1.2', - 'extra.event_id': '1614816086', - 'malware.name': 'sunburst', - 'extra.http_referer': 'http://x.communes.jp/?url=http://freescanonline.com/raftcomply.com', - 'extra.http_referer_asn': 2516, - 'extra.http_referer_city': 'SAPPORO', - 'extra.http_referer_geo': 'JP', - 'extra.http_referer_hostname': 'x.communes.jp', - 'extra.http_referer_ip': '210.172.7.8', - 'extra.http_referer_naics': 517312, - 'extra.http_referer_port': 80, - 'extra.http_referer_region': 'HOKKAIDO', - 'extra.http_referer_sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'protocol.transport': 'tcp', - 'feed.name': 'Sinkhole-Events-HTTP-Referer IPv4', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[5]])), - 'time.source': '2021-03-04T00:01:26+00:00'}] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event6_sinkhole_http.py b/intelmq/tests/bots/parsers/shadowserver/test_event6_sinkhole_http.py deleted file mode 100644 index d6ff35dc11..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event6_sinkhole_http.py +++ /dev/null @@ -1,146 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/event6_sinkhole_http.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "Sinkhole-Events-HTTP IPv6", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-event6_sinkhole_http-test-geo.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : '3ve', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'infected-system', - 'destination.asn' : 6939, - 'destination.fqdn' : 'devps.net', - 'destination.geolocation.cc' : 'US', - 'destination.geolocation.city' : 'FREMONT', - 'destination.geolocation.region' : 'CALIFORNIA', - 'destination.ip' : '2001:470:1:332::fe', - 'destination.port' : 80, - 'destination.url' : 'http://devps.net/QKMSvF2hl11j%2fbMkyPbF5EpHYhd6VWTG4u19K3Rt7JGU3lMYRqpq8wPYEuOGKKeidKW3pefVfKSjBnL0cXizZbmuWWu8AQNRqw5g9Ny5vZtiv638XKoWwCLuUOTISTV%2fLcpcS1%2f22NjWqgXkHGISAuyVtafqyCC%2f5cA0eYg9Me8VzAIFDdTArogQOdYhElf2xluhEFPsstGQ%2bwrM4VmKHJpzyjD7Y%2fN%2bQV3wnZNdVkEVk1k2iKBJkotYv3ajgYWr56xxCbY5vE1IpZBRNhhaUDNZo0kJgi%2b6knXZ4m7JHt%2fGtJeP%2bNTxHSUL2ELlTIiT3ENlPYD6FdH6ZBxT1OneW%2f0ih%2fcN7vctb5B5Qwa1ez7ZjN2QxgBYkFDDHHTs42ej5eF2BysWAQDSUr%2fcySyGxcfPveIpfQEdrynGKR6z3OYqkFnP%2bYRDQp2rt1qt0FwCB4L9cg05TQlSSTJVGfPDrtcqjvKY4c9hWwSHtE8jMRpeCYO4Es%2bWgwr5DjzMicmuZo%2f4Ycr16jpN7xlDJdJ8iCFZxbSGgVC7ksVlGE8wlfWPI4KTuX5U5s61eNWPTlAC%2fOGb8grtw%2ffzizoIX9D6ZUMvslGLQIp%2fvNmNQkZy8HhNoV6Lns%2figITP%2fpN0H8h9HjUTl9qn65xFOEVpc0motSy8alcTPtTRKq5Jvc4Ao0x3N%2fvCB1v4Epx7XC0UpFbw8TrYEvAczEfGsGM', - 'extra.destination.naics' : 518210, - 'extra.destination.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.http_agent' : 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET4.0C; .NET4.0E)', - 'extra.infection' : 'boaxxe', - 'extra.tag' : '3ve', - 'feed.name' : 'Sinkhole-Events-HTTP IPv6', - 'malware.name' : 'boaxxe', - 'protocol.application' : 'http', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 7713, - 'source.geolocation.cc' : 'ID', - 'source.geolocation.city' : 'JAKARTA', - 'source.geolocation.region' : 'JAKARTA RAYA', - 'source.ip' : '2001:448a:1082:4d9b:7491:bf9e:3d5f:a634', - 'source.port' : 49431, - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2022-03-02T09:14:19+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : '3ve', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'infected-system', - 'destination.asn' : 6939, - 'destination.fqdn' : 'devps.net', - 'destination.geolocation.cc' : 'US', - 'destination.geolocation.city' : 'FREMONT', - 'destination.geolocation.region' : 'CALIFORNIA', - 'destination.ip' : '2001:470:1:332::ef', - 'destination.port' : 80, - 'destination.url' : 'http://devps.net/QKMSvF2hl11j%2fbMkyPbF5EpHYhd6VWTG4u19K3Rt7JGU3lMYRqpq8wPYEuOGKKeidKW3pefVfKSjBnL0cXizZbmuWWu8AQNRqw5g9Ny5vZtiv638XKoWwCLuUOTISTV%2fLcpcS1%2f22NjWqgXkHGISAuyVtafqyCC%2f5cA0eYg9Me8VzAIFDdTArogQOdYhElf2xluhEFPsstGQ%2bwrM4VmKHJpzyjD7Y%2fN%2bQV3wnZNdVkEVk1k2iKBJkotYv3ajgYWr56xxCbY5vE1IpZBRNhhaUDNZo0kJgi%2b6knXZ4m7JHt%2fGtJeP%2bNTxHSUL2ELlTIiT3ENlPYD6FdH6ZBxT1OneW%2f0ih%2fcN7vctb5B5Qwa1ez7ZjN2QxgBYkFDDHHTs42ej5eF2BysWAQDSUr%2fcySyGxcfPveIpfQEdrynGKR6z3OYqkFnP%2bYRDQp2rt1qt0FwCB4L9cg05TQlSSTJVGfPDrtcqjvKY4c9hWwSHtE8jMRpeCYO4Es%2bWgwr5DjzMicmuZo%2f4Ycr16jpN7xlDJdJ8iCFZxbSGgVC7ksVlGE8wlfWPI4KTuX5U5s61eNWPTlAC%2fOGb8grtw%2ffzizoIX9D6ZUMvslGLQIp%2fvNmNQkZy8HhNoV6Lns%2figITP%2fpN0H8h9HjUTl9qn65xFOEVpc0motSy8alcTPtTRKq5Jvc4Ao0x3N%2fvCB1v4Epx7XC0UpFbw8TrYEvAczEfGsGM', - 'extra.destination.naics' : 518210, - 'extra.destination.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.http_agent' : 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET4.0C; .NET4.0E)', - 'extra.infection' : 'boaxxe', - 'extra.tag' : '3ve', - 'feed.name' : 'Sinkhole-Events-HTTP IPv6', - 'malware.name' : 'boaxxe', - 'protocol.application' : 'http', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 7713, - 'source.geolocation.cc' : 'ID', - 'source.geolocation.city' : 'JAKARTA', - 'source.geolocation.region' : 'JAKARTA RAYA', - 'source.ip' : '2001:448a:1082:4d9b:7491:bf9e:3d5f:a634', - 'source.port' : 49460, - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2022-03-02T09:15:10+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : '3ve', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'infected-system', - 'destination.asn' : 6939, - 'destination.fqdn' : 'devps.net', - 'destination.geolocation.cc' : 'US', - 'destination.geolocation.city' : 'FREMONT', - 'destination.geolocation.region' : 'CALIFORNIA', - 'destination.ip' : '2001:470:1:332::fe', - 'destination.port' : 80, - 'destination.url' : 'http://devps.net/WMoUNCvuKGzdqSCeQcadP1%2f0B%2f3bzpOmyKBU85Z25HVOhvDQUPFl%2fk8uOcLewS%2b1BsuHXalRAOIgGOYYs2igj6UX8FkdCAmDewWPvfDhPD45nwd2tx1lLf2IoIfuOtIpGR6bN5Q6hGpSBgfERqCa0ImHcwfcZ2EdO%2fWvg7R8H6SLcTiuUC0I4pzvlWt1CRLgLdIEU1hZ0nnFHIHhchb6D7ITEgBQ2chQDxy5TJMrGjm4Dac6dKl%2ft5uYhRhSjAHkLLtgrJjsqVtVbelTAkt5kdcqLlO09m1SH%2fvtAb%2fOvR2DbhBss7%2f64DG7g6cAnghNA6JrFn1uW7sw%2bnKH8koKQwzUjdSsbrQAvmg4r0KDDW8Diq64gfDzxFWkzCLOYifc%2fwlinXPCl7aJiNCoieDC1U98RNQg%2f5td4SZmJnDQ2%2f96CPbFeSpCez5WD1rCjrxLj1h2cqzIgkydEWACceWP9ztxc4QaObzEcgOGxbRckWC7H2aaLeT8jaYEYdKi1pwEKChSL3YdEt4ZIb2IFrWwzNaXEpQzFXf07f902OEdI9vVA1ZdEOBPG6rAIkzMdebfprfVyhKEWtrCd3Skg3COUFtRQks5jzG1nv4sVGijTfSgyn6xE9Taka668Nycik6nmHy8Huj3oC01j3tee%2f1Z3eI6tV7lgM5d3uFJ84slRGHUCwMfVozOGmZRwNo%2fz%2bA', - 'extra.destination.naics' : 518210, - 'extra.destination.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.http_agent' : 'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko', - 'extra.infection' : 'boaxxe', - 'extra.source.naics' : 517311, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : '3ve', - 'feed.name' : 'Sinkhole-Events-HTTP IPv6', - 'malware.name' : 'boaxxe', - 'protocol.application' : 'http', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 11427, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'GARLAND', - 'source.geolocation.region' : 'TEXAS', - 'source.ip' : '2603:8080:b20a:dc00:f06e:8304:71f6:27e2', - 'source.port' : 62932, - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2022-03-02T14:15:10+00:00' -} - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_honeypot_brute_force.py b/intelmq/tests/bots/parsers/shadowserver/test_honeypot_brute_force.py deleted file mode 100644 index c376a73fbd..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_honeypot_brute_force.py +++ /dev/null @@ -1,72 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/event4_honeypot_brute_force.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Honeypot-Brute-Force-Events', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_honeypot_brute_force.csv" - } -EVENTS = [{'__type': 'Event', - 'classification.identifier': 'ssh', - 'classification.taxonomy': 'intrusion-attempts', - 'classification.type': 'brute-force', - 'extra.client_version': "b'SSH-2.0-Go'", - 'destination.asn': 26832, - 'destination.geolocation.cc': 'CA', - 'destination.geolocation.city': 'MONTREAL', - 'destination.geolocation.region': 'QUEBEC', - 'destination.ip': '162.250.1.2', - 'destination.port': 22, - 'extra.application': 'ssh', - 'extra.end_time': '2021-03-27T00:00:01.710968+00:00', - 'extra.public_source': 'CAPRICA-EU', - 'extra.start_time': '2021-03-27T00:00:00.521730+00:00', - 'malware.name': 'ssh-brute-force', - 'feed.name': 'Honeypot-Brute-Force-Events', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 209588, - 'source.geolocation.cc': 'NL', - 'source.geolocation.city': 'AMSTERDAM', - 'source.geolocation.region': 'NOORD-HOLLAND', - 'source.ip': '141.98.1.2', - 'source.port': 30123, - 'time.source': '2021-03-27T00:00:00+00:00'}, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_honeypot_ddos_amp.py b/intelmq/tests/bots/parsers/shadowserver/test_honeypot_ddos_amp.py deleted file mode 100644 index e95e59dcb3..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_honeypot_ddos_amp.py +++ /dev/null @@ -1,91 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/event4_honeypot_ddos_amp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Amplification DDoS Victim', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2019-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_honeypot_ddos_amp.csv" - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Amplification DDoS Victim', - 'classification.identifier': 'amplification-ddos-victim', - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'time.observation': '2019-01-01T00:00:00+00:00', - 'time.source': '2021-03-28T00:00:02+00:00', - 'source.ip': '107.141.1.2', - 'destination.port': 389, - 'source.reverse_dns': '192-0-2-10.example.net', - 'source.asn': 7018, - 'source.geolocation.cc': 'US', - 'source.geolocation.region': 'VISALIA', - 'source.geolocation.city': 'VISALIA', - 'source.geolocation.region': 'CALIFORNIA', - 'extra.end_time': '2021-03-28T00:20:22+00:00', - 'extra.public_source': 'CISPA', - 'extra.source.naics': 517311, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'malware.name': 'ddos-amplification', - 'source.reverse_dns': '107-141-x-x.lightspeed.frsnca.sbcglobal.net', - }, - {'__type': 'Event', - 'feed.name': 'Amplification DDoS Victim', - 'classification.identifier': 'amplification-ddos-victim', - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'time.observation': '2019-01-01T00:00:00+00:00', - 'time.source': '2021-03-28T00:00:02+00:00', - 'source.ip': '74.59.3.4', - 'destination.port': 389, - 'source.reverse_dns': 'modemcablex-x-59-74.mc.videotron.ca', - 'source.asn': 5769, - 'source.geolocation.cc': 'CA', - 'source.geolocation.city': 'CHICOUTIMI', - 'source.geolocation.region': 'QUEBEC', - 'extra.end_time': '2021-03-28T00:13:50+00:00', - 'extra.public_source': 'CISPA', - 'extra.source.naics': 517311, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'malware.name': 'ddos-amplification', - }, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_malware_url.py b/intelmq/tests/bots/parsers/shadowserver/test_malware_url.py deleted file mode 100644 index b19b200b5f..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_malware_url.py +++ /dev/null @@ -1,107 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/malware_url.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Malware URL', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-malware_url-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'malware-url', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'extra.application' : 'http', - 'source.url' : 'http://41.86.0.0:50008/Mozi.m', - 'feed.name' : 'Malware URL', - 'malware.hash.sha256' : '12013662c71da69de977c04cd7021f13a70cf7bed4ca6c82acbc100464d4b0ef', - 'malware.name' : 'cve-2016-10372', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 37203, - 'source.geolocation.cc' : 'LR', - 'source.geolocation.city' : 'MONROVIA', - 'source.geolocation.region' : 'MONTSERRADO', - 'source.ip' : '41.86.0.0', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-07T00:02:07+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'malware-url', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'extra.application' : 'http', - 'extra.source.naics' : 517311, - 'source.url' : 'http://42.225.0.0:38173/Mozi.m', - 'feed.name' : 'Malware URL', - 'malware.name' : 'cve-2018-10562', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 4837, - 'source.geolocation.cc' : 'CN', - 'source.geolocation.city' : 'ZHUMADIAN', - 'source.geolocation.region' : 'HENAN SHENG', - 'source.ip' : '42.225.0.0', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-07T00:03:14+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'malware-url', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'extra.application' : 'http', - 'extra.source.naics' : 517311, - 'source.url' : 'http://211.52.0.0:53029/Mozi.m', - 'feed.name' : 'Malware URL', - 'malware.name' : 'cve-2018-10562', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 4766, - 'source.geolocation.cc' : 'KR', - 'source.geolocation.city' : 'SAGOK-MYEON', - 'source.geolocation.region' : 'CHUNGCHEONGNAM-DO', - 'source.ip' : '211.52.0.0', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-07T00:10:26+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_phish_url.py b/intelmq/tests/bots/parsers/shadowserver/test_phish_url.py deleted file mode 100644 index 0783372f91..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_phish_url.py +++ /dev/null @@ -1,106 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/phish_url.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Phish URL', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-phish_url-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'phish-url', - 'classification.taxonomy' : 'fraud', - 'classification.type' : 'phishing', - 'source.fqdn' : 'priceless-pare.example.net', - 'extra.source' : 'openphish.com', - 'extra.source.naics' : 518210, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'source.url' : 'https://priceless-pare.example.net/Postal-/acec6/', - 'feed.name' : 'Phish URL', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'BUFFALO', - 'source.geolocation.region' : 'NEW YORK', - 'source.ip' : '172.245.0.0', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-02-01T08:00:07+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'phish-url', - 'classification.taxonomy' : 'fraud', - 'classification.type' : 'phishing', - 'source.fqdn' : 'mailyahooattt.example.net', - 'extra.source' : 'openphish.com', - 'extra.source.sector' : 'Professional, Scientific, and Technical Services', - 'source.url' : 'https://mailyahooattt.example.net/', - 'feed.name' : 'Phish URL', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'SAN FRANCISCO', - 'source.geolocation.region' : 'CALIFORNIA', - 'source.ip' : '199.34.0.0', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-02-01T08:00:07+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'phish-url', - 'classification.taxonomy' : 'fraud', - 'classification.type' : 'phishing', - 'source.fqdn' : 'www.example.net', - 'extra.source' : 'openphish.com', - 'extra.source.naics' : 519130, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'source.url' : 'https://www.example.net/viewer/vbid-730ec2b1-omsttuer', - 'feed.name' : 'Phish URL', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'DRAPER', - 'source.geolocation.region' : 'UTAH', - 'source.ip' : '216.58.0.0', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-02-01T08:00:07+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_population_http_proxy.py b/intelmq/tests/bots/parsers/shadowserver/test_population_http_proxy.py deleted file mode 100644 index e9f11a47c3..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_population_http_proxy.py +++ /dev/null @@ -1,130 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/population_http_proxy.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible HTTP Proxy', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-population_http_proxy-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'accessible-http-proxy', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'extra.connection': 'keep-alive', - 'extra.content_length': 3741, - 'extra.content_type': 'text/html;charset=utf-8', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 407, - 'extra.http_date': '2010-02-10T00:00:00+00:00', - 'extra.http_reason': 'Proxy Authentication Required', - 'extra.proxy_authenticate': 'Basic realm=\\\\"Squid proxy-caching web ' - 'server\\"\\""', - 'extra.server': 'squid/4.10', - 'feed.name': 'Accessible HTTP Proxy', - 'malware.name': 'http-connect-proxy-closed', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 3128, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-http-proxy', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'extra.connection': 'keep-alive', - 'extra.content_length': 3833, - 'extra.content_type': 'text/html;charset=utf-8', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 407, - 'extra.http_date': '2010-02-10T00:00:01+00:00', - 'extra.http_reason': 'Proxy Authentication Required', - 'extra.proxy_authenticate': 'Basic realm=\\\\"00:23:24:43:1c:34\\"\\""', - 'feed.name': 'Accessible HTTP Proxy', - 'malware.name': 'http-connect-proxy-closed', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 3128, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-http-proxy', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'extra.connection': 'keep-alive', - 'extra.content_length': 179, - 'extra.content_type': 'text/html;charset=utf-8', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 407, - 'extra.http_date': '2010-02-10T00:00:02+00:00', - 'extra.http_reason': 'Proxy Authentication Required', - 'extra.proxy_authenticate': 'Basic realm=\\\\"Proxy\\"\\""', - 'feed.name': 'Accessible HTTP Proxy', - 'malware.name': 'http-connect-proxy-closed', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 3128, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_sandbox_conn.py b/intelmq/tests/bots/parsers/shadowserver/test_sandbox_conn.py deleted file mode 100644 index c5da823465..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_sandbox_conn.py +++ /dev/null @@ -1,99 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/sandbox_conn.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Sandbox Connections', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-sandbox_conn-test.csv", - } -EVENTS = [ - { - '__type' : 'Event', - 'classification.identifier' : 'sandbox-conn', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'destination.fqdn' : 'time.windows.com', - 'feed.name' : 'Sandbox Connections', - 'malware.hash.md5' : 'b575ce6dcce6502a8431db5610135c25', - 'protocol.transport' : 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 8075, - 'source.geolocation.cc' : 'US', - 'source.ip' : '40.119.6.228', - 'source.port' : 123, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:00:03+00:00' -}, - { - '__type' : 'Event', - 'classification.identifier' : 'sandbox-conn', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'feed.name' : 'Sandbox Connections', - 'malware.hash.md5' : 'c0d947f9a8685b0d9f3efdba966389c2', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 3356, - 'source.geolocation.cc' : 'US', - 'source.ip' : '8.252.70.126', - 'source.port' : 80, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:00:03+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'sandbox-conn', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'feed.name' : 'Sandbox Connections', - 'malware.hash.md5' : 'c0d947f9a8685b0d9f3efdba966389c2', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 8075, - 'source.geolocation.cc' : 'US', - 'source.ip' : '52.109.8.22', - 'source.port' : 443, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:00:03+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_sandbox_dns.py b/intelmq/tests/bots/parsers/shadowserver/test_sandbox_dns.py deleted file mode 100644 index 70cf1eee5e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_sandbox_dns.py +++ /dev/null @@ -1,95 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/sandbox_dns.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Sandbox DNS', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-sandbox_dns-test.csv", - } -EVENTS = [ - { - '__type' : 'Event', - 'classification.identifier' : 'sandbox-dns', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.request' : 'time.windows.com', - 'extra.response' : '40.119.6.228', - 'extra.dns_query_type' : 'A', - 'feed.name' : 'Sandbox DNS', - 'malware.hash.md5' : 'b575ce6dcce6502a8431db5610135c25', - 'protocol.application' : 'dns', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:00:02+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'sandbox-dns', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.request' : 'time.windows.com', - 'extra.response' : '40.119.6.228', - 'extra.dns_query_type' : 'A', - 'feed.name' : 'Sandbox DNS', - 'malware.hash.md5' : '807679198a39c80d3ca07e60fd51b581', - 'protocol.application' : 'dns', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:00:08+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'sandbox-dns', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.request' : 'client-office365-tas.msedge.net', - 'extra.response' : '13.107.5.88', - 'extra.dns_query_type' : 'A', - 'feed.name' : 'Sandbox DNS', - 'malware.hash.md5' : 'd97e973b9bf073bd3a217425259cea26', - 'protocol.application' : 'dns', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:00:20+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_sandbox_url.py b/intelmq/tests/bots/parsers/shadowserver/test_sandbox_url.py deleted file mode 100644 index 91b0154b84..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_sandbox_url.py +++ /dev/null @@ -1,104 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/sandbox_url.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Sandbox URL', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-sandbox_url-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'sandbox-url', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'destination.fqdn' : 'www.msftncsi.com', - 'extra.http_request_method' : 'GET', - 'destination.url' : 'http://www.msftncsi.com/ncsi.txt', - 'extra.user_agent' : 'Microsoft NCSI', - 'feed.name' : 'Sandbox URL', - 'malware.hash.md5' : '37514b54e679a5313334e830ad780ec7', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 20940, - 'source.geolocation.cc' : 'US', - 'source.ip' : '23.196.47.89', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:13+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'sandbox-url', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'destination.fqdn' : 'www.download.windowsupdate.com', - 'extra.http_request_method' : 'GET', - 'destination.url' : 'http://www.download.windowsupdate.com/msdownload/update/v3/static/trustedr/en/authrootstl.cab', - 'extra.user_agent' : 'Microsoft-CryptoAPI/6.1', - 'feed.name' : 'Sandbox URL', - 'malware.hash.md5' : '37514b54e679a5313334e830ad780ec7', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 15133, - 'source.geolocation.cc' : 'US', - 'source.ip' : '72.21.81.240', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:28+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'sandbox-url', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'destination.fqdn' : 'crl.microsoft.com', - 'extra.http_request_method' : 'GET', - 'destination.url' : 'http://crl.microsoft.com/pki/crl/products/MicTimStaPCA_2010-07-01.crl', - 'extra.user_agent' : 'Microsoft-CryptoAPI/6.1', - 'feed.name' : 'Sandbox URL', - 'malware.hash.md5' : 'e97ea2820c0d79f3f3ca241d4dcd1060', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 20940, - 'source.geolocation.cc' : 'US', - 'source.ip' : '23.56.4.57', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:08:24+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_adb.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_adb.py deleted file mode 100644 index 6bc6e61461..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_adb.py +++ /dev/null @@ -1,98 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_adb.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible ADB', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2018-07-30T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_adb-test-test.csv", - - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Accessible ADB', - 'time.observation': '2018-07-30T00:00:00+00:00', - 'time.source': '2018-07-26T02:07:16+00:00', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-adb', - 'protocol.application': 'adb', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 3462, - 'source.geolocation.cc': 'TW', - 'source.geolocation.city': 'TAOYUAN CITY', - 'source.geolocation.region': 'TAOYUAN COUNTY', - 'source.ip': '36.239.124.210', - 'source.port': 5555, - 'extra.name': 'hlteuc', - 'extra.model': 'SAMSUNG-SM-N900A', - 'extra.device': 'hlteatt', - 'extra.tag': 'adb', - 'extra.naics': 518210, - 'extra.sic': 737415, - 'source.reverse_dns': '36-239-124-210.dynamic-ip.hinet.net', - }, - {'__type': 'Event', - 'feed.name': 'Accessible ADB', - 'time.observation': '2018-07-30T00:00:00+00:00', - 'time.source': '2018-07-26T02:07:16+00:00', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-adb', - 'protocol.application': 'adb', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn': 3462, - 'source.geolocation.cc': 'TW', - 'source.geolocation.city': 'TAIPEI', - 'source.geolocation.region': 'TAIPEI CITY', - 'source.ip': '36.236.108.107', - 'source.port': 5555, - 'extra.name': 'marlin', - 'extra.model': 'Pixel XL', - 'extra.device': 'marlin', - 'extra.features': 'cmd,shell_v2', - 'extra.naics': 518210, - 'extra.sic': 737415, - 'extra.tag': 'adb', - 'source.reverse_dns': '36-236-108-107.dynamic-ip.hinet.net', - }, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_afp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_afp.py deleted file mode 100644 index cc30b1e4c0..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_afp.py +++ /dev/null @@ -1,106 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_afp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible AFP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2018-07-30T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_afp-test-test.csv", - - } -EVENTS = [{ - '__type': 'Event', - 'feed.name': 'Accessible AFP', - "classification.identifier": "accessible-afp", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.afp_versions": "AFP3.3,AFP3.2,AFP3.1", - "extra.flags": "SupportsCopyFile,SupportsChgPwd,SupportsServerMessages,SupportsServerSignature,SupportsTCP/IP,SupportsSrvrNotifications,SupportsReconnect,SupportsOpenDirectory,SupportsUTF8Servername,SupportsUUIDs,SupportsSuperClient", - "extra.machine_type": "TimeCapsule8,119", - "extra.naics": 517311, - "extra.network_address": "198.33.24.165:548,10.0.1.1:548,fe80:0008:0000:0000:6e70:9fff:fed4::548,fe80:0009:0000:0000:6e70:9fff:fed4::548,179.24.24.165 (DNS address),", - "extra.server_name": "airport-time-capsule-de-jack", - "extra.signature": "4338364e37364442463948350069672d", - "extra.tag": "afp", - "extra.uams": "DHCAST128,DHX2,SRP,Recon1", - "extra.utf8_servername": "AirPort Time Capsule de jack", - "protocol.application": "afp", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 6057, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.13.34.22", - "source.port": 548, - "source.reverse_dns": "host.local", - "time.observation": "2018-07-30T00:00:00+00:00", - "time.source": "2019-09-04T05:05:53+00:00" -}, -{ - '__type': 'Event', - 'feed.name': 'Accessible AFP', - "classification.identifier": "accessible-afp", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.afp_versions": "AFP3.3,AFP3.2,AFP3.1", - "extra.flags": "SupportsCopyFile,SupportsChgPwd,SupportsServerMessages,SupportsServerSignature,SupportsTCP/IP,SupportsSrvrNotifications,SupportsReconnect,SupportsOpenDirectory,SupportsUTF8Servername,SupportsUUIDs,SupportsSuperClient", - "extra.machine_type": "TimeCapsule8,119", - "extra.naics": 517311, - "extra.network_address": "0.0.0.1:548,10.0.1.1:548,198.33.42.1:548,fe80:000b:0000:0000:dea4:caff:feba::548,fe80:000c:0000:0000:dea4:caff:feba::548,fe80:000d:0000:0000:4c7d:ffff:fec7::548,0.0.0.1 (DNS address),", - "extra.server_name": "time-capsule-del-jack", - "extra.signature": "433836544b303147463948360069672d", - "extra.tag": "afp", - "extra.uams": "DHCAST128,DHX2,SRP,Recon1", - "extra.utf8_servername": "Time Capsule del Jack", - "protocol.application": "afp", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 6057, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.40.27.212", - "source.port": 548, - "source.reverse_dns": "host.local", - "time.observation": "2018-07-30T00:00:00+00:00", - "time.source": "2019-09-04T05:05:56+00:00" - },] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_amqp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_amqp.py deleted file mode 100644 index df707f30b0..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_amqp.py +++ /dev/null @@ -1,144 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_amqp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible AMQP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_amqp-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'accessible-amqp', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.capabilities' : 'publisher_confirms,exchange_exchange_bindings,basic.nack,consumer_cancel_notify,connection.blocked,consumer_priorities,authentication_failure_close,per_consumer_qos', - 'extra.class' : '10', - 'extra.cluster_name' : 'rabbit@iZuf63m0nnq9bwf7lhjxrkZ', - 'extra.locales' : 'en_US', - 'extra.mechanisms' : 'PLAIN AMQPLAIN', - 'extra.message_length' : 509, - 'extra.method' : '10', - 'extra.platform' : 'Erlang/OTP', - 'extra.product' : 'RabbitMQ', - 'extra.product_version' : '3.3.5', - 'extra.naics' : 518210, - 'extra.tag' : 'amqp', - 'extra.version_minor' : '9', - 'feed.name' : 'Accessible AMQP', - 'protocol.application' : 'amqp', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 37963, - 'source.geolocation.cc' : 'CN', - 'source.geolocation.city' : 'SHANGHAI', - 'source.geolocation.region' : 'SHANGHAI SHI', - 'source.ip' : '47.103.0.0', - 'source.port' : 5672, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T04:32:13+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'accessible-amqp', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.capabilities' : 'publisher_confirms,exchange_exchange_bindings,basic.nack,consumer_cancel_notify,connection.blocked,consumer_priorities,authentication_failure_close,per_consumer_qos,direct_reply_to', - 'extra.class' : '10', - 'extra.cluster_name' : 'rabbit@mtk-breizh', - 'extra.locales' : 'en_US', - 'extra.mechanisms' : 'AMQPLAIN PLAIN', - 'extra.message_length' : 509, - 'extra.method' : '10', - 'extra.platform' : 'Erlang/OTP 24.0.3', - 'extra.product' : 'RabbitMQ', - 'extra.product_version' : '3.8.19', - 'extra.naics' : 518210, - 'extra.tag' : 'amqp', - 'extra.version_minor' : '9', - 'feed.name' : 'Accessible AMQP', - 'protocol.application' : 'amqp', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 16276, - 'source.geolocation.cc' : 'DE', - 'source.geolocation.city' : 'SAARBRUCKEN', - 'source.geolocation.region' : 'SAARLAND', - 'source.ip' : '141.95.0.0', - 'source.port' : 5672, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T04:32:13+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'accessible-amqp', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.capabilities' : 'publisher_confirms,exchange_exchange_bindings,basic.nack,consumer_cancel_notify,connection.blocked,consumer_priorities,authentication_failure_close,per_consumer_qos,direct_reply_to', - 'extra.class' : '10', - 'extra.cluster_name' : 'rabbit@1397a0e9629b', - 'extra.locales' : 'en_US', - 'extra.mechanisms' : 'PLAIN AMQPLAIN', - 'extra.message_length' : 509, - 'extra.method' : '10', - 'extra.platform' : 'Erlang/OTP 24.2', - 'extra.product' : 'RabbitMQ', - 'extra.product_version' : '3.9.11', - 'extra.naics' : 454110, - 'extra.tag' : 'amqp', - 'extra.version_minor' : '9', - 'feed.name' : 'Accessible AMQP', - 'protocol.application' : 'amqp', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 14618, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'ASHBURN', - 'source.geolocation.region' : 'VIRGINIA', - 'source.ip' : '54.234.0.0', - 'source.port' : 5672, - 'source.reverse_dns' : 'ec2-54.234.0.0.compute-1.amazonaws.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T04:32:13+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ard.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ard.py deleted file mode 100644 index 4d8420c3bb..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ard.py +++ /dev/null @@ -1,111 +0,0 @@ -# SPDX-FileCopyrightText: 2020 Tomas Bellus -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ard.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible ARD', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2020-07-20T00:00:00+00:00", - "extra.file_name": "2020-01-01-scan_ard-test-test.csv", - - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'accessible-ard', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 201.2, - 'extra.machine_name': 'Macmini (radio)', - 'extra.response_size': 1006, - 'extra.tag': 'ard', - 'feed.name': 'Accessible ARD', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 3283, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-ard', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 201.2, - 'extra.machine_name': 'biuro-rip-org-pl', - 'extra.response_size': 1006, - 'extra.tag': 'ard', - 'feed.name': 'Accessible ARD', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 3283, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-ard', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 201.2, - 'extra.machine_name': '127.0.0.1', - 'extra.response_size': 1006, - 'extra.tag': 'ard', - 'feed.name': 'Accessible ARD', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 3283, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_chargen.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_chargen.py deleted file mode 100644 index 3b72baa8db..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_chargen.py +++ /dev/null @@ -1,110 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_chargen.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open Chargen', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_chargen-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-chargen', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 74.0, - 'extra.response_size': 74, - 'extra.tag': 'chargen', - 'feed.name': 'Open Chargen', - 'protocol.application': 'chargen', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 19, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-chargen', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 74.0, - 'extra.response_size': 74, - 'extra.tag': 'chargen', - 'feed.name': 'Open Chargen', - 'protocol.application': 'chargen', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 19, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-chargen', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 74.0, - 'extra.response_size': 74, - 'extra.sector': 'Government', - 'extra.tag': 'chargen', - 'feed.name': 'Open Chargen', - 'protocol.application': 'chargen', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 19, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_cisco_smart_install.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_cisco_smart_install.py deleted file mode 100644 index 46c963a79e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_cisco_smart_install.py +++ /dev/null @@ -1,82 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_cisco_smart_install.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible Cisco Smart Install', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_cisco_smart_install-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Accessible Cisco Smart Install', - 'classification.identifier': 'accessible-cisco-smart-install', - 'classification.type': 'vulnerable-system', - 'classification.taxonomy': 'vulnerable', - 'protocol.application': 'cisco-smart-install', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 8559, - 'source.geolocation.cc': 'AT', - 'source.geolocation.city': 'VIENNA', - 'source.geolocation.region': 'WIEN', - 'source.ip': '198.51.100.103', - 'source.port': 4786, - 'extra.tag': 'cisco-smart-install', - 'source.reverse_dns': '198-51-100-103.example.net', - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2017-11-18T08:42:45+00:00'}, - {'__type': 'Event', - 'feed.name': 'Accessible Cisco Smart Install', - 'classification.identifier': 'accessible-cisco-smart-install', - 'classification.type': 'vulnerable-system', - 'classification.taxonomy': 'vulnerable', - 'protocol.application': 'cisco-smart-install', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn': 35609, - 'source.geolocation.cc': 'AT', - 'source.geolocation.city': 'VIENNA', - 'source.geolocation.region': 'WIEN', - 'source.ip': '198.51.100.218', - 'source.port': 4786, - 'extra.tag': 'cisco-smart-install', - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2017-11-18T08:47:54+00:00'}, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_coap.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_coap.py deleted file mode 100644 index 773fc04d51..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_coap.py +++ /dev/null @@ -1,121 +0,0 @@ -# SPDX-FileCopyrightText: 2020 Thomas Hungenberg -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_coap.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible-CoAP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2020-06-29T00:00:00+00:00", - "extra.file_name": "2020-06-28-scan_coap-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'accessible-coap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 2.05, - 'extra.response': ',,', - 'extra.response_size': 43, - 'extra.tag': 'coap', - 'extra.version': '2', - 'feed.name': 'Accessible-CoAP', - 'protocol.application': 'coap', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 5683, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-coap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 5.38, - 'extra.response': ',,,,,,,,,', - 'extra.response_size': 113, - 'extra.tag': 'coap', - 'extra.version': '2', - 'feed.name': 'Accessible-CoAP', - 'protocol.application': 'coap', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 5683, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-coap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 113.5, - 'extra.response': '`EsjAy************************************************************|CoAP ' - 'RFC 7252 ' - '|************************************************************|This ' - 'server is using the Eclipse Californium (Cf) CoAP ' - 'framework|published under EPL+EDL: ' - 'http://www.eclipse.org/californium/||(c) 2014, 2015, 2016 ' - 'Institute for Pervasive Computing, ETH Zurich and ' - 'others|************************************************************', - 'extra.response_size': 454, - 'extra.tag': 'coap', - 'extra.version': '1', - 'feed.name': 'Accessible-CoAP', - 'protocol.application': 'coap', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 5683, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_couchdb.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_couchdb.py deleted file mode 100644 index 1bf6f321c6..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_couchdb.py +++ /dev/null @@ -1,128 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_couchdb.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible CouchDB Server', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_couchdb-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-couchdb', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.couchdb_message' : 'Welcome', - 'extra.couchdb_version' : '1.6.1', - 'extra.server_version' : 'CouchDB/1.6.1 (Erlang OTP/18)', - 'extra.tag' : 'couchdb', - 'extra.vendor' : 'Ubuntu 16.04', - 'extra.visible_databases' : '_replicator;_users;test;shops;god', - 'feed.name' : 'Accessible CouchDB Server', - 'protocol.application' : 'couchdb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 5984, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-couchdb', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.couchdb_message' : 'Welcome', - 'extra.couchdb_version' : '3.2.1', - 'extra.features' : 'access-ready,partitioned,pluggable-storage-engines,reshard,scheduler', - 'extra.git_sha' : '244d428af', - 'extra.server_version' : 'CouchDB/3.2.1 (Erlang OTP/23)', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'couchdb', - 'extra.vendor' : 'The Apache Software Foundation', - 'feed.name' : 'Accessible CouchDB Server', - 'protocol.application' : 'couchdb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 5984, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-couchdb', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.couchdb_message' : 'Welcome', - 'extra.couchdb_version' : '3.2.1', - 'extra.features' : 'access-ready,partitioned,pluggable-storage-engines,reshard,scheduler', - 'extra.git_sha' : '244d428af', - 'extra.server_version' : 'CouchDB/3.2.1 (Erlang OTP/20)', - 'extra.source.sector' : 'Retail Trade', - 'extra.tag' : 'couchdb', - 'extra.vendor' : 'The Apache Software Foundation', - 'feed.name' : 'Accessible CouchDB Server', - 'protocol.application' : 'couchdb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 5984, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_cwmp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_cwmp.py deleted file mode 100644 index b508b64508..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_cwmp.py +++ /dev/null @@ -1,103 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_cwmp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible CWMP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2018-07-30T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_cwmp-test-test.csv", - - } -EVENTS = [{ - '__type': 'Event', - 'feed.name': 'Accessible CWMP', - "classification.identifier": "open-cwmp", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.connection": "keep-alive", - "extra.content_length": 5678, - "extra.content_type": "text/html", - "extra.date": "Wed, 04 Sep 2019 07:42:37 GMT", - "extra.http": "HTTP/1.1", - "extra.http_code": 200, - "extra.http_reason": "OK", - "extra.naics": 517311, - "extra.server": "DNVRS-Webs", - "extra.tag": "cwmp", - "protocol.application": "cwmp", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 5678, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.142", - "source.port": 30005, - "time.observation": "2018-07-30T00:00:00+00:00", - "time.source": "2019-09-04T10:44:55+00:00" -}, -{ - '__type': 'Event', - 'feed.name': 'Accessible CWMP', - "classification.identifier": "open-cwmp", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.content_type": "text/html", - "extra.http": "HTTP/1.1", - "extra.http_code": 404, - "extra.http_reason": "Not Found", - "extra.naics": 517311, - "extra.server": "RomPager/4.07 UPnP/1.0", - "extra.tag": "cwmp", - "extra.transfer_encoding": "chunked", - "protocol.application": "cwmp", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 5678, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.162", - "source.port": 5678, - "source.reverse_dns": "localhost.localdomain", - "time.observation": "2018-07-30T00:00:00+00:00", - "time.source": "2019-09-04T11:06:50+00:00" - },] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_db2.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_db2.py deleted file mode 100644 index 423ebe8c53..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_db2.py +++ /dev/null @@ -1,91 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Sebastian Wagner -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_db2.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "ShadowServer Open-DB2-Discovery-Service", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_db2-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-db2-discovery-service', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 14.9, - 'extra.db2_hostname': 'NOWAK_SERWER', - 'extra.servername': 'node01.example.com', - 'extra.size': 298, - 'extra.tag': 'db2', - 'feed.name': 'ShadowServer Open-DB2-Discovery-Service', - 'protocol.application': 'db2', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 523, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-db2-discovery-service', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 14.9, - 'extra.db2_hostname': 'SPZOZ-DZIEWIN', - 'extra.servername': 'node02.example.com', - 'extra.size': 298, - 'extra.tag': 'db2', - 'feed.name': 'ShadowServer Open-DB2-Discovery-Service', - 'protocol.application': 'db2', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 523, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ddos_middlebox.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ddos_middlebox.py deleted file mode 100644 index 9038a79ef1..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ddos_middlebox.py +++ /dev/null @@ -1,119 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ddos_middlebox.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'DDoS Middlebox', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_ddos_middlebox-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ddos-middlebox', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.amplification' : 2, - 'extra.bytes' : 99, - 'extra.method' : 'SYN+ACK:PSH', - 'extra.source_port' : '49002', - 'feed.name' : 'DDoS Middlebox', - 'protocol.application' : 'ddos-middlebox', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 80, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ddos-middlebox', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.amplification' : 2, - 'extra.bytes' : 99, - 'extra.method' : 'SYN+ACK:PSH', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.source_port' : '41200', - 'feed.name' : 'DDoS Middlebox', - 'protocol.application' : 'ddos-middlebox', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 80, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ddos-middlebox', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.amplification' : 2, - 'extra.bytes' : 99, - 'extra.method' : 'SYN+ACK:PSH', - 'extra.source_port' : '47492', - 'feed.name' : 'DDoS Middlebox', - 'protocol.application' : 'ddos-middlebox', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 80, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_dns.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_dns.py deleted file mode 100644 index 3492f82cec..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_dns.py +++ /dev/null @@ -1,91 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_dns.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'DNS Open Resolvers', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2018-07-30T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_dns-test-test.csv", - } -EVENTS = [{ - '__type': 'Event', - 'feed.name': 'DNS Open Resolvers', - "classification.identifier": "dns-open-resolver", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.dns_version": "dnsmasq-2.66", - "extra.min_amplification": 4.619, - "extra.tag": "openresolver", - "protocol.application": "dns", - "protocol.transport": "udp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 25255, - "source.geolocation.cc": "AT", - "source.geolocation.city": "VIENNA", - "source.geolocation.region": "WIEN", - "source.ip": "198.51.100.179", - "source.port": 53, - "source.reverse_dns": "198-51-100-189.example.net", - "time.observation": "2018-07-30T00:00:00+00:00", - "time.source": "2018-04-14T00:14:34+00:00" -}, -{ - '__type': 'Event', - 'feed.name': 'DNS Open Resolvers', - "classification.identifier": "dns-open-resolver", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.dns_version": "dnsmasq-2.51", - "extra.min_amplification": 4.619, - "extra.tag": "openresolver", - "protocol.application": "dns", - "protocol.transport": "udp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 25255, - "source.geolocation.cc": "AT", - "source.geolocation.city": "VIENNA", - "source.geolocation.region": "WIEN", - "source.ip": "198.51.100.8", - "source.port": 53, - "source.reverse_dns": "198-51-100-111.example.net", - "time.observation": "2018-07-30T00:00:00+00:00", - "time.source": "2018-04-14T00:14:36+00:00" -},] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_docker.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_docker.py deleted file mode 100644 index 31d0e4417e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_docker.py +++ /dev/null @@ -1,159 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_docker.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible Docker Service', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_docker-test.csv", - } -EVENTS = [ - { - '__type' : 'Event', - 'classification.identifier' : 'open-docker', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.api_version' : '1.37', - 'extra.arch' : 'amd64', - 'extra.build_time' : '2018-05-09T22:18:36.000000000+00:00', - 'extra.content_type' : 'application/json; charset=UTF-8', - 'extra.date' : 'Fri, 06 May 2022 14:06:30 GMT', - 'extra.experimental' : 'false', - 'extra.git_commit' : 'f150324', - 'extra.go_version' : 'go1.9.5', - 'extra.http' : 'HTTP/1.1', - 'extra.http_code' : 200, - 'extra.http_reason' : 'OK', - 'extra.kernel_version' : '3.10.0-514.26.2.el7.x86_64', - 'extra.min_api_version' : '1.12', - 'extra.os.name' : 'linux', - 'extra.server' : 'Docker/18.05.0-ce (linux)', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'docker', - 'extra.version' : '18.05.0-ce', - 'feed.name' : 'Accessible Docker Service', - 'protocol.application' : 'docker', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 2375, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-docker', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.api_version' : '1.26', - 'extra.arch' : 'amd64', - 'extra.build_time' : '2022-03-02T15:25:43.414574467+00:00', - 'extra.content_type' : 'application/json', - 'extra.date' : 'Fri, 06 May 2022 14:08:07 GMT', - 'extra.experimental' : 'false', - 'extra.git_commit' : '7d71120/1.13.1', - 'extra.go_version' : 'go1.10.3', - 'extra.http' : 'HTTP/1.1', - 'extra.http_code' : 200, - 'extra.http_reason' : 'OK', - 'extra.kernel_version' : '3.10.0-693.2.2.el7.x86_64', - 'extra.min_api_version' : '1.12', - 'extra.os.name' : 'linux', - 'extra.pkg_version' : 'docker-1.13.1-209.git7d71120.el7.centos.x86_64', - 'extra.server' : 'Docker/1.13.1 (linux)', - 'extra.tag' : 'docker', - 'extra.version' : '1.13.1', - 'feed.name' : 'Accessible Docker Service', - 'protocol.application' : 'docker', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 2375, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-docker', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.api_version' : '1.37', - 'extra.arch' : 'amd64', - 'extra.build_time' : '2018-05-09T22:18:36.000000000+00:00', - 'extra.content_type' : 'application/json; charset=UTF-8', - 'extra.date' : 'Fri, 06 May 2022 14:08:06 GMT', - 'extra.experimental' : 'false', - 'extra.git_commit' : 'f150324', - 'extra.go_version' : 'go1.9.5', - 'extra.http' : 'HTTP/1.1', - 'extra.http_code' : 200, - 'extra.http_reason' : 'OK', - 'extra.kernel_version' : '3.10.0-514.26.2.el7.x86_64', - 'extra.min_api_version' : '1.12', - 'extra.os.name' : 'linux', - 'extra.server' : 'Docker/18.05.0-ce (linux)', - 'extra.tag' : 'docker', - 'extra.version' : '18.05.0-ce', - 'feed.name' : 'Accessible Docker Service', - 'protocol.application' : 'docker', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 2375, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_dvr_dhcpdiscover.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_dvr_dhcpdiscover.py deleted file mode 100644 index 01e68db94b..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_dvr_dhcpdiscover.py +++ /dev/null @@ -1,178 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_dvr_dhcpdiscover.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible DVR DHCPDiscover', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_dvr_dhcpdiscover-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-dvr-dhcpdiscover', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.alarm_input_channels': 0, - 'extra.alarm_output_channels': 0, - 'extra.amplification': 794.0, - 'extra.device_model': 'BCS-TIP3401IR-E-V', - 'extra.device_serial': '6J0E022PAG35073', - 'extra.device_type': 'IPC', - 'extra.device_vendor': 'General', - 'extra.device_version': '2.800.106F004.0.R', - 'extra.http_port': 80, - 'extra.internal_port': 37777, - 'extra.ipv4_address': '192.168.0.1', - 'extra.ipv4_dhcp_enable': False, - 'extra.ipv4_gateway': '192.168.0.240', - 'extra.ipv4_subnet_mask': '255.255.255.0', - 'extra.ipv6_address': 'fd09:4ab5:dae9:b078::1', - 'extra.ipv6_dhcp_enable': False, - 'extra.ipv6_gateway': 'fd09:4ab5:dae9:b078::ff', - 'extra.ipv6_link_local': 'fe80::3ac4:e8ff:fe03:b3e2/64', - 'extra.mac_address': '38:c4:e8:03:b3:e2', - 'extra.machine_name': '6J0E022PAG35073', - 'extra.manufacturer': 'General', - 'extra.method': 'client.notifyDevInfo', - 'extra.remote_video_input_channels': 0, - 'extra.response_size': 794, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.video_input_channels': 1, - 'extra.video_output_channels': 0, - 'feed.name': 'Accessible DVR DHCPDiscover', - 'protocol.application': 'dvrdhcpdiscover', - 'protocol.transport': 'udp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 37810, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, - {'__type': 'Event', - 'classification.identifier': 'open-dvr-dhcpdiscover', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.alarm_input_channels': 0, - 'extra.alarm_output_channels': 0, - 'extra.amplification': 761.0, - 'extra.device_model': 'HCVR', - 'extra.device_serial': '2K0488CPAGS0ND6', - 'extra.device_type': 'HCVR', - 'extra.device_vendor': 'Private', - 'extra.device_version': '3.210.1.4', - 'extra.http_port': 80, - 'extra.internal_port': 37777, - 'extra.ipv4_address': '192.168.0.2', - 'extra.ipv4_dhcp_enable': False, - 'extra.ipv4_gateway': '192.168.0.240', - 'extra.ipv4_subnet_mask': '255.255.255.0', - 'extra.ipv6_address': 'fd09:4ab5:dae9:b078::2', - 'extra.ipv6_gateway': 'fd09:4ab5:dae9:b078::ff', - 'extra.ipv6_link_local': 'fe80::3eef:8cff:fe18:a507/64', - 'extra.mac_address': '3c:ef:8c:18:a5:07', - 'extra.machine_name': 'HCVR', - 'extra.manufacturer': 'Private', - 'extra.method': 'client.notifyDevInfo', - 'extra.remote_video_input_channels': 9, - 'extra.response_size': 761, - 'extra.video_input_channels': 3, - 'extra.video_output_channels': 0, - 'feed.name': 'Accessible DVR DHCPDiscover', - 'protocol.application': 'dvrdhcpdiscover', - 'protocol.transport': 'udp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 37810, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-dvr-dhcpdiscover', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.alarm_input_channels': 0, - 'extra.alarm_output_channels': 0, - 'extra.amplification': 711.0, - 'extra.device_model': 'BCS-XVR0401-IV', - 'extra.device_serial': '5L034FAPAZA0E30', - 'extra.device_type': 'HCVR', - 'extra.device_vendor': 'General', - 'extra.device_version': '4.000.0000002.11', - 'extra.http_port': 80, - 'extra.internal_port': 37777, - 'extra.ipv4_address': '192.168.0.3', - 'extra.ipv4_dhcp_enable': False, - 'extra.ipv4_gateway': '192.168.0.240', - 'extra.ipv4_subnet_mask': '255.255.255.0', - 'extra.ipv6_address': 'fd09:4ab5:dae9:b078::3', - 'extra.ipv6_gateway': 'fd09:4ab5:dae9:b078::ff', - 'extra.ipv6_link_local': 'fe80::3ac4:e8ff:fe02:74da/64', - 'extra.mac_address': '38:c4:e8:02:74:da', - 'extra.machine_name': 'XVR', - 'extra.manufacturer': 'General', - 'extra.method': 'client.notifyDevInfo', - 'extra.remote_video_input_channels': 0, - 'extra.response_size': 711, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.video_input_channels': 4, - 'extra.video_output_channels': 0, - 'feed.name': 'Accessible DVR DHCPDiscover', - 'protocol.application': 'dvrdhcpdiscover', - 'protocol.transport': 'udp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 37810, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_elasticsearch.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_elasticsearch.py deleted file mode 100644 index 4e12a1b076..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_elasticsearch.py +++ /dev/null @@ -1,126 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_elasticsearch.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open Elasticsearch', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_elasticsearch-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-elasticsearch', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.build_hash': '90f439ff60a3c0f497f91663701e64ccd01edbb4', - 'extra.build_snapshot': False, - 'extra.build_timestamp': '2016-07-27T10:36:52Z', - 'extra.cluster_name': 'elasticsearch', - 'extra.lucene_version': '5.5.0', - 'extra.name': 'Red Skull', - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'elasticsearch', - 'extra.tagline': 'You Know, for Search', - 'extra.version': '2.3.5', - 'feed.name': 'Open Elasticsearch', - 'protocol.application': 'elasticsearch', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 9200, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, - {'__type': 'Event', - 'classification.identifier': 'open-elasticsearch', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.build_hash': 'bee86328705acaa9a6daede7140defd4d9ec56bd', - 'extra.build_snapshot': False, - 'extra.cluster_name': 'docker-cluster', - 'extra.lucene_version': '8.11.1', - 'extra.name': 'allinonepod', - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'elasticsearch', - 'extra.tagline': 'You Know, for Search', - 'extra.version': '7.17.0', - 'feed.name': 'Open Elasticsearch', - 'protocol.application': 'elasticsearch', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 9200, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-elasticsearch', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.build_hash': '79d65f6e357953a5b3cbcc5e2c7c21073d89aa29', - 'extra.build_snapshot': False, - 'extra.cluster_name': 'docker-cluster', - 'extra.lucene_version': '8.9.0', - 'extra.name': 'f547c2952610', - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'elasticsearch', - 'extra.tagline': 'You Know, for Search', - 'extra.version': '7.15.0', - 'feed.name': 'Open Elasticsearch', - 'protocol.application': 'elasticsearch', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 9200, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_exchange.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_exchange.py deleted file mode 100644 index aeeffa3c29..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_exchange.py +++ /dev/null @@ -1,149 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), "testdata/scan_exchange.csv")) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = { - "feed.name": "Shadowserver CVE-2021-26855", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2020-08-19T00:00:00+00:00", - "extra.file_name": "2020-08-19-scan_exchange.csv", -} - -EVENTS = [ - { - "__type": "Event", - "feed.name": "Shadowserver CVE-2021-26855", - "time.source": "2021-05-14T00:11:30+00:00", - "source.ip": "12.237.1.2", - "source.port": 443, - "source.asn": 7018, - "source.geolocation.cc": "US", - "source.geolocation.region": "CALIFORNIA", - "source.geolocation.city": "TURLOCK", - "source.reverse_dns": 'afs-exch-cas2.xxx.com', - "extra.version": '15.2.721', - "extra.source.sector": "Communications, Service Provider, and Hosting Service", - "extra.source.naics": 517311, - "classification.identifier": "vulnerable-exchange-server", - "extra.tag": "exchange;cve-2021-26855", - "classification.taxonomy": "vulnerable", - "classification.type": "infected-system", - "extra.servername": "AFS-EXCH2019", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[1]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Shadowserver CVE-2021-26855", - "time.source": "2021-05-14T00:11:37+00:00", - "source.ip": "98.153.3.4", - "source.port": 443, - "source.asn": 20001, - "source.geolocation.cc": "US", - "source.geolocation.region": "CALIFORNIA", - "source.geolocation.city": "LOS ANGELES", - "source.reverse_dns": 'rrcs-98-153-x-x.west.biz.rr.com', - "extra.version": '15.0.847', - "extra.source.sector": "Communications, Service Provider, and Hosting Service", - "extra.source.naics": 517311, - "extra.tag": "exchange;webshell", - "classification.taxonomy": "intrusions", - "classification.type": "system-compromise", - "classification.identifier": "exchange-server-webshell", - "extra.servername": "SSAMAIL", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[2]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Shadowserver CVE-2021-26855", - "time.source": "2021-05-14T00:11:38+00:00", - "source.ip": "206.210.5.6", - "source.port": 443, - "source.asn": 17054, - "source.geolocation.cc": "US", - "source.geolocation.region": "PENNSYLVANIA", - "source.geolocation.city": "PITTSBURGH", - "source.reverse_dns": 'webmail.xxx.com', - "extra.source.naics": 518210, - "extra.version": '15.0.1178', - "extra.servername": "OMNYXEXCH02", - "extra.tag": "exchange;webshell", - "classification.taxonomy": "intrusions", - "classification.type": "system-compromise", - "classification.identifier": "exchange-server-webshell", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[3]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Shadowserver CVE-2021-26855", - "time.source": "2021-05-14T00:11:38+00:00", - "source.ip": "12.33.7.8", - "source.port": 443, - "source.asn": 7018, - "source.geolocation.cc": "US", - "source.geolocation.region": "ARKANSAS", - "source.geolocation.city": "LITTLE ROCK", - "source.reverse_dns": 'mail.xxx.org', - "extra.version": '15.1.2176', - "extra.source.sector": "Communications, Service Provider, and Hosting Service", - "extra.source.naics": 921120, - "extra.servername": "MHASVR02", - "classification.identifier": "vulnerable-exchange-server", - "extra.tag": "exchange;cve-2021-26855", - "classification.taxonomy": "vulnerable", - "classification.type": "infected-system", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[4]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Shadowserver CVE-2021-26855", - "time.source": "2021-05-14T00:11:38+00:00", - "source.ip": "41.204.9.10", - "source.port": 443, - "source.asn": 21042, - "source.geolocation.cc": 'MG', - "source.geolocation.city": 'ANTANANARIVO', - "source.geolocation.region": 'ANTANANARIVO', - "source.reverse_dns": 'mail.xxx.mg', - "extra.servername": "SABMHQE0232", - "classification.identifier": "vulnerable-exchange-server", - "extra.tag": "exchange;cve-2021-26855", - "classification.taxonomy": "vulnerable", - "classification.type": "infected-system", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[5]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, -] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == "__main__": - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ftp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ftp.py deleted file mode 100644 index 33daefd75e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ftp.py +++ /dev/null @@ -1,120 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ftp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible FTP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2019-03-25T00:00:00+00:00", - "extra.file_name": "2019-03-25-scan_ftp-test-test.csv", - } -EVENTS = [{ - '__type': 'Event', - 'feed.name': 'Accessible FTP', - 'time.observation': '2019-03-25T00:00:00+00:00', - 'time.source': '2019-03-06T06:37:00+00:00', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-ftp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.ip': '61.126.3.70', - 'source.port': 21, - 'protocol.transport': 'tcp', - 'protocol.application': 'ftp', - 'source.reverse_dns': 'arcus-net.co.jp', - 'extra.tag': 'ftp', - 'source.asn': 4713, - 'source.geolocation.cc': 'JP', - 'source.geolocation.region': 'TOKYO', - 'source.geolocation.city': 'TOKYO', - 'extra.naics': 517311, - 'extra.sic': 737401, - 'extra.banner': '220 FTP Server ready.|', - 'extra.handshake': 'TLSv1.2', - 'extra.cipher_suite': 'TLS_RSA_WITH_AES_128_CBC_SHA', - 'extra.cert_length': 2048, - 'extra.subject_common_name': '*.bizmw.com', - 'extra.issuer_common_name': 'GlobalSign Organization Validation CA - SHA256 - G2', - 'extra.cert_issue_date': 'Jan 14 08:04:50 2015 GMT', - 'extra.cert_expiration_date': 'Jan 14 08:04:50 2020 GMT', - 'extra.sha1_fingerprint': 'D9:98:3F:2E:F9:D1:BE:9A:10:1E:DE:51:2C:C1:DF:01:18:0A:20:65', - 'extra.cert_serial_number': '1121DC7421AB7924C3B1D396AEA3707E9E29', - 'extra.ssl_version': 2, - 'extra.signature_algorithm': 'sha256WithRSAEncryption', - 'extra.key_algorithm': 'rsaEncryption', - 'extra.subject_organization_name': 'NTT Communications Corporation', - 'extra.subject_country': 'JP', - 'extra.subject_state_or_province_name': 'Tokyo', - 'extra.subject_locality_name': 'Minato-ku', - 'extra.issuer_organization_name': 'GlobalSign nv-sa', - 'extra.issuer_country': 'BE', - 'extra.sha256_fingerprint': '27:4A:8A:3A:A7:DF:82:D0:43:03:0E:6F:48:30:30:C9:24:77:11:1A:08:EF:F7:B9:74:0C:CE:40:87:03:D2:51', - 'extra.sha512_fingerprint': 'E5:93:8B:72:84:0F:35:52:8E:7A:6C:E3:EF:36:90:4C:F2:86:A7:4D:B2:DD:C0:C6:23:83:18:EF:DD:86:34:92:91:57:22:29:75:45:71:8B:3A:CD:F1:27:A9:CA:5F:70:5E:AC:15:A5:E6:63:FD:6F:BB:C5:E2:45:99:73:E9:E6', - 'extra.md5_fingerprint': 'D1:A7:BC:96:78:1D:16:D0:24:A8:62:7C:3A:95:5A:4A', - 'extra.cert_valid': False, - 'extra.self_signed': False, - 'extra.cert_expired': False, - 'extra.validation_level': 'OV', - 'extra.auth_tls_response': '234 AUTH TLS successful', - }, - { - '__type': 'Event', - 'feed.name': 'Accessible FTP', - 'time.observation': '2019-03-25T00:00:00+00:00', - 'time.source': '2019-03-06T06:37:00+00:00', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-ftp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.ip': '62.48.156.65', - 'source.port': 21, - 'protocol.transport': 'tcp', - 'protocol.application': 'ftp', - 'source.reverse_dns': 'dial-62-48-156-65.ptprime.net', - 'extra.tag': 'ftp', - 'source.asn': 15525, - 'source.geolocation.cc': 'PT', - 'source.geolocation.region': 'LISBOA', - 'source.geolocation.city': 'FRIELAS', - 'extra.banner': '220-================================================================| PT Empresas| Acesso Reservado| Acesso nao autorizado punido por lei: 109/91; 67/98| ----------------------------------------------------------------| HENNES & MAURITZ LDA - 149093| SITE: PT303 - Cascais Shopping| MORADA: | NIR: EWS1822940| ================================================================|220 FTP server ready, 1 active clients of 4 simultaneous clients allowed.|', - 'extra.auth_tls_response': '500 Syntax error, command unrecognized.', - 'extra.auth_ssl_response': '500 Syntax error, command unrecognized.' - } - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_hadoop.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_hadoop.py deleted file mode 100644 index 0b5794cb7b..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_hadoop.py +++ /dev/null @@ -1,94 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Sebastian Wagner -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_hadoop.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "ShadowServer Accessible-Hadoop", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_hadoop-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'ShadowServer Accessible-Hadoop', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-hadoop', - 'extra.version': '2.7.3, rbaa91f7c6bc9cb92be5982de4719c1c8af91ccff', - 'extra.server_type': 'namenode', - 'extra.clusterid': 'CID-64471a53-60cb-4302-9832-92f321f111fe', - 'extra.total_disk': 41567956992, - 'extra.used_disk': 53248, - 'extra.free_disk': 25160089600, - 'extra.livenodes': 'edmonton:50010', - 'protocol.application': 'hadoop', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 15296, - 'source.geolocation.cc': 'CA', - 'source.geolocation.city': 'CALGARY', - 'source.geolocation.region': 'ALBERTA', - 'source.ip': '199.116.235.200', - 'source.port': 50070, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2017-09-13T02:06:05+00:00'}, - {'__type': 'Event', - 'feed.name': 'ShadowServer Accessible-Hadoop', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-hadoop', - 'extra.version': '2.7.1.2.4.0.0-169', - 'extra.naics': 334111, - 'extra.sic': 357101, - 'extra.server_type': 'datanode', - 'extra.clusterid': 'CID-771bae52-9e4f-4ec4-bc1a-c867585751f0', - 'extra.namenodeaddress': 'sandbox.hortonworks.com', - 'extra.volumeinfo': '/hadoop/hdfs/data/current', - 'protocol.application': 'hadoop', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn': 8075, - 'source.geolocation.cc': 'US', - 'source.geolocation.city': 'DES MOINES', - 'source.geolocation.region': 'IOWA', - 'source.ip': '104.43.235.92', - 'source.port': 50075, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2017-09-13T02:07:48+00:00'}, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_http.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_http.py deleted file mode 100644 index 793a95f221..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_http.py +++ /dev/null @@ -1,100 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_http.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible HTTP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-03-25-scan_http-test-test.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Accessible HTTP', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'classification.identifier': 'accessible-http', - 'extra.source.naics': 518111, - 'extra.source.sic': 737401, - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 200, - 'extra.http_reason': 'OK', - 'extra.content_type': 'text/html', - 'extra.server': 'lighttpd', - 'extra.transfer_encoding': 'chunked', - 'extra.http_date': '2018-04-19T00:02:28+00:00', - 'extra.tag': 'http', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.reverse_dns': 'c-75-74-78-113.hsd1.fl.comcast.net', - 'source.asn': 7922, - 'source.geolocation.cc': 'US', - 'source.geolocation.city': 'MIAMI', - 'source.geolocation.region': 'FLORIDA', - 'source.ip': '75.74.78.113', - 'source.port': 8080, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2018-04-19T00:02:26+00:00'}, - {'__type': 'Event', - 'feed.name': 'Accessible HTTP', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'classification.identifier': 'accessible-http', - 'extra.source.naics': 518210, - 'extra.source.sic': 737415, - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 200, - 'extra.http_reason': 'OK', - 'extra.content_type': 'text/html', - 'extra.content_length': 17729, - 'extra.http_date': '2018-04-19T02:02:28+00:00', - 'extra.tag': 'http', - 'protocol.transport': 'tcp', - 'protocol.application': 'http', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.reverse_dns': 'sto95-3-88-162-174-130.fbx.proxad.net', - 'source.asn': 12322, - 'source.geolocation.cc': 'FR', - 'source.geolocation.city': 'SAINT-OUEN-LAUMONE', - 'source.ip': '88.162.174.130', - 'source.port': 8080, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2018-04-19T00:02:26+00:00'}, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_http_proxy.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_http_proxy.py deleted file mode 100644 index dc5e94e5ec..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_http_proxy.py +++ /dev/null @@ -1,118 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_http_proxy.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open HTTP Proxy', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_http_proxy-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-http-proxy', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 200, - 'extra.http_date': '2010-02-10T00:00:00+00:00', - 'extra.http_reason': 'Connection established', - 'extra.tag': 'http-connect-proxy', - 'feed.name': 'Open HTTP Proxy', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 3128, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-http-proxy', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 200, - 'extra.http_date': '2010-02-10T00:00:01+00:00', - 'extra.http_reason': 'Connection established', - 'extra.tag': 'http-connect-proxy', - 'extra.via': 'HTTP/1.1 s_proxy_den1', - 'feed.name': 'Open HTTP Proxy', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 3128, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-http-proxy', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 200, - 'extra.http_date': '2010-02-10T00:00:02+00:00', - 'extra.http_reason': 'Connection established', - 'extra.tag': 'http-connect-proxy', - 'extra.via': 'HTTP/1.1 s_proxy_yvr', - 'feed.name': 'Open HTTP Proxy', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 3128, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_http_vulnerable.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_http_vulnerable.py deleted file mode 100644 index d15232eaf7..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_http_vulnerable.py +++ /dev/null @@ -1,125 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Mikk Margus Möll -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_http_vulnerable.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Vulnerable HTTP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2021-08-01T09:00:00+00:00", - "extra.file_name": "2021-08-01-scan_http_vulnerable-test-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'accessible-http', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.content_length': 149, - 'extra.content_type': 'text/html; charset=utf-8', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 401, - 'extra.http_date': '2010-02-10T00:00:00+00:00', - 'extra.http_reason': 'Unauthorized', - 'extra.server': 'TwistedWeb/19.7.0', - 'extra.set_cookie': 'TWISTED_SESSION=5473ad3faa3de66685fb3a53bffb390b4fcec2039893009a06caf38e1bec8aa8', - 'extra.tag': 'basic-auth,http', - 'extra.www_authenticate': 'Basic realm=\\\\"OpenWebif\\"\\""', - 'feed.name': 'Vulnerable HTTP', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 8080, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-http', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.content_length': 149, - 'extra.content_type': 'text/html; charset=utf-8', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 401, - 'extra.http_date': '2010-02-10T00:00:01+00:00', - 'extra.http_reason': 'Unauthorized', - 'extra.server': 'TwistedWeb/19.7.0', - 'extra.set_cookie': 'TWISTED_SESSION=d2460d37b7fdbdd6c27dd74423ead5704e553d4f2c230672313edc5602059e33', - 'extra.tag': 'basic-auth,http', - 'extra.www_authenticate': 'Basic realm=\\\\"OpenWebif\\"\\""', - 'feed.name': 'Vulnerable HTTP', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 80, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-http', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.detail': 'repositoryformatversion = 0;filemode = false;bare = ' - 'false;logallrefupdates = true;symlinks = false;ignorecase = ' - 'true', - 'extra.http_date': '2010-02-10T00:00:02+00:00', - 'extra.tag': 'git-config-file', - 'feed.name': 'Vulnerable HTTP', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 443, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ics.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ics.py deleted file mode 100644 index f673f40c80..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ics.py +++ /dev/null @@ -1,125 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ics.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Acessible ICS', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_ics-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ics', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.device_model' : 'device_model', - 'extra.device_type' : 'device_type', - 'extra.device_vendor' : 'Vendor 1', - 'extra.device_version' : 'device_version', - 'extra.raw_response' : 'dGVzdDE=', - 'extra.response_size' : 5, - 'extra.source.sector' : 'Sector', - 'feed.name' : 'Acessible ICS', - 'protocol.application' : 'modbus', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'CITY', - 'source.geolocation.region' : 'REGION', - 'source.ip' : '192.168.0.1', - 'source.port' : 502, - 'source.reverse_dns' : 'host1.example.net', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-03-02T00:34:22+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ics', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.device_model' : 'device_model', - 'extra.device_type' : 'device_type', - 'extra.device_vendor' : 'Vendor 2', - 'extra.device_version' : 'device_version', - 'extra.raw_response' : 'dGVzdDI=', - 'extra.response_size' : 5, - 'extra.source.sector' : 'Sector', - 'feed.name' : 'Acessible ICS', - 'protocol.application' : 'modbus', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64513, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'CITY', - 'source.geolocation.region' : 'REGION', - 'source.ip' : '192.168.0.2', - 'source.port' : 502, - 'source.reverse_dns' : 'host2.example.net', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-03-02T00:34:22+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ics', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.device_model' : 'device_model', - 'extra.device_type' : 'device_type', - 'extra.device_vendor' : 'Vendor 3', - 'extra.device_version' : 'device_version', - 'extra.raw_response' : 'dGVzdDM=', - 'extra.response_size' : 5, - 'extra.source.sector' : 'Sector', - 'feed.name' : 'Acessible ICS', - 'protocol.application' : 'modbus', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64514, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'CITY', - 'source.geolocation.region' : 'REGION', - 'source.ip' : '192.168.0.3', - 'source.port' : 502, - 'source.reverse_dns' : 'host3.example.net', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-03-02T00:34:22+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ipmi.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ipmi.py deleted file mode 100644 index 08a9082af9..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ipmi.py +++ /dev/null @@ -1,106 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_ipmi.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open IPMI', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ipmi-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Open IPMI', - "classification.identifier": "open-ipmi", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.anon_login": False, - "extra.defaultkg": "-", - "extra.ipmi_version": "1.5", - "extra.md2_auth": False, - "extra.md5_auth": True, - "extra.none_auth": True, - "extra.nulluser": True, - "extra.oem_auth": False, - "extra.passkey_auth": True, - "extra.permessage_auth": True, - "extra.tag": "ipmi", - "extra.userlevel_auth": True, - "extra.usernames": False, - "protocol.application": "ipmi", - "protocol.transport": "udp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 2914, - "source.geolocation.cc": "DE", - "source.geolocation.city": "BERLIN", - "source.geolocation.region": "BERLIN", - "source.ip": "198.51.100.4", - "source.port": 623, - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2016-07-24T00:09:42+00:00" - }, - {'__type': 'Event', - 'feed.name': 'Open IPMI', - "classification.identifier": "open-ipmi", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.anon_login": False, - "extra.defaultkg": "default", - "extra.ipmi_version": "2.0", - "extra.md2_auth": False, - "extra.md5_auth": False, - "extra.none_auth": False, - "extra.nulluser": False, - "extra.oem_auth": False, - "extra.passkey_auth": False, - "extra.permessage_auth": False, - "extra.tag": "ipmi", - "extra.userlevel_auth": True, - "extra.usernames": True, - "protocol.application": "ipmi", - "protocol.transport": "udp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 28753, - "source.geolocation.cc": "DE", - "source.geolocation.city": "FRANKFURT AM MAIN", - "source.geolocation.region": "HESSEN", - "source.ip": "198.51.100.182", - "source.port": 623, - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2016-07-24T00:09:43+00:00" - }, - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ipp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ipp.py deleted file mode 100644 index 9adc8485e0..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ipp.py +++ /dev/null @@ -1,79 +0,0 @@ -# SPDX-FileCopyrightText: 2020 Thomas Hungenberg -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_ipp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open-IPP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2020-06-09T00:00:00+00:00", - "extra.file_name": "2020-06-08-scan_ipp-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Open-IPP', - "classification.identifier": "open-ipp", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517311, - "extra.tag": "ipp", - "extra.ipp_version": "IPP/2.1", - "extra.cups_version": "CUPS/2.0", - "extra.printer_uris": "ipp://123.45.67.89:631/ipp/print", - "extra.printer_name": "NPI3F0D22", - "extra.printer_info": "HP Color LaserJet MFP M277dw", - "extra.printer_more_info": "http://123.45.67.89:631/hp/device/info_config_AirPrint.html?tab=Networking&menu=AirPrintStatus", - "extra.printer_make_and_model": "HP Color LaserJet MFP M277dw", - "extra.printer_firmware_name": "20191203", - "extra.printer_firmware_string_version": "20191203", - "extra.printer_firmware_version": "20191203", - "extra.printer_organization": "org", - "extra.printer_organization_unit": "unit", - "extra.printer_uuid": "urn:uuid:456e4238-4a44-4643-4c42-10e1813f0a18", - "extra.printer_wifi_ssid": "wifissid", - "protocol.application": "ipp", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 12345, - "source.geolocation.cc": "AA", - "source.geolocation.city": "CITY", - "source.geolocation.region": "REGION", - "source.ip": "123.45.67.89", - "source.port": 631, - 'source.reverse_dns': 'some.host.com', - "time.observation": "2020-06-09T00:00:00+00:00", - "time.source": "2020-06-08T11:30:14+00:00" - }, - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_isakmp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_isakmp.py deleted file mode 100644 index 3192f508f8..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_isakmp.py +++ /dev/null @@ -1,105 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_isakmp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Vulnerable ISAKMP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_isakmp-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Vulnerable ISAKMP', - "classification.identifier": "open-ike", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.domain_of_interpretation": '00', - "extra.exchange_type": '05', - "extra.flags": '00', - "extra.initiator_spi": "3e35c70729dfedef", - "extra.message_id": "00000000", - "extra.naics": 517311, - "extra.next_payload": '11', - "extra.next_payload2": '00', - "extra.notify_message_type": '14', - "extra.responder_spi": "253acab7cbfda607", - "extra.spi_size": 0, - "extra.tag": "isakmp-vulnerable", - "protocol.application": "ipsec", - "protocol.transport": "udp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 5678, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.42", - "source.port": 500, - "source.reverse_dns": "example.local", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T00:17:25+00:00" - }, - {'__type': 'Event', - 'feed.name': 'Vulnerable ISAKMP', - "classification.identifier": "open-ike", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.domain_of_interpretation": '00', - "extra.exchange_type": '05', - "extra.flags": '00', - "extra.initiator_spi": "3e35c70729dfedef", - "extra.message_id": "00000000", - "extra.next_payload": '11', - "extra.next_payload2": '00', - "extra.notify_message_type": '14', - "extra.responder_spi": "b274460e7adc1bf0", - "extra.spi_size": 0, - "extra.tag": "isakmp-vulnerable", - "protocol.application": "ipsec", - "protocol.transport": "udp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 20255, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.67", - "source.port": 500, - "source.reverse_dns": "example.local", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T00:17:28+00:00" - }, - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_kubernetes.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_kubernetes.py deleted file mode 100644 index 2bac336a79..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_kubernetes.py +++ /dev/null @@ -1,214 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_kubernetes.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible Kubernetes API Server', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_kubernetes-test.csv", - } -EVENTS = [ - { - '__type' : 'Event', - 'classification.identifier' : 'open-kubernetes', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.browser_error' : 'x509: failed to load system roots and no roots provided', - 'extra.browser_trusted' : False, - 'extra.build_date' : '2021-11-17T13:00:29Z', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.compiler' : 'gc', - 'extra.content_type' : 'application/json', - 'extra.date' : 'Tue, 10 May 2022 14:24:13 GMT', - 'extra.git_commit' : '2444b3347a2c45eb965b182fb836e1f51dc61b70', - 'extra.git_tree_state' : 'clean', - 'extra.git_version' : 'v1.20.13', - 'extra.go_version' : 'go1.15.15', - 'extra.handshake' : 'TLSv1.2', - 'extra.http' : 'HTTP/1.1', - 'extra.http_code' : 200, - 'extra.http_reason' : 'OK', - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.major' : '1', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.minor' : '20', - 'extra.platform' : 'linux/amd64', - 'extra.self_signed' : False, - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.tag' : 'kubernetes', - 'feed.name' : 'Accessible Kubernetes API Server', - 'protocol.application' : 'kubernetes', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 6443, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-kubernetes', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.browser_error' : 'x509: failed to load system roots and no roots provided', - 'extra.browser_trusted' : False, - 'extra.build_date' : '2022-02-25T06:26:46Z', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.compiler' : 'gc', - 'extra.content_type' : 'application/json', - 'extra.date' : 'Tue, 10 May 2022 14:24:12 GMT', - 'extra.git_commit' : '6f5a5295923a614a4202a7ad274b38b69f9ca8c0', - 'extra.git_tree_state' : 'clean', - 'extra.git_version' : 'v1.23.3+e419edf', - 'extra.go_version' : 'go1.17.5', - 'extra.handshake' : 'TLSv1.2', - 'extra.http' : 'HTTP/1.1', - 'extra.http_code' : 200, - 'extra.http_reason' : 'OK', - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.major' : '1', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.minor' : '23', - 'extra.platform' : 'linux/amd64', - 'extra.self_signed' : False, - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.source.sector' : 'Retail Trade', - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.tag' : 'kubernetes', - 'feed.name' : 'Accessible Kubernetes API Server', - 'protocol.application' : 'kubernetes', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 6443, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-kubernetes', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.browser_error' : 'x509: failed to load system roots and no roots provided', - 'extra.browser_trusted' : False, - 'extra.build_date' : '2020-05-08T07:29:59Z', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.compiler' : 'gc', - 'extra.content_type' : 'application/json', - 'extra.date' : 'Tue, 10 May 2022 14:24:12 GMT', - 'extra.git_commit' : '4f7ea78', - 'extra.git_version' : 'v1.16.9-aliyun.1', - 'extra.go_version' : 'go1.13.9', - 'extra.handshake' : 'TLSv1.2', - 'extra.http' : 'HTTP/1.1', - 'extra.http_code' : 200, - 'extra.http_reason' : 'OK', - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.major' : '1', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.minor' : '16+', - 'extra.platform' : 'linux/amd64', - 'extra.self_signed' : False, - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.tag' : 'kubernetes', - 'feed.name' : 'Accessible Kubernetes API Server', - 'protocol.application' : 'kubernetes', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 6443, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_tcp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_tcp.py deleted file mode 100644 index b6abf6eba9..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_tcp.py +++ /dev/null @@ -1,154 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_ldap_tcp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open LDAP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ldap_tcp-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.configuration_naming_context': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.default_naming_context': 'DC=ad,DC=example,DC=com', - 'extra.domain_controller_functionality': 7, - 'extra.ds_service_name': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.forest_functionality': 2, - 'extra.ldap_service_name': 'node01.example.com', - 'extra.naming_contexts': 'DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.root_domain_naming_context': 'DC=example,DC=com', - 'extra.schema_naming_context': 'CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.server_name': 'CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com', - 'extra.size': 0, - 'extra.subschema_subentry': 'CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.supported_control': '1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|', - 'extra.supported_ldap_policies': 'MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent', - 'extra.supported_ldap_version': '3|2', - 'extra.supported_sasl_mechanisms': 'GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5', - 'extra.tag': 'ldap-tcp', - 'feed.name': 'Open LDAP', - 'protocol.application': 'ldap', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.local_hostname': 'node01.example.com', - 'source.port': 389, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, - {'__type': 'Event', - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.configuration_naming_context': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.current_time': '20220821124435.0Z', - 'extra.default_naming_context': 'DC=ad,DC=example,DC=com', - 'extra.domain_controller_functionality': 7, - 'extra.domain_functionality': 7, - 'extra.ds_service_name': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.forest_functionality': 7, - 'extra.highest_committed_usn': 25029662, - 'extra.is_global_catalog_ready': True, - 'extra.is_synchronized': True, - 'extra.ldap_service_name': 'node02.example.com', - 'extra.naming_contexts': 'DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.root_domain_naming_context': 'DC=example,DC=com', - 'extra.schema_naming_context': 'CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.server_name': 'CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com', - 'extra.size': 0, - 'extra.subschema_subentry': 'CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.supported_capabilities': '1.2.840.113556.1.4.800|1.2.840.113556.1.4.1670|1.2.840.113556.1.4.1791|1.2.840.113556.1.4.1935|1.2.840.113556.1.4.2080|1.2.840.113556.1.4.2237', - 'extra.supported_control': '1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|1.2.840.113556.1.4.2206|1.2.840.113556.1.4.2211|1.2.840.113556.1.4.2239|1.2.840.113556.1.4.2255|1.2.840.113556.1.4.2256|1.2.840.113556.1.4.2309', - 'extra.supported_ldap_policies': 'MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent', - 'extra.supported_ldap_version': '3|2', - 'extra.supported_sasl_mechanisms': 'GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5', - 'extra.tag': 'ldap-tcp', - 'feed.name': 'Open LDAP', - 'protocol.application': 'ldap', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.local_hostname': 'node02.example.com', - 'source.port': 389, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.configuration_naming_context': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.current_time': '20220821124539.0Z', - 'extra.default_naming_context': 'DC=ad,DC=example,DC=com', - 'extra.ds_service_name': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.ldap_service_name': 'node03.example.com', - 'extra.naming_contexts': 'DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.root_domain_naming_context': 'DC=example,DC=com', - 'extra.schema_naming_context': 'CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.server_name': 'CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com', - 'extra.size': 0, - 'extra.subschema_subentry': 'CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.supported_control': '1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|', - 'extra.tag': 'ldap-tcp', - 'feed.name': 'Open LDAP', - 'protocol.application': 'ldap', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.local_hostname': 'node03.example.com', - 'source.port': 389, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_udp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_udp.py deleted file mode 100644 index aa4deefb87..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_udp.py +++ /dev/null @@ -1,162 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_ldap_udp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open LDAP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ldap_udp-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 58.42, - 'extra.configuration_naming_context': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.current_time': '20220821044533.0Z', - 'extra.default_naming_context': 'DC=ad,DC=example,DC=com', - 'extra.domain_controller_functionality': 7, - 'extra.domain_functionality': 7, - 'extra.ds_service_name': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.forest_functionality': 7, - 'extra.highest_committed_usn': 222537, - 'extra.is_global_catalog_ready': True, - 'extra.is_synchronized': True, - 'extra.ldap_service_name': 'node01.example.com', - 'extra.naming_contexts': 'DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.root_domain_naming_context': 'DC=example,DC=com', - 'extra.schema_naming_context': 'CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.server_name': 'CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com', - 'extra.size': 3038, - 'extra.subschema_subentry': 'CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.supported_capabilities': '1.2.840.113556.1.4.800|1.2.840.113556.1.4.1670|1.2.840.113556.1.4.1791|1.2.840.113556.1.4.1935|1.2.840.113556.1.4.2080|1.2.840.113556.1.4.2237', - 'extra.supported_control': '1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|1.2.840.113556.1.4.2206|1.2.840.113556.1.4.2211|1.2.840.113556.1.4.2239|1.2.840.113556.1.4.2255|1.2.840.113556.1.4.2256|1.2.840.113556.1.4.2309|1.2.840.113556.1.4.2330|1.2.840.113556.1.4.2354', - 'extra.supported_ldap_policies': 'MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent', - 'extra.supported_ldap_version': '3|2', - 'extra.supported_sasl_mechanisms': 'GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5', - 'extra.tag': 'ldap-udp', - 'feed.name': 'Open LDAP', - 'protocol.application': 'ldap', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.local_hostname': 'node01.example.com', - 'source.port': 389, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 58.88, - 'extra.configuration_naming_context': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.current_time': '20220821044948.0Z', - 'extra.default_naming_context': 'DC=ad,DC=example,DC=com', - 'extra.domain_controller_functionality': 7, - 'extra.domain_functionality': 7, - 'extra.ds_service_name': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.forest_functionality': 7, - 'extra.highest_committed_usn': 1478714, - 'extra.is_global_catalog_ready': True, - 'extra.is_synchronized': True, - 'extra.ldap_service_name': 'node02.example.com', - 'extra.naming_contexts': 'DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.root_domain_naming_context': 'DC=example,DC=com', - 'extra.schema_naming_context': 'CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.server_name': 'CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com', - 'extra.size': 3062, - 'extra.subschema_subentry': 'CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.supported_capabilities': '1.2.840.113556.1.4.800|1.2.840.113556.1.4.1670|1.2.840.113556.1.4.1791|1.2.840.113556.1.4.1935|1.2.840.113556.1.4.2080|1.2.840.113556.1.4.2237', - 'extra.supported_control': '1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|1.2.840.113556.1.4.2206|1.2.840.113556.1.4.2211|1.2.840.113556.1.4.2239|1.2.840.113556.1.4.2255|1.2.840.113556.1.4.2256|1.2.840.113556.1.4.2309|1.2.840.113556.1.4.2330|1.2.840.113556.1.4.2354', - 'extra.supported_ldap_policies': 'MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent', - 'extra.supported_ldap_version': '3|2', - 'extra.supported_sasl_mechanisms': 'GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5', - 'extra.tag': 'ldap-udp', - 'feed.name': 'Open LDAP', - 'protocol.application': 'ldap', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.local_hostname': 'node02.example.com', - 'source.port': 389, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 0.69, - 'extra.configuration_naming_context': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.default_naming_context': 'DC=ad,DC=example,DC=com', - 'extra.ds_service_name': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.ldap_service_name': 'node03.example.com', - 'extra.naming_contexts': 'DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.root_domain_naming_context': 'DC=example,DC=com', - 'extra.schema_naming_context': 'CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.server_name': 'CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com', - 'extra.size': 36, - 'extra.subschema_subentry': 'CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.tag': 'ldap-udp', - 'feed.name': 'Open LDAP', - 'protocol.application': 'ldap', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.local_hostname': 'node03.example.com', - 'source.port': 389, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_mdns.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_mdns.py deleted file mode 100644 index 9207aaf365..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_mdns.py +++ /dev/null @@ -1,127 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_mdns.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open mDNS', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_mdns-test-geo.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mdns', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.http_ipv4' : '192.168.0.1', - 'extra.http_ipv6' : 'fd09:4ab5:dae9:b078::1', - 'extra.services' : '_smb._tcp.local.; _device-info._tcp.local.; _http._tcp.local.; _dacp._tcp.local.;', - 'extra.tag' : 'mdns', - 'extra.workstation_ipv4' : '192.168.0.1', - 'extra.workstation_ipv6' : 'fd09:4ab5:dae9:b078::1', - 'feed.name' : 'Open mDNS', - 'protocol.application' : 'mdns', - 'protocol.transport' : 'udp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 5353, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mdns', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.http_ipv4' : '192.168.0.2', - 'extra.http_ipv6' : 'fd09:4ab5:dae9:b078::2', - 'extra.services' : '_home-assistant._tcp.local.;', - 'extra.tag' : 'mdns', - 'extra.workstation_ipv4' : '192.168.0.2', - 'extra.workstation_ipv6' : 'fd09:4ab5:dae9:b078::2', - 'feed.name' : 'Open mDNS', - 'protocol.application' : 'mdns', - 'protocol.transport' : 'udp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 5353, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mdns', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.http_info' : '\\\\\"vendor=Synology\\\"\\\" \\\"\\\"model=DS218+\\\"\\\" \\\"\\\"serial=17A0PCN482002\\\"\\\" \\\"\\\"version_major=6\\\"\\\" \\\"\\\"version_minor=2\\\"\\\" \\\"\\\"version_build=25556\\\"\\\" \\\"\\\"admin_port=5000\\\"\\\" \\\"\\\"secure_admin_port=5001\\\"\\\" \\\"\\\"mac_address=00:11:32:80:fd:b5\\\"\\\"\"', - 'extra.http_ipv4' : '192.168.0.3', - 'extra.http_ipv6' : 'fd09:4ab5:dae9:b078::3', - 'extra.http_name' : 'snmeijer.local.', - 'extra.http_port' : 5000, - 'extra.http_ptr' : 'snmeijer._http._tcp.local.', - 'extra.http_target' : 'snmeijer.local.', - 'extra.services' : '_webdav._tcp.local.; _adisk._tcp.local.; _smb._tcp.local.; _http._tcp.local.; _dacp._tcp.local.; _afpovertcp._tcp.local.; _device-info._tcp.local.;', - 'extra.tag' : 'mdns,iot', - 'extra.workstation_ipv4' : '192.168.0.3', - 'extra.workstation_ipv6' : 'fd09:4ab5:dae9:b078::3', - 'feed.name' : 'Open mDNS', - 'protocol.application' : 'mdns', - 'protocol.transport' : 'udp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 5353, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_memcached.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_memcached.py deleted file mode 100644 index b54fc0ea53..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_memcached.py +++ /dev/null @@ -1,130 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_memcached.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open Memcached', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_memcached-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-memcached', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 81.71, - 'extra.curr_connections': 243, - 'extra.pid': 1010, - 'extra.pointer_size': 64, - 'extra.response_size': 1144, - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'memcached', - 'extra.time': '2022-08-21 10:34:06', - 'extra.total_connections': 6106, - 'extra.uptime': 32908114, - 'extra.version': '1.4.15', - 'feed.name': 'Open Memcached', - 'protocol.application': 'memcached', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 50260, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-memcached', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 75.21, - 'extra.curr_connections': 9, - 'extra.pid': 5316, - 'extra.pointer_size': 64, - 'extra.response_size': 1053, - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'memcached', - 'extra.time': '2022-08-21 10:39:21', - 'extra.total_connections': 2962, - 'extra.uptime': 9618498, - 'extra.version': '1.4.13', - 'feed.name': 'Open Memcached', - 'protocol.application': 'memcached', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 11211, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, - {'__type': 'Event', - 'classification.identifier': 'open-memcached', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 31.57, - 'extra.curr_connections': 2, - 'extra.pid': 1460, - 'extra.pointer_size': 32, - 'extra.response_size': 442, - 'extra.tag': 'memcached', - 'extra.time': '2022-08-21 10:39:39', - 'extra.total_connections': 534, - 'extra.uptime': 1375159, - 'extra.version': '1.2.6', - 'feed.name': 'Open Memcached', - 'protocol.application': 'memcached', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 11211, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_mongodb.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_mongodb.py deleted file mode 100644 index 3ecf7b21f9..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_mongodb.py +++ /dev/null @@ -1,103 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_mongodb.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open MongoDB', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_mongodb-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Open MongoDB', - "classification.identifier": "open-mongodb", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.allocator": "tcmalloc", - "extra.bits": "64", - "extra.gitversion": "a2ddc68ba7c9cee17bfe69ed840383ec3506602b", - "extra.javascriptengine": "V8", - "extra.maxbsonobjectsize": "16777216", - "extra.ok": True, - "extra.sysinfo": "Linux ip-198-51-100-100 198.51.100.103-2.ec2.v1.2.fc8xen #1 SMP Fri Nov 20 17:48:28 EST 2009 x86_64 BOOST_LIB_VERSION=1_49", - "extra.tag": "mongodb", - "extra.version": "2.4.5", - "extra.visible_databases": "local | countly | admin", - "protocol.application": "mongodb", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 20773, - "source.geolocation.cc": "DE", - "source.geolocation.city": "WEEZE", - "source.geolocation.region": "NORDRHEIN-WESTFALEN", - "source.ip": "198.51.100.203", - "source.port": 27017, - "source.reverse_dns": "198-51-100-203.example.net", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2016-07-24T00:40:07+00:00" - }, - {'__type': 'Event', - 'feed.name': 'Open MongoDB', - "classification.identifier": "open-mongodb", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.allocator": "tcmalloc", - "extra.bits": "64", - "extra.gitversion": "d73c92b1c85703828b55c2916a5dd4ad46535f6a", - "extra.javascriptengine": "V8", - "extra.maxbsonobjectsize": "16777216", - "extra.ok": True, - "extra.sector": "Information Technology", - "extra.sysinfo": "Linux build5.ny.cbi.10gen.cc 2.6.32-431.3.1.el6.x86_64 #1 SMP Fri Jan 3 21:39:27 UTC 2014 x86_64 BOOST_LIB_VERSION=1_49", - "extra.tag": "mongodb", - "extra.version": "2.6.12", - "extra.visible_databases": "none visible", - "protocol.application": "mongodb", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 24940, - "source.geolocation.cc": "DE", - "source.geolocation.city": "GUNZENHAUSEN", - "source.geolocation.region": "BAYERN", - "source.ip": "198.51.100.42", - "source.port": 27017, - "source.reverse_dns": "198-51-100-208.example.net", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2016-07-24T00:40:07+00:00" - }, - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt.py deleted file mode 100644 index 45d19f9eea..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt.py +++ /dev/null @@ -1,89 +0,0 @@ -# SPDX-FileCopyrightText: 2020 Thomas Hungenberg -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_mqtt.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open-MQTT', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2020-03-15T00:00:00+00:00", - "extra.file_name": "2020-03-14-scan_mqtt-test-geo.csv", - } -EVENTS = [ - { - '__type' : 'Event', - 'classification.identifier' : 'open-mqtt', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.anonymous_access' : False, - 'extra.cert_expiration_date' : '2022-11-14 00:00:00', - 'extra.cert_issue_date' : '2020-08-12 00:00:00', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : '085699743A23114C9B6B8DC975A8AF42', - 'extra.cipher_suite' : 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', - 'extra.code' : 'Connection Refused, not authorized', - 'extra.hex_code' : '05', - 'extra.issuer_common_name' : 'Sectigo RSA Domain Validation Secure Server CA', - 'extra.issuer_country' : 'GB', - 'extra.issuer_locality_name' : 'Salford', - 'extra.issuer_organization_name' : 'Sectigo Limited', - 'extra.issuer_state_or_province_name' : 'Greater Manchester', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'DE:2C:98:30:27:2E:7D:C9:ED:A3:9D:AF:9E:CE:14:CC', - 'extra.raw_response' : '20020005', - 'extra.sha1_fingerprint' : '70:84:F1:6D:28:DA:B6:E6:27:60:13:8B:2C:93:52:B6:7B:4B:13:7B', - 'extra.sha256_fingerprint' : 'D2:D7:54:52:EB:86:4E:2D:34:4D:FC:CE:CD:CF:39:41:E1:06:5C:8B:B8:54:E6:0C:DF:FD:6E:E3:F1:B5:41:00', - 'extra.sha512_fingerprint' : '17:57:FB:88:9D:BE:A7:F0:29:A5:31:FC:79:DF:F7:8A:1C:D6:4A:DF:1B:4A:DC:BF:05:E7:E8:2F:79:9A:FA:FE:F7:E8:66:22:CB:B9:4C:72:F7:FB:6C:1D:59:8C:54:63:70:05:DE:7F:3C:2F:BA:B8:37:18:CE:29:6F:11:E8:AB', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.naics' : 454110, - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : '*.tracesafe.io', - 'extra.tag' : 'mqtt', - 'feed.name' : 'Open-MQTT', - 'protocol.application' : 'mqtt', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 12345, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'COLUMBUS', - 'source.geolocation.region' : 'OHIO', - 'source.ip' : '18.220.0.0', - 'source.port' : 8883, - 'source.reverse_dns' : '18-220-0-0.example.com', - 'time.observation' : '2020-03-15T00:00:00+00:00', - 'time.source' : '2022-02-07T12:56:53+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt_anon.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt_anon.py deleted file mode 100644 index 4618957240..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt_anon.py +++ /dev/null @@ -1,173 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_mqtt_anon.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open Anonymous MQTT', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_mqtt_anon-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mqtt-anon', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.cert_expiration_date' : '2030-05-06 08:07:05', - 'extra.cert_issue_date' : '2020-05-08 08:07:05', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : '02', - 'extra.cipher_suite' : 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', - 'extra.code' : 'Connection Accepted', - 'extra.hex_code' : '00', - 'extra.issuer_common_name' : 'RootCA', - 'extra.issuer_country' : 'CN', - 'extra.issuer_organization_name' : 'EMQ', - 'extra.issuer_state_or_province_name' : 'hangzhou', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'AB:A8:E0:2C:EF:AE:BF:9D:DD:FA:70:BA:2F:F2:CA:5C', - 'extra.raw_response' : '20020000', - 'extra.sha1_fingerprint' : '70:1A:1E:1F:EC:5F:7E:A9:12:32:B2:C9:8A:C9:EE:91:8E:0B:82:45', - 'extra.sha256_fingerprint' : '85:26:A2:F2:A2:50:CD:96:33:19:A6:2D:12:2E:97:6B:D3:06:3C:11:EA:01:B4:B7:25:2A:B7:4F:0A:8F:45:40', - 'extra.sha512_fingerprint' : '72:50:07:30:9A:6F:CB:FD:E2:80:69:02:65:62:77:16:C3:B4:0C:98:44:4E:D4:2C:AC:6B:AF:F8:9E:AB:51:C2:FA:A8:72:A3:45:DF:81:09:50:08:18:EB:03:34:FC:92:33:A7:12:46:FE:90:20:91:86:C5:4D:89:48:86:4C:CD', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.source.naics' : 518210, - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'Server', - 'extra.subject_country' : 'CN', - 'extra.subject_organization_name' : 'EMQ', - 'extra.subject_state_or_province_name' : 'hangzhou', - 'extra.tag' : 'mqtt,mqtt-anon', - 'feed.name' : 'Open Anonymous MQTT', - 'protocol.application' : 'mqtt', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 37963, - 'source.geolocation.cc' : 'CN', - 'source.geolocation.city' : 'SHENZHEN', - 'source.geolocation.region' : 'GUANGDONG SHENG', - 'source.ip' : '47.106.0.0', - 'source.port' : 8883, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:59:34+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mqtt-anon', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.cert_expiration_date' : '2022-03-06 13:48:03', - 'extra.cert_issue_date' : '2021-12-06 13:48:04', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : '06B25BEAD1F43266ABCFCDDE408D3544D04B', - 'extra.cipher_suite' : 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', - 'extra.code' : 'Connection Accepted', - 'extra.hex_code' : '00', - 'extra.issuer_common_name' : 'R3', - 'extra.issuer_country' : 'US', - 'extra.issuer_organization_name' : 'Lets Encrypt', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : '23:99:39:C6:77:D8:9F:55:90:FC:A5:FB:BA:72:8B:42', - 'extra.raw_response' : '20020000', - 'extra.sha1_fingerprint' : '20:0E:AC:E7:AF:07:8D:D3:16:7C:63:D1:B9:12:AD:1D:2C:F0:46:86', - 'extra.sha256_fingerprint' : 'DD:7A:4C:8A:1D:66:1D:7C:F5:17:04:5B:A0:B4:C4:E0:80:58:44:B4:DB:A7:5E:61:AE:43:9D:85:4C:9E:DC:83', - 'extra.sha512_fingerprint' : '55:B6:3D:56:A4:39:6E:99:B6:AF:72:AF:4D:3C:7C:C5:A8:C5:4F:A1:79:92:D0:46:8A:A2:9B:2A:48:0D:00:68:39:F0:B8:67:B4:E0:88:51:2A:D7:55:46:83:BD:ED:1E:09:6E:DB:3D:21:E2:AA:DB:42:6A:33:45:1A:2A:DB:4C', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.source.naics' : 518210, - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'example.com', - 'extra.tag' : 'mqtt,mqtt-anon', - 'feed.name' : 'Open Anonymous MQTT', - 'protocol.application' : 'mqtt', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 24940, - 'source.geolocation.cc' : 'DE', - 'source.geolocation.city' : 'WERNIGERODE', - 'source.geolocation.region' : 'SACHSEN-ANHALT', - 'source.ip' : '144.76.0.0', - 'source.port' : 8883, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:59:34+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mqtt-anon', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.cert_expiration_date' : '2030-08-05 16:51:57', - 'extra.cert_issue_date' : '2020-08-07 16:51:57', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'A71541EFAE529B03', - 'extra.cipher_suite' : 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA', - 'extra.code' : 'Connection Accepted', - 'extra.hex_code' : '00', - 'extra.issuer_common_name' : 'ClearView2Dev', - 'extra.issuer_organization_name' : 'Sohonet', - 'extra.issuer_organization_unit_name' : 'ClearView2Dev', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : '43:0D:A7:89:9E:76:8D:6E:D5:AD:95:CC:F2:91:87:56', - 'extra.raw_response' : '20020000', - 'extra.sha1_fingerprint' : '32:4B:66:98:FA:5B:D2:D1:F2:53:83:21:19:11:5A:A9:BE:85:56:16', - 'extra.sha256_fingerprint' : 'AE:0D:65:34:2F:51:F7:32:1E:DF:B1:DA:12:C7:6A:DE:42:B5:4B:FF:80:2C:E5:EF:99:F6:CC:01:4B:C9:77:68', - 'extra.sha512_fingerprint' : '44:C4:B8:19:FA:39:55:51:EC:E4:6D:C4:6D:0F:A5:46:BB:D5:F9:FD:A6:8D:DF:F3:2D:D2:92:6C:0B:D5:D3:25:CB:19:50:9D:A6:A4:D4:D3:2E:53:10:F5:8D:77:F7:90:F8:65:A7:79:AB:14:62:72:01:F3:EA:38:E2:68:C7:25', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.ssl_version' : 0, - 'extra.subject_common_name' : 'foo.example.com', - 'extra.subject_locality_name' : '<', - 'extra.subject_organization_name' : 'Sohonet', - 'extra.tag' : 'mqtt,mqtt-anon', - 'feed.name' : 'Open Anonymous MQTT', - 'protocol.application' : 'mqtt', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 5555, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'BURBANK', - 'source.geolocation.region' : 'CALIFORNIA', - 'source.ip' : '173.0.0.0', - 'source.port' : 8883, - 'source.reverse_dns' : 'example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:59:34+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_mssql.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_mssql.py deleted file mode 100644 index 0f12014e68..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_mssql.py +++ /dev/null @@ -1,123 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_mssql.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open MSSQL', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_mssql-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-mssql', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 310.0, - 'extra.instance_name': 'OPTIMA', - 'extra.named_pipe': '\\\\\\\\ERPOPTIMA\\\\pipe\\\\MSSQL$OPTIMA\\\\sql\\\\query', - 'extra.response_size': 310, - 'extra.tag': 'mssql', - 'extra.tcp_port': 49729, - 'extra.version': '13.2.5026.0', - 'feed.name': 'Open MSSQL', - 'protocol.application': 'mssql', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.local_hostname': 'ERPOPTIMA', - 'source.port': 1434, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-mssql', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 226.0, - 'extra.instance_name': 'MSSQLSERVER', - 'extra.response_size': 226, - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'mssql', - 'extra.tcp_port': 1433, - 'extra.version': '13.0.1601.5', - 'feed.name': 'Open MSSQL', - 'protocol.application': 'mssql', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.local_hostname': 'SERWER', - 'source.port': 1434, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-mssql', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 304.0, - 'extra.instance_name': 'INSERTGT', - 'extra.named_pipe': '\\\\\\\\ILONY\\\\pipe\\\\MSSQL$INSERTGT\\\\sql\\\\query', - 'extra.response_size': 304, - 'extra.tag': 'mssql', - 'extra.tcp_port': 49358, - 'extra.version': '10.50.2500.0', - 'feed.name': 'Open MSSQL', - 'protocol.application': 'mssql', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.local_hostname': 'ILONY', - 'source.port': 1434, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_mysql.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_mysql.py deleted file mode 100644 index 3e008f9502..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_mysql.py +++ /dev/null @@ -1,258 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_mysql.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible MySQL Server', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_mysql-test.csv", - } -EVENTS = [ - { - '__type' : 'Event', - 'classification.identifier' : 'open-mysql', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.browser_error' : 'x509: failed to load system roots and no roots provided', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.client_can_handle_expired_passwords' : True, - 'extra.client_compress' : True, - 'extra.client_connect_attrs' : True, - 'extra.client_connect_with_db' : True, - 'extra.client_deprecated_eof' : True, - 'extra.client_found_rows' : True, - 'extra.client_ignore_sigpipe' : True, - 'extra.client_ignore_space' : True, - 'extra.client_interactive' : True, - 'extra.client_local_files' : True, - 'extra.client_long_flag' : True, - 'extra.client_long_password' : True, - 'extra.client_multi_results' : True, - 'extra.client_multi_statements' : True, - 'extra.client_no_schema' : True, - 'extra.client_odbc' : True, - 'extra.client_plugin_auth' : True, - 'extra.client_plugin_auth_len_enc_client_data' : True, - 'extra.client_protocol_41' : True, - 'extra.client_ps_multi_results' : True, - 'extra.client_reserved' : True, - 'extra.client_secure_connection' : True, - 'extra.client_session_track' : False, - 'extra.client_ssl' : False, - 'extra.client_transactions' : False, - 'extra.error_code' : '1', - 'extra.error_id' : '1', - 'extra.error_message' : '1', - 'extra.handshake' : 'TLSv1.2', - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.mysql_protocol_version' : '10', - 'extra.server_version' : '5.7.37-0ubuntu0.18.04.1', - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.tag' : 'mysql', - 'feed.name' : 'Accessible MySQL Server', - 'protocol.application' : 'mysql', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 3306, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mysql', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.browser_error' : 'x509: failed to load system roots and no roots provided', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.client_can_handle_expired_passwords' : True, - 'extra.client_compress' : True, - 'extra.client_connect_attrs' : True, - 'extra.client_connect_with_db' : True, - 'extra.client_deprecated_eof' : True, - 'extra.client_found_rows' : True, - 'extra.client_ignore_sigpipe' : True, - 'extra.client_ignore_space' : True, - 'extra.client_interactive' : True, - 'extra.client_local_files' : True, - 'extra.client_long_flag' : True, - 'extra.client_long_password' : True, - 'extra.client_multi_results' : True, - 'extra.client_multi_statements' : True, - 'extra.client_no_schema' : True, - 'extra.client_odbc' : True, - 'extra.client_plugin_auth' : True, - 'extra.client_plugin_auth_len_enc_client_data' : True, - 'extra.client_protocol_41' : True, - 'extra.client_ps_multi_results' : True, - 'extra.client_reserved' : True, - 'extra.client_secure_connection' : True, - 'extra.client_session_track' : False, - 'extra.client_ssl' : False, - 'extra.client_transactions' : False, - 'extra.error_code' : '1', - 'extra.error_id' : '1', - 'extra.error_message' : '1', - 'extra.handshake' : 'TLSv1.2', - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.mysql_protocol_version' : '10', - 'extra.server_version' : '5.7.30-0ubuntu0.18.04.1-log', - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.tag' : 'mysql', - 'feed.name' : 'Accessible MySQL Server', - 'protocol.application' : 'mysql', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 3306, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mysql', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.browser_error' : 'x509: failed to load system roots and no roots provided', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.client_can_handle_expired_passwords' : True, - 'extra.client_compress' : True, - 'extra.client_connect_attrs' : True, - 'extra.client_connect_with_db' : True, - 'extra.client_deprecated_eof' : True, - 'extra.client_found_rows' : True, - 'extra.client_ignore_sigpipe' : True, - 'extra.client_ignore_space' : True, - 'extra.client_interactive' : True, - 'extra.client_local_files' : True, - 'extra.client_long_flag' : True, - 'extra.client_long_password' : True, - 'extra.client_multi_results' : True, - 'extra.client_multi_statements' : True, - 'extra.client_no_schema' : True, - 'extra.client_odbc' : True, - 'extra.client_plugin_auth' : True, - 'extra.client_plugin_auth_len_enc_client_data' : True, - 'extra.client_protocol_41' : True, - 'extra.client_ps_multi_results' : True, - 'extra.client_reserved' : True, - 'extra.client_secure_connection' : True, - 'extra.client_session_track' : False, - 'extra.client_ssl' : False, - 'extra.client_transactions' : False, - 'extra.error_code' : '1', - 'extra.error_id' : '1', - 'extra.error_message' : '1', - 'extra.handshake' : 'TLSv1.2', - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.mysql_protocol_version' : '10', - 'extra.server_version' : '8.0.23', - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.source.sector' : 'Retail Trade', - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.tag' : 'mysql', - 'feed.name' : 'Accessible MySQL Server', - 'protocol.application' : 'mysql', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 3306, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_nat_pmp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_nat_pmp.py deleted file mode 100644 index beeac2717f..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_nat_pmp.py +++ /dev/null @@ -1,116 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_nat_pmp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open NATPMP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_nat_pmp-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-natpmp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 6.0, - 'extra.external_ip': '192.168.0.1', - 'extra.opcode': '128', - 'extra.response_size': 12, - 'extra.tag': 'nat-pmp', - 'extra.uptime': 291278940, - 'feed.name': 'Open NATPMP', - 'protocol.application': 'natpmp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 5351, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-natpmp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 6.0, - 'extra.external_ip': '192.168.0.2', - 'extra.opcode': '128', - 'extra.response_size': 12, - 'extra.tag': 'nat-pmp', - 'extra.uptime': 768416, - 'feed.name': 'Open NATPMP', - 'protocol.application': 'natpmp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 5351, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, - {'__type': 'Event', - 'classification.identifier': 'open-natpmp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 6.0, - 'extra.external_ip': '192.168.0.3', - 'extra.opcode': '128', - 'extra.response_size': 12, - 'extra.tag': 'nat-pmp', - 'extra.uptime': 19629454, - 'feed.name': 'Open NATPMP', - 'protocol.application': 'natpmp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 5351, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_netbios.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_netbios.py deleted file mode 100644 index febe8305c1..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_netbios.py +++ /dev/null @@ -1,121 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_netbios.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Netbios', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_netbios-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-netbios-nameservice', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 4.58, - 'extra.mac_address': '00-00-00-00-00-00', - 'extra.machine_name': 'NBG6503', - 'extra.response_size': 229, - 'extra.tag': 'netbios', - 'feed.name': 'Netbios', - 'protocol.application': 'netbios-nameservice', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.account': 'NBG6503', - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 137, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-netbios-nameservice', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3.86, - 'extra.mac_address': '00-00-00-00-00-00', - 'extra.machine_name': 'NAS-OLD', - 'extra.response_size': 193, - 'extra.tag': 'netbios', - 'extra.workgroup': 'PRACOWNIAELN.', - 'feed.name': 'Netbios', - 'protocol.application': 'netbios-nameservice', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.account': 'NAS-OLD', - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 137, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-netbios-nameservice', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3.14, - 'extra.mac_address': '00-25-90-F0-64-64', - 'extra.machine_name': 'HR-SRV01', - 'extra.response_size': 157, - 'extra.sector': 'Government', - 'extra.tag': 'netbios', - 'extra.workgroup': 'HRSIGMA', - 'feed.name': 'Netbios', - 'protocol.application': 'netbios-nameservice', - 'protocol.transport': 'udp', - 'raw': 'InRpbWVzdGFtcCIsImlwIiwicHJvdG9jb2wiLCJwb3J0IiwiaG9zdG5hbWUiLCJ0YWciLCJtYWNfYWRkcmVzcyIsImFzbiIsImdlbyIsInJlZ2lvbiIsImNpdHkiLCJ3b3JrZ3JvdXAiLCJtYWNoaW5lX25hbWUiLCJ1c2VybmFtZSIsIm5haWNzIiwic2ljIiwic2VjdG9yIiwicmVzcG9uc2Vfc2l6ZSIsImFtcGxpZmljYXRpb24iCiIyMDEwLTAyLTEwIDAwOjAwOjAyIiwxOTIuMTY4LjAuMyx1ZHAsMTM3LG5vZGUwMy5leGFtcGxlLmNvbSxuZXRiaW9zLDAwLTI1LTkwLUYwLTY0LTY0LDY0NTEyLFpaLFJlZ2lvbixDaXR5LEhSU0lHTUEsSFItU1JWMDEsLDAsMCxHb3Zlcm5tZW50LDE1NywzLjE0', - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 137, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_netis_router.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_netis_router.py deleted file mode 100644 index 043cdf1aad..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_netis_router.py +++ /dev/null @@ -1,107 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Sebastian Wagner -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_netis_router.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_netis_router-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-netis', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 18.0, - 'extra.response': 'Login:', - 'extra.response_size': 18, - 'extra.tag': 'netis_vulnerability', - 'feed.name': 'Open-Netis', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 53413, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-netis', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 18.0, - 'extra.response': 'Login:', - 'extra.response_size': 18, - 'extra.tag': 'netis_vulnerability', - 'feed.name': 'Open-Netis', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 53413, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-netis', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 18.0, - 'extra.response': 'Login:', - 'extra.response_size': 18, - 'extra.tag': 'netis_vulnerability', - 'feed.name': 'Open-Netis', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 53413, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ntp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ntp.py deleted file mode 100644 index 85ef710d4e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ntp.py +++ /dev/null @@ -1,161 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_ntp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'NTP Version', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ntp-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'ntp-version', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 27.0, - 'extra.clock': '0xe6ac3809.363028e7', - 'extra.frequency': 2.018, - 'extra.jitter': 0.977, - 'extra.leap': 0.0, - 'extra.noise': '0.984', - 'extra.offset': 0.557, - 'extra.peer': 18986, - 'extra.poll': 10, - 'extra.precision': -10, - 'extra.refid': '81.15.252.130', - 'extra.reftime': '0xe6ac35ba.2d2e8f2b', - 'extra.response_size': 324, - 'extra.rootdelay': 17.685, - 'extra.rootdispersion': 61.254, - 'extra.stability': '0.027', - 'extra.state': '4', - 'extra.stratum': 4, - 'extra.system': 'UNIX', - 'extra.tag': 'ntpversion', - 'extra.version': '4', - 'feed.name': 'NTP Version', - 'protocol.application': 'ntp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 123, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'ntp-version', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 27.33, - 'extra.clk_wander': 0.007, - 'extra.clock': '0xE6AC3806.7DF3B7A0', - 'extra.frequency': -20.407, - 'extra.jitter': 8.776, - 'extra.leap': 0.0, - 'extra.mintc': '3', - 'extra.offset': -14.502, - 'extra.peer': 19244, - 'extra.precision': -10, - 'extra.refid': '10.48.21.21', - 'extra.reftime': '0xE6AC3431.B3B64790', - 'extra.response_size': 328, - 'extra.rootdelay': 32.25, - 'extra.rootdispersion': 105.778, - 'extra.sector': 'Transportation and Warehousing', - 'extra.stratum': 8, - 'extra.system': 'UNIX', - 'extra.tag': 'ntpversion', - 'extra.tc': 10, - 'extra.version': '4', - 'feed.name': 'NTP Version', - 'protocol.application': 'ntp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 123, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'ntp-version', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 27.0, - 'extra.clk_wander': 0.001, - 'extra.clock': '0xE6AC380A.5A1CAD00', - 'extra.frequency': -24.01, - 'extra.jitter': 2.343, - 'extra.leap': 0.0, - 'extra.mintc': '3', - 'extra.offset': 0.49, - 'extra.peer': 51892, - 'extra.precision': -10, - 'extra.refid': '172.28.0.1', - 'extra.reftime': '0xE6AC3020.0C49BA80', - 'extra.response_size': 324, - 'extra.rootdelay': 7.749, - 'extra.rootdispersion': 81.612, - 'extra.stratum': 4, - 'extra.system': 'UNIX', - 'extra.tag': 'ntpversion', - 'extra.tc': 10, - 'extra.version': '4', - 'feed.name': 'NTP Version', - 'protocol.application': 'ntp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 123, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ntpmonitor.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ntpmonitor.py deleted file mode 100644 index ff0e95f3ea..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ntpmonitor.py +++ /dev/null @@ -1,108 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_ntpmonitor.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'NTP Monitor', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ntpmonitor-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'ntp-monitor', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 55.33, - 'extra.packets': 2, - 'extra.size': 664, - 'feed.name': 'NTP Monitor', - 'protocol.application': 'ntp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 123, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'ntp-monitor', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3666.67, - 'extra.packets': 100, - 'extra.size': 44000, - 'feed.name': 'NTP Monitor', - 'protocol.application': 'ntp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 123, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'ntp-monitor', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3666.67, - 'extra.packets': 100, - 'extra.size': 44000, - 'feed.name': 'NTP Monitor', - 'protocol.application': 'ntp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 123, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_portmapper.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_portmapper.py deleted file mode 100644 index 11caec78a1..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_portmapper.py +++ /dev/null @@ -1,120 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_portmapper.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open Portmapper', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_portmapper-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-portmapper', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3.7, - 'extra.exports': '/mnt/export 192.168.0.0', - 'extra.programs': '100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp; ' - '100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp;', - 'extra.response_size': 148, - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'portmapper', - 'feed.name': 'Open Portmapper', - 'protocol.application': 'portmapper', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 111, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-portmapper', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3.7, - 'extra.exports': '/mnt/export 192.168.0.0', - 'extra.programs': '100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp; ' - '100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp;', - 'extra.response_size': 148, - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'portmapper', - 'feed.name': 'Open Portmapper', - 'protocol.application': 'portmapper', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 111, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-portmapper', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3.7, - 'extra.exports': '/mnt/export 192.168.0.0', - 'extra.programs': '100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp; ' - '100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp;', - 'extra.response_size': 148, - 'extra.sector': 'Government', - 'extra.tag': 'portmapper', - 'feed.name': 'Open Portmapper', - 'protocol.application': 'portmapper', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 111, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_postgres.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_postgres.py deleted file mode 100644 index 43a297f787..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_postgres.py +++ /dev/null @@ -1,199 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_postgres.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible-PostgreSQL', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_postgres-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-postgres', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.client_ssl' : False, - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.protocol_error_code' : '0A000', - 'extra.protocol_error_file' : 'postmaster.c', - 'extra.protocol_error_line' : '1798', - 'extra.protocol_error_message' : 'unsupported frontend protocol 255.255: server supports 1.0 to 3.0', - 'extra.protocol_error_routine' : 'ProcessStartupPacket', - 'extra.protocol_error_severity' : 'FATAL', - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.startup_error_code' : '28000', - 'extra.startup_error_file' : 'postmaster.c', - 'extra.startup_error_line' : 1893, - 'extra.startup_error_message' : 'no PostgreSQL user name specified in startup packet', - 'extra.startup_error_routine' : 'ProcessStartupPacket', - 'extra.startup_error_severity' : 'FATAL', - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.supported_protocols' : '1.0-3.0', - 'extra.tag' : 'postgres', - 'feed.name' : 'Accessible-PostgreSQL', - 'protocol.application' : 'postgres', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 5432, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-postgres', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.client_ssl' : False, - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.protocol_error_code' : '0A000', - 'extra.protocol_error_file' : 'postmaster.c', - 'extra.protocol_error_line' : '1798', - 'extra.protocol_error_message' : 'unsupported frontend protocol 255.255: server supports 1.0 to 3.0', - 'extra.protocol_error_routine' : 'ProcessStartupPacket', - 'extra.protocol_error_severity' : 'FATAL', - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.startup_error_code' : '28000', - 'extra.startup_error_file' : 'postmaster.c', - 'extra.startup_error_line' : 1893, - 'extra.startup_error_message' : 'no PostgreSQL user name specified in startup packet', - 'extra.startup_error_routine' : 'ProcessStartupPacket', - 'extra.startup_error_severity' : 'FATAL', - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.supported_protocols' : '1.0-3.0', - 'extra.tag' : 'postgres', - 'feed.name' : 'Accessible-PostgreSQL', - 'protocol.application' : 'postgres', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 5432, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-postgres', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.client_ssl' : False, - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.protocol_error_code' : '0A000', - 'extra.protocol_error_file' : 'postmaster.c', - 'extra.protocol_error_line' : '1798', - 'extra.protocol_error_message' : 'unsupported frontend protocol 255.255: server supports 1.0 to 3.0', - 'extra.protocol_error_routine' : 'ProcessStartupPacket', - 'extra.protocol_error_severity' : 'FATAL', - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.startup_error_code' : '28000', - 'extra.startup_error_file' : 'postmaster.c', - 'extra.startup_error_line' : 1893, - 'extra.startup_error_message' : 'no PostgreSQL user name specified in startup packet', - 'extra.startup_error_routine' : 'ProcessStartupPacket', - 'extra.startup_error_severity' : 'FATAL', - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.supported_protocols' : '1.0-3.0', - 'extra.tag' : 'postgres', - 'feed.name' : 'Accessible-PostgreSQL', - 'protocol.application' : 'postgres', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 5432, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_qotd.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_qotd.py deleted file mode 100644 index de52af6259..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_qotd.py +++ /dev/null @@ -1,119 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_qotd.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open QOTD', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_qotd-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-qotd', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 166.0, - 'extra.quote': '_The secret of being miserable is to have leisure to bother ' - 'about whether?? you are happy or not. The cure for it is ' - 'occupation._?? George Bernard Shaw (1856-1950)?', - 'extra.response_size': 166, - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'qotd', - 'feed.name': 'Open QOTD', - 'protocol.application': 'qotd', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 17, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-qotd', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 162.0, - 'extra.quote': '_Oh the nerves, the nerves; the mysteries of this machine ' - 'called man!?? Oh the little that unhinges it, poor creatures ' - 'that we are!_?? Charles Dickens (1812-70)?', - 'extra.response_size': 162, - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'qotd', - 'feed.name': 'Open QOTD', - 'protocol.application': 'qotd', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 17, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-qotd', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 162.0, - 'extra.quote': '_Oh the nerves, the nerves; the mysteries of this machine ' - 'called man!?? Oh the little that unhinges it, poor creatures ' - 'that we are!_?? Charles Dickens (1812-70)?', - 'extra.response_size': 162, - 'extra.tag': 'qotd', - 'feed.name': 'Open QOTD', - 'protocol.application': 'qotd', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 17, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_quic.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_quic.py deleted file mode 100644 index 23d11ce996..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_quic.py +++ /dev/null @@ -1,118 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_quic.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible QUIC Report', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_quic-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-quic', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.source.naics' : 517311, - 'extra.tag' : 'quic', - 'extra.version_field_1' : 'Q050', - 'extra.version_field_3' : 'Q046', - 'extra.version_field_4' : 'Q043', - 'feed.name' : 'Accessible QUIC Report', - 'protocol.transport' : 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 5607, - 'source.geolocation.cc' : 'UK', - 'source.geolocation.city' : 'LONDON', - 'source.geolocation.region' : 'LONDON', - 'source.ip' : '176.255.0.0', - 'source.port' : 443, - 'source.reverse_dns' : 'test1.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T14:31:17+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-quic', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.source.naics' : 517311, - 'extra.tag' : 'quic', - 'extra.version_field_1' : 'Q050', - 'extra.version_field_2' : 'Q046', - 'extra.version_field_4' : 'Q043', - 'feed.name' : 'Accessible QUIC Report', - 'protocol.transport' : 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 6327, - 'source.geolocation.cc' : 'CA', - 'source.geolocation.city' : 'MEACHAM', - 'source.geolocation.region' : 'SASKATCHEWAN', - 'source.ip' : '24.244.0.0', - 'source.port' : 443, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T14:31:17+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-quic', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.source.naics' : 517919, - 'extra.tag' : 'quic', - 'extra.version_field_2' : 'Q050', - 'extra.version_field_3' : 'Q046', - 'extra.version_field_4' : 'Q043', - 'feed.name' : 'Accessible QUIC Report', - 'protocol.transport' : 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 20940, - 'source.geolocation.cc' : 'JP', - 'source.geolocation.city' : 'OSAKA', - 'source.geolocation.region' : 'OSAKA', - 'source.ip' : '23.60.0.0', - 'source.port' : 443, - 'source.reverse_dns' : 'test3.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T14:31:17+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_radmin.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_radmin.py deleted file mode 100644 index 7c052c451c..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_radmin.py +++ /dev/null @@ -1,236 +0,0 @@ -# SPDX-FileCopyrightText: 2020 sinus-x -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), "testdata/scan_radmin.csv")) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = { - "feed.name": "Accessible Radmin", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2020-08-19T00:00:00+00:00", - "extra.file_name": "2020-08-19-scan_radmin-test-test.csv", -} - -EVENTS = [ - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517312, - "extra.tag": "radmin", - "extra.version": "Radmin (Details Unknown)", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.asn": 701, - "source.geolocation.cc": "US", - "source.geolocation.city": "BROOKLYN", - "source.geolocation.region": "NEW YORK", - "source.ip": "74.101.218.75", - "source.port": 4899, - "source.reverse_dns": "static-74-101-218-75.nycmny.fios.verizon.net", - "time.source": "2020-07-06T13:55:26+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[1]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.tag": "radmin", - "extra.version": "Radmin v3.X Radmin Authentication", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.asn": 56618, - "source.geolocation.cc": "RU", - "source.geolocation.city": "MURMANSK", - "source.geolocation.region": "MURMANSKAYA OBLAST", - "source.ip": "192.162.189.171", - "source.port": 4899, - "source.reverse_dns": "rubin.an.ru", - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[2]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517311, - "extra.tag": "radmin", - "extra.version": "Radmin (Details Unknown)", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.geolocation.cc": "CN", - "source.geolocation.city": "BEIJING", - "source.geolocation.region": "BEIJING SHI", - "source.asn": 4808, - "source.ip": "111.197.143.69", - "source.port": 4899, - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[3]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517311, - "extra.tag": "radmin", - "extra.version": "Radmin v3.X Radmin Authentication", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.geolocation.cc": "KR", - "source.geolocation.city": "DAEIN-DONG", - "source.geolocation.region": "GWANGJU-GWANGYEOKSI", - "source.asn": 4766, - "source.ip": "121.147.215.220", - "source.port": 4899, - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[4]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517311, - "extra.tag": "radmin", - "extra.version": "Radmin v3.X Radmin Authentication", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.geolocation.cc": "KR", - "source.geolocation.city": "DAEIN-DONG", - "source.geolocation.region": "GWANGJU-GWANGYEOKSI", - "source.asn": 4766, - "source.ip": "121.147.215.178", - "source.port": 4899, - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[5]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517312, - "extra.tag": "radmin", - "extra.version": "Radmin v3.X Radmin Authentication", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.geolocation.cc": "CN", - "source.geolocation.city": "CHONGQING", - "source.geolocation.region": "CHONGQING SHI", - "source.asn": 9808, - "source.ip": "183.230.5.219", - "source.port": 4899, - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[6]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.tag": "radmin", - "extra.version": "Radmin v3.X Radmin Authentication", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.geolocation.cc": "RU", - "source.geolocation.city": "MOSCOW", - "source.geolocation.region": "MOSKVA", - "source.asn": 34300, - "source.ip": "85.93.154.74", - "source.port": 4899, - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[7]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517311, - "extra.tag": "radmin", - "extra.version": "Radmin v3.X Radmin Authentication", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.geolocation.cc": "BE", - "source.geolocation.city": "BRASSCHAAT", - "source.geolocation.region": "ANTWERPEN", - "source.asn": 5432, - "source.ip": "81.246.135.247", - "source.port": 4899, - "source.reverse_dns": "247.135-246-81.adsl-dyn.isp.belgacom.be", - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[8]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517312, - "extra.tag": "radmin", - "extra.version": "Radmin v3.X Radmin Authentication", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.geolocation.cc": "ES", - "source.geolocation.city": "LAS PALMAS DE GRAN CANARIA", - "source.geolocation.region": "LAS PALMAS", - "source.asn": 12430, - "source.ip": "46.27.146.22", - "source.port": 4899, - "source.reverse_dns": "static-22-146-27-46.ipcom.comunitel.net", - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[9]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == "__main__": - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_rdp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_rdp.py deleted file mode 100644 index 28a4a02c23..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_rdp.py +++ /dev/null @@ -1,117 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_rdp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible RDP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_rdp-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Accessible RDP', - "classification.identifier": "open-rdp", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.bluekeep_vulnerable": False, - "extra.cert_expiration_date": "2019-10-29 02:22:06", - "extra.cert_issue_date": "2019-04-29 02:22:06", - "extra.cert_length": 5678, - "extra.cert_serial_number": "1EF2B37AF850C9BF4E88F18177001D6B", - "extra.cve20190708_vulnerable": False, - "extra.issuer_common_name": "KABESRV.KABE.local", - "extra.key_algorithm": "rsaEncryption", - "extra.md5_fingerprint": "BC:6E:C3:E2:98:22:EC:BA:5B:30:E2:53:FD:4A:9D:FF", - "extra.naics": 517311, - "extra.rdp_protocol": "RDP", - "extra.sha1_fingerprint": "EC:BB:4D:DB:9F:0C:D3:FF:5B:49:EA:B1:56:62:B6:A7:5D:60:54:42", - "extra.sha256_fingerprint": "B7:C9:F4:07:D5:C0:75:1D:EA:0C:40:E7:26:39:C2:30:C6:13:83:7E:18:46:D8:E9:4C:45:3F:88:1B:0B:70:76", - "extra.sha512_fingerprint": "08:AC:75:FA:EB:A3:2B:44:15:DE:6D:A7:0B:C0:AE:17:94:F3:55:D9:EC:70:AC:5B:B7:94:79:F0:D7:84:83:89:CB:A9:11:E0:08:D7:54:4D:33:85:89:D2:A8:DD:9D:15:F4:CC:95:DE:6A:E3:DF:6B:FA:8B:27:E3:DA:16:AF:0A", - "extra.signature_algorithm": "sha256WithRSAEncryption", - "extra.ssl_version": 2, - "extra.subject_common_name": "KABESRV.KABE.local", - "extra.tag": "rdp", - "protocol.application": "rdp", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 5678, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.178", - "source.port": 5678, - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T15:45:51+00:00" - }, - {'__type': 'Event', - 'feed.name': 'Accessible RDP', - "classification.identifier": "open-rdp", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.bluekeep_vulnerable": False, - "extra.cert_expiration_date": "2019-10-16 06:15:20", - "extra.cert_issue_date": "2019-04-16 06:15:20", - "extra.cert_length": 5678, - "extra.cert_serial_number": "3FF3EBC5CF154BA54D128A8548C8AAF5", - "extra.cve20190708_vulnerable": False, - "extra.issuer_common_name": "RAMBLA01.rambla.local", - "extra.key_algorithm": "rsaEncryption", - "extra.md5_fingerprint": "38:73:6A:B3:AA:41:69:C9:BA:E7:3D:D7:40:16:F8:AA", - "extra.naics": 517311, - "extra.rdp_protocol": "RDP", - "extra.sector": "Information Technology", - "extra.sha1_fingerprint": "7A:67:1F:F8:87:C6:B0:AC:A9:84:15:B7:40:EC:CB:19:AA:E3:19:52", - "extra.sha256_fingerprint": "8F:CD:7D:C4:80:2D:8D:9B:06:A0:40:18:9F:ED:73:7A:BA:83:55:BE:1B:56:83:A2:97:DF:BB:B4:06:57:CB:F1", - "extra.sha512_fingerprint": "E8:9B:9A:93:69:B4:58:01:D8:46:C2:DC:01:20:1E:DD:93:E1:EB:E3:9D:6B:65:A0:C5:00:6C:A4:44:08:FE:A4:A6:19:FF:55:79:F2:AA:61:68:C8:1C:B0:CE:78:EB:84:DD:29:9D:64:2F:4E:25:31:3A:6C:B8:02:C9:AF:F5:1F", - "extra.signature_algorithm": "sha1WithRSAEncryption", - "extra.ssl_version": 2, - "extra.subject_common_name": "RAMBLA01.rambla.local", - "extra.tag": "rdp", - "protocol.application": "rdp", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 5678, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.233", - "source.port": 5678, - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T15:45:51+00:00" - }, - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_rdpeudp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_rdpeudp.py deleted file mode 100644 index 54be35a26f..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_rdpeudp.py +++ /dev/null @@ -1,109 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Sebastian Waldbauer -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_rdpeudp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible MS RDPEUDP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_rdpeudp-test-geo.csv", - } - -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'accessible-msrdpeudp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 77.0, - 'extra.response_size': 1232, - 'extra.sessionid': '05b28c0c', - 'extra.tag': 'rdpeudp', - 'feed.name': 'Accessible MS RDPEUDP', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 3389, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-msrdpeudp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 77.0, - 'extra.response_size': 1232, - 'extra.sessionid': '053d355f', - 'extra.tag': 'rdpeudp', - 'feed.name': 'Accessible MS RDPEUDP', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 3389, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-msrdpeudp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 77.0, - 'extra.response_size': 1232, - 'extra.sessionid': '0567a8cb', - 'extra.tag': 'rdpeudp', - 'feed.name': 'Accessible MS RDPEUDP', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 3389, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_redis.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_redis.py deleted file mode 100644 index 04552e2ec0..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_redis.py +++ /dev/null @@ -1,107 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_redis.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open Redis', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_redis-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Open Redis', - "classification.identifier": "open-redis", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.build_id": "26069fb482f6334b", - "extra.connected_clients": "50", - "extra.gcc_version": "4.7.2", - "extra.git_sha1": "00000000", - "extra.mode": "standalone", - "extra.multiplexing_api": "epoll", - "extra.naics": 541512, - "extra.os.name": "Linux 3.2.0-4-amd64 x86_64", - "extra.process_id": "2127", - "extra.run_id": "d440b0b2fb3d1db655ad607e11e6f38011a0f599", - "extra.sic": 737999, - "extra.tag": "redis", - "extra.uptime": 27946314, - "extra.version": "2.8.19", - "protocol.application": "redis", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 201229, - "source.geolocation.cc": "DE", - "source.geolocation.city": "FRANKFURT AM MAIN", - "source.geolocation.region": "HESSEN", - "source.ip": "198.51.100.152", - "source.port": 6379, - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2016-07-24T00:42:33+00:00" - }, - {'__type': 'Event', - 'feed.name': 'Open Redis', - "classification.identifier": "open-redis", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.build_id": "e41bf84a0cecf09d", - "extra.connected_clients": "25376", - "extra.gcc_version": "4.8.4", - "extra.git_sha1": "00000000", - "extra.mode": "standalone", - "extra.multiplexing_api": "epoll", - "extra.os.name": "Linux 3.18.24-sirzion x86_64", - "extra.process_id": "343519", - "extra.run_id": "53d63f23511dc0080b49aaa8e8203d65619f1c8c", - "extra.tag": "redis", - "extra.uptime": 310556, - "extra.version": "3.0.6", - "protocol.application": "redis", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 12586, - "source.geolocation.cc": "DE", - "source.geolocation.city": "FRANKFURT AM MAIN", - "source.geolocation.region": "HESSEN", - "source.ip": "198.51.100.67", - "source.port": 6379, - "source.reverse_dns": "198-51-100-67.example.net", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2016-07-24T00:42:43+00:00" - }, - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_rsync.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_rsync.py deleted file mode 100644 index e2a961f710..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_rsync.py +++ /dev/null @@ -1,116 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_rsync.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible Rsync', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_rsync-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'accessible-rsync', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.has_password': 'N', - 'extra.module': 'system|Backup system;system_full|Backup full ' - 'system;mysql|Backup virtual mysql;netadmin|Backup virtual ' - 'netadmin;', - 'extra.tag': 'rsync', - 'feed.name': 'Accessible Rsync', - 'protocol.application': 'rsync', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 873, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-rsync', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.has_password': 'N', - 'extra.module': 'system|Backup system;system_full|Backup full ' - 'system;mysql|Backup virtual mysql;netadmin|Backup virtual ' - 'netadmin;', - 'extra.tag': 'rsync', - 'feed.name': 'Accessible Rsync', - 'protocol.application': 'rsync', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 873, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-rsync', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.has_password': 'N', - 'extra.module': 'system|Backup system;system_full|Backup full ' - 'system;mysql|Backup virtual mysql;netadmin|Backup virtual ' - 'netadmin;', - 'extra.tag': 'rsync', - 'feed.name': 'Accessible Rsync', - 'protocol.application': 'rsync', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 873, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_sip.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_sip.py deleted file mode 100644 index 6b972ec5d5..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_sip.py +++ /dev/null @@ -1,124 +0,0 @@ -# SPDX-FileCopyrightText: 2023 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_sip.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible-SIP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_sip-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-sip', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.sip_allow': 'INVITE,ACK,BYE,CANCEL,REGISTER', - 'extra.amplification': 15.57, - 'extra.content_length': 0, - 'extra.response_size': 109, - 'extra.sip': 'SIP/2.0', - 'extra.sip_code': '489', - 'extra.sip_reason': 'Event Package Not Supported', - 'extra.tag': 'sip', - 'feed.name': 'Accessible-SIP', - 'protocol.application': 'sip', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 5060, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-sip', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 62.57, - 'extra.content_length': 364, - 'extra.content_type': 'text/plain', - 'extra.response_size': 438, - 'extra.sip': 'SIP/2.0', - 'extra.sip_code': '400', - 'extra.sip_reason': 'Bad Request', - 'extra.tag': 'sip', - 'feed.name': 'Accessible-SIP', - 'protocol.application': 'sip', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 5060, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-sip', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 6.57, - 'extra.content_length': 0, - 'extra.response_size': 46, - 'extra.sip': 'SIP/2.0', - 'extra.sip_code': '400', - 'extra.sip_reason': 'Bad Request', - 'extra.tag': 'sip', - 'feed.name': 'Accessible-SIP', - 'protocol.application': 'sip', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 5060, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_slp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_slp.py deleted file mode 100644 index f05973cf5c..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_slp.py +++ /dev/null @@ -1,137 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_slp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible SLP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_slp-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-slp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.error_code': '5', - 'extra.error_code_text': 'Unsupported SLP SPI', - 'extra.flags': '0x0000', - 'extra.function': '2', - 'extra.function_text': 'Service reply', - 'extra.language_tag': 'en', - 'extra.language_tag_length': '2', - 'extra.raw_response': 'MDIwMjAwMDAxNDAwMDAwMDAwMDAwMDA1MDAwMjY1NmUwMDA1MDAwMA==', - 'extra.response_size': 40, - 'extra.tag': 'slp', - 'extra.version': '2', - 'extra.xid': '5', - 'feed.name': 'Accessible SLP', - 'protocol.application': 'slp', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 427, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-slp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.error_code': '5', - 'extra.error_code_text': 'Unsupported SLP SPI', - 'extra.flags': '0x0000', - 'extra.function': '2', - 'extra.function_text': 'Service reply', - 'extra.language_tag': 'en', - 'extra.language_tag_length': '2', - 'extra.raw_response': 'MDIwMjAwMDAxNDAwMDAwMDAwMDAwMDA1MDAwMjY1NmUwMDA1MDAwMA==', - 'extra.response_size': 40, - 'extra.tag': 'slp', - 'extra.version': '2', - 'extra.xid': '5', - 'feed.name': 'Accessible SLP', - 'protocol.application': 'slp', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 427, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-slp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.error_code': '5', - 'extra.error_code_text': 'Unsupported SLP SPI', - 'extra.flags': '0x0000', - 'extra.function': '2', - 'extra.function_text': 'Service reply', - 'extra.language_tag': 'en', - 'extra.language_tag_length': '2', - 'extra.raw_response': 'MDIwMjAwMDAxNDAwMDAwMDAwMDAwMDA1MDAwMjY1NmUwMDA1MDAwMA==', - 'extra.response_size': 40, - 'extra.tag': 'slp', - 'extra.version': '2', - 'extra.xid': '5', - 'feed.name': 'Accessible SLP', - 'protocol.application': 'slp', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 427, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_smb.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_smb.py deleted file mode 100644 index 921525122c..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_smb.py +++ /dev/null @@ -1,124 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_smb.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible SMB', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_smb-test-geo.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Accessible SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 445, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Accessible SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 445, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Accessible SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 445, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_smb_json.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_smb_json.py deleted file mode 100644 index cae83d2733..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_smb_json.py +++ /dev/null @@ -1,123 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Sebastian Waldbauer -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest -import json - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser_json import ShadowserverJSONParserBot -from intelmq.tests.bots.parsers.shadowserver.test_testdata import csvtojson - -EXAMPLE_FILE = csvtojson(os.path.join(os.path.dirname(__file__), 'testdata/scan_smb.csv')) - -EXAMPLE_REPORT = { - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_smb-test-geo.json", - } - -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Accessible-SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode(json.dumps([json.loads(EXAMPLE_FILE)[0]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 445, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Accessible-SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode(json.dumps([json.loads(EXAMPLE_FILE)[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 445, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Accessible-SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode(json.dumps([json.loads(EXAMPLE_FILE)[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 445, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverJSONParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverJSONParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverJSONParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_smtp_vulnerable.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_smtp_vulnerable.py deleted file mode 100644 index 4428420cfb..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_smtp_vulnerable.py +++ /dev/null @@ -1,92 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Mikk Margus Möll -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_smtp_vulnerable.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Vulnerable SMTP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2021-07-08T00:00:00+00:00", - "extra.file_name": "2021-07-08-scan_smtp_vulnerable-test-test.csv", - } - -EVENTS = [ - { - '__type' : 'Event', - 'classification.identifier' : 'vulnerable-smtp', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.banner' : '220 smtp-server.invalid ESMTP Exim 4.80 Wed, 11 Jun 2021 10:00:00 +0300|', - 'extra.tag' : 'smtp;21nails', - 'feed.name' : 'Vulnerable SMTP', - 'protocol.application' : 'smtp', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 12345, - 'source.geolocation.cc' : 'EE', - 'source.geolocation.city' : 'TALLINN', - 'source.geolocation.region' : 'HARJUMAA', - 'source.ip' : '1.2.3.4', - 'source.port' : 25, - 'source.reverse_dns' : 'smtp-server.invalid', - 'time.observation' : '2021-07-08T00:00:00+00:00', - 'time.source' : '2021-07-08T11:58:42+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'vulnerable-smtp', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.banner' : '220 smtp-out.invalid, ESMTP EXIM 4.86_2|', - 'extra.tag' : 'smtp;21nails', - 'feed.name' : 'Vulnerable SMTP', - 'protocol.application' : 'smtp', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 23456, - 'source.geolocation.cc' : 'EE', - 'source.geolocation.city' : 'TALLINN', - 'source.geolocation.region' : 'HARJUMAA', - 'source.ip' : '5.6.7.8', - 'source.port' : 25, - 'source.reverse_dns' : 'smtp-out.invalid', - 'time.observation' : '2021-07-08T00:00:00+00:00', - 'time.source' : '2021-07-08T11:58:44+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_snmp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_snmp.py deleted file mode 100644 index e6da5b34f9..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_snmp.py +++ /dev/null @@ -1,120 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_snmp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open SNMP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_snmp-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-snmp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 1.94, - 'extra.community': 'public', - 'extra.response_size': 165, - 'extra.sysdesc': 'Linux localhost 3.18.20 #1 SMP Mon Jul 9 14:11:21 CST 2018 ' - 'armv7l', - 'extra.tag': 'snmp', - 'extra.version': '2', - 'feed.name': 'Open SNMP', - 'protocol.application': 'snmp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 161, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-snmp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 1.35, - 'extra.community': 'public', - 'extra.device_sector': 'consumer', - 'extra.device_type': 'router', - 'extra.device_vendor': 'MikroTik', - 'extra.response_size': 115, - 'extra.sysdesc': 'RouterOS CCR1009-8G-1S-1S+', - 'extra.tag': 'snmp,iot', - 'extra.version': '2', - 'feed.name': 'Open SNMP', - 'protocol.application': 'snmp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 161, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-snmp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 1.0, - 'extra.community': 'public', - 'extra.response_size': 85, - 'extra.tag': 'snmp', - 'extra.version': '2', - 'feed.name': 'Open SNMP', - 'protocol.application': 'snmp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 161, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_socks.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_socks.py deleted file mode 100644 index 067602aa10..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_socks.py +++ /dev/null @@ -1,107 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_socks.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open SOCKS', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_socks-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-socks', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'feed.name' : 'Open SOCKS', - 'protocol.application' : 'socks4', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 1080, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-socks', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'feed.name' : 'Open SOCKS', - 'protocol.application' : 'socks5', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 1080, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-socks', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.source.sector' : 'Retail Trade', - 'feed.name' : 'Open SOCKS', - 'protocol.application' : 'socks4', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 1080, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssdp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ssdp.py deleted file mode 100644 index 0811f15eda..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssdp.py +++ /dev/null @@ -1,136 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ssdp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open SSDP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ssdp-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-ssdp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3.35, - 'extra.cache_control': 'max-age=100', - 'extra.header': 'HTTP/1.1 200 OK', - 'extra.host': 'node01.example.com', - 'extra.location': 'http://192.168.200.254:49152/description.xml', - 'extra.response_size': 325, - 'extra.search_target': 'upnp:rootdevice', - 'extra.sector': 'Government', - 'extra.server': 'Linux/2.6.26, UPnP/1.0, Portable SDK for UPnP devices/1.3.1', - 'extra.systime': 'Sun, 21 Aug 2022 09:51:13 GMT', - 'extra.tag': 'ssdp', - 'extra.unique_service_name': 'uuid:28802880-2880-1880-a880-001bc502f600::upnp:rootdevice', - 'feed.name': 'Open SSDP', - 'protocol.application': 'ssdp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 60194, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-ssdp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 2.71, - 'extra.cache_control': 'max-age = 1800', - 'extra.header': 'HTTP/1.1 200 OK', - 'extra.host': 'node02.example.com', - 'extra.location': 'http://95.160.216.14:52235/dmr/SamsungMRDesc.xml', - 'extra.response_size': 263, - 'extra.search_target': 'upnp:rootdevice', - 'extra.server': 'Linux/9.0 UPnP/1.0 PROTOTYPE/1.0', - 'extra.tag': 'ssdp', - 'extra.unique_service_name': 'uuid:f144ca92-6816-94b5-b95f-b58180834044::upnp:rootdevice', - 'feed.name': 'Open SSDP', - 'protocol.application': 'ssdp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 38732, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-ssdp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 4.79, - 'extra.cache_control': 'max-age=1800', - 'extra.header': 'HTTP/1.1 200 OK', - 'extra.host': 'node03.example.com', - 'extra.location': 'http://192.168.1.3:8008/ssdp/device-desc.xml', - 'extra.response_size': 465, - 'extra.search_target': 'upnp:rootdevice', - 'extra.sector': 'Government', - 'extra.server': 'Linux/3.10.79, UPnP/1.0, Portable SDK for UPnP ' - 'devices/1.6.18', - 'extra.systime': 'Sun, 03 Jan 2016 21:37:50 GMT', - 'extra.tag': 'ssdp', - 'extra.unique_service_name': 'uuid:62fa0fc8-079d-d00f-2e22-59b49fb488f9::upnp:rootdevice', - 'feed.name': 'Open SSDP', - 'protocol.application': 'ssdp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 57626, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssh.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ssh.py deleted file mode 100644 index a01383713b..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssh.py +++ /dev/null @@ -1,182 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ssh.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible SSH', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_ssh-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ssh', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.algorithm' : 'ecdsa-sha2-nistp256', - 'extra.available_ciphers' : 'chacha20-poly1305@openssh.com, aes128-ctr, aes192-ctr, aes256-ctr, aes128-gcm@openssh.com, aes256-gcm@openssh.com, aes128-cbc, aes192-cbc, aes256-cbc, blowfish-cbc, cast128-cbc, 3des-cbc', - 'extra.available_compression' : 'none, zlib@openssh.com', - 'extra.available_kex' : 'curve25519-sha256, curve25519-sha256@libssh.org, ecdh-sha2-nistp256, ecdh-sha2-nistp384, ecdh-sha2-nistp521, diffie-hellman-group-exchange-sha256, diffie-hellman-group16-sha512, diffie-hellman-group18-sha512, diffie-hellman-group-exchange-sha1, diffie-hellman-group14-sha256, diffie-hellman-group14-sha1, diffie-hellman-group1-sha1', - 'extra.available_mac' : 'umac-64-etm@openssh.com, umac-128-etm@openssh.com, hmac-sha2-256-etm@openssh.com, hmac-sha2-512-etm@openssh.com, hmac-sha1-etm@openssh.com, umac-64@openssh.com, umac-128@openssh.com, hmac-sha2-256, hmac-sha2-512, hmac-sha1', - 'extra.ecdsa_curve' : 'P-256', - 'extra.ecdsa_curve25519' : '1xx7ASut7BF4ED8b592bebZBMBKTCzOsmbH4cjwx/0U=', - 'extra.ecdsa_public_key_b' : 'WsY12Ko6k+ez671VdpiGvGUdBrDMU7D2O848PifSYEs=', - 'extra.ecdsa_public_key_gx' : 'axfR8uEsQkf4vOblY6RA8ncDfYEt6zOg9KE5RdiYwpY=', - 'extra.ecdsa_public_key_gy' : 'T+NC4v4af5uO5+tKfA+eFivOM1drMV7Oy7ZAaDe/UfU=', - 'extra.ecdsa_public_key_length' : '256', - 'extra.ecdsa_public_key_n' : '/////wAAAAD//////////7zm+q2nF56E87nKwvxjJVE=', - 'extra.ecdsa_public_key_p' : '/////wAAAAEAAAAAAAAAAAAAAAD///////////////8=', - 'extra.ecdsa_public_key_x' : 'NIQdotpy2HAHfSu0DjEmA3cb3NeQKabZaFX9OU0GsPY=', - 'extra.ecdsa_public_key_y' : '0fuNQAZX7XciX2YkqIHtK2dWLBYwVCCqvl//zoM42kI=', - 'extra.selected_cipher' : 'aes128-ctr', - 'extra.selected_compression' : 'none', - 'extra.selected_kex' : 'curve25519-sha256@libssh.org', - 'extra.selected_mac' : 'hmac-sha2-256', - 'extra.server_cookie' : 'bGjsifbPIDWT7tAu8BMjyg==', - 'extra.server_host_key' : 'AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDSEHaLacthwB30rtA4xJgN3G9zXkCmm2WhV/TlNBrD20fuNQAZX7XciX2YkqIHtK2dWLBYwVCCqvl//zoM42kI=', - 'extra.server_host_key_sha256' : 'a6e4e1c16ba25d51bcddc58a6e16797144575dd18d02d9dedf75093d2b15c557', - 'extra.server_signature_raw' : 'AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAABKAAAAIQCd+X/B/OEx+FrwJSlVecOvNMuS5w2vTRz0z4prM+5VBwAAACEArU60b9CHs/d5BgyaOd7vmFygTMK5SyL90bS8VIztX/4=', - 'extra.server_signature_value' : 'AAAAIQCd+X/B/OEx+FrwJSlVecOvNMuS5w2vTRz0z4prM+5VBwAAACEArU60b9CHs/d5BgyaOd7vmFygTMK5SyL90bS8VIztX/4=', - 'extra.serverid_raw' : 'SSH-2.0-OpenSSH_7.4', - 'extra.serverid_software' : 'OpenSSH_7.4', - 'extra.serverid_version' : '2.0', - 'extra.source.naics' : 454110, - 'extra.tag' : 'ssh', - 'extra.userauth_methods' : 'publickey', - 'feed.name' : 'Accessible SSH', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 16509, - 'source.geolocation.cc' : 'JP', - 'source.geolocation.city' : 'TOKYO', - 'source.geolocation.region' : 'TOKYO', - 'source.ip' : '18.179.0.0', - 'source.port' : 22, - 'source.reverse_dns' : 'ec2-18-179-0-0.ap-northeast-1.compute.amazonaws.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T02:20:37+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ssh', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.algorithm' : 'ssh-rsa', - 'extra.available_ciphers' : 'aes128-cbc, 3des-cbc, aes256-cbc, twofish256-cbc, twofish-cbc, twofish128-cbc, blowfish-cbc', - 'extra.available_compression' : 'none', - 'extra.available_kex' : 'diffie-hellman-group1-sha1', - 'extra.available_mac' : 'hmac-sha1-96, hmac-sha1, hmac-md5', - 'extra.device_vendor' : 'Arris', - 'extra.rsa_exponent' : '65537', - 'extra.rsa_length' : '1040', - 'extra.rsa_modulus' : 'g6tZBqqsHUFa8gjXKMg2waZy/6JLzNnfTZkdDo6a3E6Amdk2eVlGbdgRsDxNUCyWLJI4b+xsJX0OKoRKhw4N+cDK5Du4/S00Auylt8YrDi7lIpeBZjMT5KJerN/feOcwRqY8NJfwZiZ1A5brcWr5HXfOH6aRTMFzadr248VdUmFXPQ==', - 'extra.selected_cipher' : 'aes128-cbc', - 'extra.selected_compression' : 'none', - 'extra.selected_kex' : 'diffie-hellman-group1-sha1', - 'extra.selected_mac' : 'hmac-sha1', - 'extra.server_cookie' : 'Y4RQS9sdRgEFwNJKVP6bZg==', - 'extra.server_host_key' : 'AAAAB3NzaC1yc2EAAAADAQABAAAAgwCDq1kGqqwdQVryCNcoyDbBpnL/okvM2d9NmR0OjprcToCZ2TZ5WUZt2BGwPE1QLJYskjhv7GwlfQ4qhEqHDg35wMrkO7j9LTQC7KW3xisOLuUil4FmMxPkol6s39945zBGpjw0l/BmJnUDlutxavkdd84fppFMwXNp2vbjxV1SYVc9', - 'extra.server_host_key_sha256' : 'd53fedbfe92e631264629882b2e85bfd213ca4b07b824cd31f8de1fcb8d0ddcb', - 'extra.server_signature_raw' : 'AAAAB3NzaC1yc2EAAACCLQj+UTJEQqdb/p/c/19yVc63eo+rnedwXKjP6eNNxxijN2cFoOjVMeqT2QTBjyoN7yRWBU2EID+3y2jUYT8mCqmqfyUv1eEbiCfLVlUyQ0X/CY9I5DDb5l6yEjNkuH2xVNNV6R7GFRwyYKAsYzfy+i9o1OORlUh3tozkkPfA9z/NlA==', - 'extra.server_signature_value' : 'LQj+UTJEQqdb/p/c/19yVc63eo+rnedwXKjP6eNNxxijN2cFoOjVMeqT2QTBjyoN7yRWBU2EID+3y2jUYT8mCqmqfyUv1eEbiCfLVlUyQ0X/CY9I5DDb5l6yEjNkuH2xVNNV6R7GFRwyYKAsYzfy+i9o1OORlUh3tozkkPfA9z/NlA==', - 'extra.serverid_raw' : 'SSH-2.0-ARRIS_0.50', - 'extra.serverid_software' : 'ARRIS_0.50', - 'extra.serverid_version' : '2.0', - 'extra.tag' : 'ssh', - 'extra.userauth_methods' : 'publickey, password', - 'feed.name' : 'Accessible SSH', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 11976, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'MARSHALL', - 'source.geolocation.region' : 'TEXAS', - 'source.ip' : '170.10.0.0', - 'source.port' : 22, - 'source.reverse_dns' : '170-10-0-0.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T02:20:37+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ssh', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.algorithm' : 'ssh-rsa', - 'extra.available_ciphers' : 'aes128-cbc, 3des-cbc, aes192-cbc, aes256-cbc', - 'extra.available_compression' : 'none', - 'extra.available_kex' : 'diffie-hellman-group-exchange-sha1, diffie-hellman-group14-sha1, diffie-hellman-group1-sha1', - 'extra.available_mac' : 'hmac-sha1, hmac-sha1-96, hmac-md5, hmac-md5-96', - 'extra.device_sector' : 'enterprise', - 'extra.device_vendor' : 'Cisco', - 'extra.rsa_exponent' : '65537', - 'extra.rsa_length' : '4096', - 'extra.rsa_modulus' : 'yFVwcChoYt+YGm8BzWYugcZbNQRrQ1VWRYcL4U6SSkyoVeE9h5wxRu/hQaWHo3PdsB9Nuln/riRyKZypFUEZ5zlffMyl1uvE8/jp8E/GgUHSyPkGAwu8C8BkX/nDolxAJKTK6djiZnvhsEPe6AXHBMHbto/b3GABUNPngjzX8D63GYcFW9NJLf5qC1UsVkXbAzM0IjQ2X9s3pfhUCAJeXAn2i0gEGtUyF8vEjNdwdG655aXciKrpEEtM1L/zy/+gLH4YC13kAYI7NVyH+qi/mXbULLOQClA7iYK1g3Et58jWUIPwgLfF3SLC57bt2wp/lRgNTv4FBi0tWvRqBnf5UQK5ZjgzbW3bO+Ju4cWgH/4M4NCxSceh4cLm5lQs01xB5feSh2ByqA7wrVDoFJu81LoMVo4bCz30+lH2QsLwmNtUhlWLKBD4k09g4bgBa4jPj0/Nya3rBR4GQ6LG6ltFQotm8wCkgbv76YWqk20nQ6NMYZFvSQm981JFtoHv3vxq48VeHDV0QvV0P12BCFprRf4B0otIvSsHl+LDeUxJAf+Nbw78gzncjyfCbWtCPbwaJQ8CeqnTBzj5TluaFvN8goG5lCTWJGfjIrwAZXOokv9NOqmIiMJJx3s22OX6GHfJAzje2ALLDsAiXBub4iCOdGdTfVbBpFL+bGTK9qfa8vE=', - 'extra.selected_cipher' : 'aes128-cbc', - 'extra.selected_compression' : 'none', - 'extra.selected_kex' : 'diffie-hellman-group14-sha1', - 'extra.selected_mac' : 'hmac-sha1', - 'extra.server_cookie' : 'Z2fOfWsrLlh76Y0bOqa1cw==', - 'extra.server_host_key' : 'AAAAB3NzaC1yc2EAAAADAQABAAACAQDIVXBwKGhi35gabwHNZi6Bxls1BGtDVVZFhwvhTpJKTKhV4T2HnDFG7+FBpYejc92wH026Wf+uJHIpnKkVQRnnOV98zKXW68Tz+OnwT8aBQdLI+QYDC7wLwGRf+cOiXEAkpMrp2OJme+GwQ97oBccEwdu2j9vcYAFQ0+eCPNfwPrcZhwVb00kt/moLVSxWRdsDMzQiNDZf2zel+FQIAl5cCfaLSAQa1TIXy8SM13B0brnlpdyIqukQS0zUv/PL/6AsfhgLXeQBgjs1XIf6qL+ZdtQss5AKUDuJgrWDcS3nyNZQg/CAt8XdIsLntu3bCn+VGA1O/gUGLS1a9GoGd/lRArlmODNtbds74m7hxaAf/gzg0LFJx6HhwubmVCzTXEHl95KHYHKoDvCtUOgUm7zUugxWjhsLPfT6UfZCwvCY21SGVYsoEPiTT2DhuAFriM+PT83JresFHgZDosbqW0VCi2bzAKSBu/vphaqTbSdDo0xhkW9JCb3zUkW2ge/e/GrjxV4cNXRC9XQ/XYEIWmtF/gHSi0i9KweX4sN5TEkB/41vDvyDOdyPJ8Jta0I9vBolDwJ6qdMHOPlOW5oW83yCgbmUJNYkZ+MivABlc6iS/006qYiIwknHezbY5foYd8kDON7YAssOwCJcG5viII50Z1N9VsGkUv5sZMr2p9ry8Q==', - 'extra.server_host_key_sha256' : '06ff3cce443ed832927576d982b69d5a526d0e63334c72e87201deda61679406', - 'extra.server_signature_raw' : 'AAAAB3NzaC1yc2EAAAIAlrzL2DY9fVvwYg6CgB75uf2s8CLo+rL8Mp9tU1Ja3sDfBzj9QJjVDykupiy8s3usHfxMrHS2v3DhTiZjz/b5K6tVTgUBTXL94JfM4lwB+3EbLggPzKnlm1jQgnnU9c+tb7RX3IhBqU9Yj1gqxhErv9NFotgajQOOLgY0Ua5C0Ee+AIaMlLaNZe3LTejMsNUZMN5tl+sEmtutMHkGQsmjJxiJ3feF+Pys0I2+ojiiAfzqlMYar/5xOPl4Dj+HO+h91xVQ1/8nQRBc082fM7+ZJtDbRLtt4G8srlB5gew26jqfVASc/ui5gx4+BR9DG9VH8w+rJWBGfhOAaWqLFE2M3YuEWkjEmQMR1SQK1WFQ/oNiWJO2K5L3rk2LcAmyR6nQMtClVxYZ7CQOwa3uFL+JNXp9AhiiAtVaqhrEK81NJrJNh/+egTBl5STphxIShXd4KI9wyvkGlCIvNIMO94iXPVaWUXXbsGnU03+dsUkBzGf0eJ4DePInCk/RtunlSmOsjGld+rpS9g0VRxPrzbQRWuhpkgpV+CldyrI3C/rOxJRs2vSAKXocRsGwhqEKseAJzHXmiZ5ncsaGKoeB5lUkWLwcKjyok2tHVCDlzDUpE4aA/JHNEhT48det9RqtjC71yz8m0PeK2ySI/I+Qb7eBgevgduBmt+OUxgvfKi2UB6s=', - 'extra.server_signature_value' : 'lrzL2DY9fVvwYg6CgB75uf2s8CLo+rL8Mp9tU1Ja3sDfBzj9QJjVDykupiy8s3usHfxMrHS2v3DhTiZjz/b5K6tVTgUBTXL94JfM4lwB+3EbLggPzKnlm1jQgnnU9c+tb7RX3IhBqU9Yj1gqxhErv9NFotgajQOOLgY0Ua5C0Ee+AIaMlLaNZe3LTejMsNUZMN5tl+sEmtutMHkGQsmjJxiJ3feF+Pys0I2+ojiiAfzqlMYar/5xOPl4Dj+HO+h91xVQ1/8nQRBc082fM7+ZJtDbRLtt4G8srlB5gew26jqfVASc/ui5gx4+BR9DG9VH8w+rJWBGfhOAaWqLFE2M3YuEWkjEmQMR1SQK1WFQ/oNiWJO2K5L3rk2LcAmyR6nQMtClVxYZ7CQOwa3uFL+JNXp9AhiiAtVaqhrEK81NJrJNh/+egTBl5STphxIShXd4KI9wyvkGlCIvNIMO94iXPVaWUXXbsGnU03+dsUkBzGf0eJ4DePInCk/RtunlSmOsjGld+rpS9g0VRxPrzbQRWuhpkgpV+CldyrI3C/rOxJRs2vSAKXocRsGwhqEKseAJzHXmiZ5ncsaGKoeB5lUkWLwcKjyok2tHVCDlzDUpE4aA/JHNEhT48det9RqtjC71yz8m0PeK2ySI/I+Qb7eBgevgduBmt+OUxgvfKi2UB6s=', - 'extra.serverid_raw' : 'SSH-1.99-Cisco-1.25', - 'extra.serverid_software' : 'Cisco-1.25', - 'extra.serverid_version' : '1.99', - 'extra.source.naics' : 517311, - 'extra.tag' : 'ssh', - 'extra.userauth_methods' : 'publickey, keyboard-interactive, password', - 'feed.name' : 'Accessible SSH', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 33363, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'ORLANDO', - 'source.geolocation.region' : 'FLORIDA', - 'source.ip' : '72.17.0.0', - 'source.port' : 22, - 'source.reverse_dns' : '072-017-0-0.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T02:20:37+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl.py deleted file mode 100644 index f96c03e567..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl.py +++ /dev/null @@ -1,218 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ssl.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible SSL', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_ssl-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ssl', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.browser_error' : 'x509: unknown error', - 'extra.browser_trusted' : False, - 'extra.cert_expiration_date' : '2038-01-19 03:14:07', - 'extra.cert_expired' : False, - 'extra.cert_issue_date' : '2014-06-23 09:56:32', - 'extra.cert_length' : 1024, - 'extra.cert_serial_number' : '168CAE', - 'extra.cert_valid' : True, - 'extra.cipher_suite' : 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA', - 'extra.content_length' : 131, - 'extra.content_type' : 'text/html', - 'extra.freak_vulnerable' : False, - 'extra.handshake' : 'TLSv1.2', - 'extra.http_code' : 200, - 'extra.http_date' : '2022-01-10T00:01:44+00:00', - 'extra.http_reason' : 'OK', - 'extra.http_response_type' : 'HTTP/1.1', - 'extra.issuer_common_name' : 'support', - 'extra.issuer_country' : 'US', - 'extra.issuer_email_address' : 'support@fortinet.com', - 'extra.issuer_locality_name' : 'Sunnyvale', - 'extra.issuer_organization_name' : 'Fortinet', - 'extra.issuer_organization_unit_name' : 'Certificate Authority', - 'extra.issuer_state_or_province_name' : 'California', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : '99:45:1F:2E:AE:EB:88:91:27:43:33:79:FA:93:7D:CA', - 'extra.self_signed' : False, - 'extra.server_type' : 'xxxxxxxx-xxxxx', - 'extra.sha1_fingerprint' : '5A:3D:FF:06:F9:E9:25:37:57:F9:09:52:33:A4:85:15:24:2D:88:7F', - 'extra.sha256_fingerprint' : '35:AB:B6:76:2A:3D:17:B2:FB:40:45:1B:FC:0A:99:0A:6E:48:57:F7:30:0A:3B:B1:1A:E6:99:70:5B:7C:32:41', - 'extra.sha512_fingerprint' : '88:7B:16:DB:39:44:0C:47:0E:4A:8F:0B:C5:FB:4D:45:BC:93:5A:00:43:A1:D9:7F:05:1D:86:33:02:F8:FC:57:67:A6:1D:C0:FF:F7:D2:40:D8:9A:21:AE:4E:6D:DC:E7:FF:72:BF:13:CB:EE:A7:5F:CD:83:EA:8A:5E:FB:87:DD', - 'extra.signature_algorithm' : 'sha1WithRSAEncryption', - 'extra.source.naics' : 517311, - 'extra.ssl_poodle' : False, - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'FGT60D4614030700', - 'extra.subject_country' : 'US', - 'extra.subject_email_address' : 'support@fortinet.com', - 'extra.subject_locality_name' : 'Sunnyvale', - 'extra.subject_organization_name' : 'Fortinet', - 'extra.subject_organization_unit_name' : 'FortiGate', - 'extra.subject_state_or_province_name' : 'California', - 'extra.tag' : 'ssl,vpn', - 'feed.name' : 'Accessible SSL', - 'protocol.application': 'https', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 4181, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'MILWAUKEE', - 'source.geolocation.region' : 'WISCONSIN', - 'source.ip' : '96.60.0.0', - 'source.port' : 10443, - 'source.reverse_dns' : '96-60-0-0.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:42+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ssl', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.browser_error' : 'x509: unknown error', - 'extra.browser_trusted' : False, - 'extra.cert_expiration_date' : '2023-02-06 01:01:34', - 'extra.cert_expired' : False, - 'extra.cert_issue_date' : '2022-01-04 01:01:34', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : '36974C4C6B1B3785', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', - 'extra.content_type' : 'text/html; charset=UTF-8', - 'extra.freak_vulnerable' : False, - 'extra.handshake' : 'TLSv1.2', - 'extra.http_code' : 200, - 'extra.http_connection' : 'keep-alive', - 'extra.http_date' : '2022-01-10T00:01:44+00:00', - 'extra.http_reason' : 'OK', - 'extra.http_response_type' : 'HTTP/1.1', - 'extra.issuer_common_name' : '1078-btb-tbi-HungHa-61d39c6d5a7e2', - 'extra.issuer_organization_name' : 'pfSense webConfigurator Self-Signed Certificate', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : '16:93:9A:F4:35:7F:9A:85:45:71:91:C7:7C:80:88:00', - 'extra.self_signed' : True, - 'extra.server_type' : 'nginx', - 'extra.set_cookie' : 'PHPSESSID=e15bdfa5739c36877608eb4cf46cc388; path=/; secure; HttpO', - 'extra.sha1_fingerprint' : 'A9:00:BB:E1:54:4D:56:54:59:F1:B7:EA:F1:1A:D5:36:5C:63:90:8E', - 'extra.sha256_fingerprint' : '38:85:F0:44:1E:AD:84:B8:2F:43:68:BA:AC:EE:17:13:A4:BF:86:1D:48:75:7E:22:FA:08:4C:28:5F:AC:3E:5F', - 'extra.sha512_fingerprint' : 'AE:1B:4F:D1:E4:C0:35:9D:2A:4F:7A:37:B8:7B:11:9D:84:25:23:21:AB:EF:B2:0F:DC:C9:F2:A3:72:28:92:E1:74:72:FA:E1:09:6C:E1:F6:B6:E3:A7:61:1C:58:89:34:D7:06:5C:3D:0A:A7:F6:CC:8A:D6:24:D0:04:4C:03:02', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.source.naics' : 517311, - 'extra.ssl_poodle' : False, - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : '1078-btb-tbi-HungHa-61d39c6d5a7e2', - 'extra.subject_organization_name' : 'pfSense webConfigurator Self-Signed Certificate', - 'extra.tag' : 'ssl', - 'extra.transfer_encoding' : 'chunked', - 'feed.name' : 'Accessible SSL', - 'protocol.application': 'https', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 45899, - 'source.geolocation.cc' : 'VN', - 'source.geolocation.city' : 'THAI BINH', - 'source.geolocation.region' : 'THAI BINH', - 'source.ip' : '113.160.0.0', - 'source.port' : 10443, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:42+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ssl', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.browser_trusted' : True, - 'extra.cert_expiration_date' : '2022-11-06 15:30:28', - 'extra.cert_expired' : False, - 'extra.cert_issue_date' : '2021-10-07 15:30:28', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : '7B388364A24B88E77E5553B5C6748100', - 'extra.cert_valid' : True, - 'extra.cipher_suite' : 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA', - 'extra.content_length' : 131, - 'extra.content_type' : 'text/html', - 'extra.freak_vulnerable' : False, - 'extra.handshake' : 'TLSv1.2', - 'extra.http_code' : 200, - 'extra.http_date' : '2022-01-10T00:01:44+00:00', - 'extra.http_reason' : 'OK', - 'extra.http_response_type' : 'HTTP/1.1', - 'extra.issuer_common_name' : 'Entrust Certification Authority - L1K', - 'extra.issuer_country' : 'US', - 'extra.issuer_organization_name' : 'Entrust, Inc.', - 'extra.issuer_organization_unit_name' : '(c) 2012 Entrust, Inc. - for authorized use only', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'E7:34:BC:92:84:FA:39:DE:E1:46:6C:27:DA:5A:01:F4', - 'extra.self_signed' : False, - 'extra.server_type' : 'xxxxxxxx-xxxxx', - 'extra.sha1_fingerprint' : 'AD:19:B2:1C:CB:88:70:9B:DB:8E:7E:F5:65:50:13:D6:43:6C:BE:6E', - 'extra.sha256_fingerprint' : '9A:64:73:0B:8A:FA:DE:22:D4:6D:5A:C6:C4:6F:D4:A4:2A:28:FA:41:1E:FF:81:DC:D4:D9:00:FD:78:DF:C4:DD', - 'extra.sha512_fingerprint' : '9A:B7:BD:68:7D:F3:E7:C1:B7:D3:F4:2F:01:B6:C4:77:90:A3:2B:1E:C0:89:F5:08:EC:43:87:35:60:36:D4:87:61:AA:B8:A8:B3:8A:E9:F1:04:AA:5B:67:12:FF:63:D5:14:80:77:6E:8F:7D:C3:E2:3A:F3:13:DF:08:43:6C:B0', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.source.naics' : 454110, - 'extra.source.sector' : 'Retail Trade', - 'extra.ssl_poodle' : False, - 'extra.ssl_version' : 2, - 'extra.subject_country' : 'US', - 'extra.subject_locality_name' : 'Hanover', - 'extra.subject_organization_name' : 'Ciena Corporation', - 'extra.subject_state_or_province_name' : 'Maryland', - 'extra.tag' : 'ssl,vpn', - 'extra.validation_level' : 'OV', - 'feed.name' : 'Accessible SSL', - 'protocol.application': 'https', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 14618, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'ASHBURN', - 'source.geolocation.region' : 'VIRGINIA', - 'source.ip' : '34.224.0.0', - 'source.port' : 10443, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:42+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_freak.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_freak.py deleted file mode 100644 index 42221bda2b..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_freak.py +++ /dev/null @@ -1,136 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ssl_freak.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'SSL FREAK Vulnerable Servers', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ssl_freak-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'SSL FREAK Vulnerable Servers', - 'protocol.application': 'https', - "classification.identifier": "ssl-freak", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.browser_error": "x509: unknown error", - "extra.browser_trusted": False, - "extra.cert_expiration_date": "2032-05-05 00:01:19", - "extra.cert_expired": False, - "extra.cert_issue_date": "2012-05-10 00:01:19", - "extra.cert_length": 1024, - "extra.cert_serial_number": "4FAB054F", - "extra.cert_valid": True, - "extra.cipher_suite": "TLS_RSA_WITH_RC4_128_SHA", - "extra.content_type": "text/html", - "extra.freak_cipher_suite": "TLS_RSA_EXPORT_WITH_RC4_40_MD5", - "extra.freak_vulnerable": True, - "extra.handshake": "TLSv1.0", - "extra.http_code": 200, - "extra.http_date": "2018-04-23T13:25:26+00:00", - "extra.http_reason": "OK", - "extra.http_response_type": "HTTP/1.1", - "extra.issuer_common_name": "usg50_B0B2DC2FA69D", - "extra.key_algorithm": "rsaEncryption", - "extra.md5_fingerprint": "1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE", - "extra.self_signed": True, - "extra.sha1_fingerprint": "14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2", - "extra.sha256_fingerprint": "57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1", - "extra.sha512_fingerprint": "E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87", - "extra.signature_algorithm": "sha1WithRSAEncryption", - "extra.subject_common_name": "usg50_B0B2DC2FA69D", - "extra.tag": "ssl-freak", - "extra.transfer_encoding": "chunked", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 8447, - "source.geolocation.cc": "AT", - "source.geolocation.city": "VIENNA", - "source.geolocation.region": "WIEN", - "source.ip": "198.51.100.232", - "source.port": 443, - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2018-04-23T13:25:21+00:00" - }, - {'__type': 'Event', - 'feed.name': 'SSL FREAK Vulnerable Servers', - 'protocol.application': 'https', - "classification.identifier": "ssl-freak", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.browser_error": "x509: unknown error", - "extra.browser_trusted": False, - "extra.cert_expiration_date": "2029-12-27 00:00:53", - "extra.cert_expired": False, - "extra.cert_issue_date": "2010-01-01 00:00:53", - "extra.cert_length": 1024, - "extra.cert_serial_number": "4B3D3B35", - "extra.cert_valid": True, - "extra.cipher_suite": "TLS_RSA_WITH_RC4_128_SHA", - "extra.content_type": "text/html", - "extra.freak_cipher_suite": "TLS_RSA_EXPORT_WITH_RC4_40_MD5", - "extra.freak_vulnerable": True, - "extra.handshake": "TLSv1.0", - "extra.http_code": 200, - "extra.http_date": "2018-04-23T13:25:29+00:00", - "extra.http_reason": "OK", - "extra.http_response_type": "HTTP/1.1", - "extra.issuer_common_name": "usg20w_C86C870287EC", - "extra.key_algorithm": "rsaEncryption", - "extra.md5_fingerprint": "1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE", - "extra.self_signed": True, - "extra.sha1_fingerprint": "14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2", - "extra.sha256_fingerprint": "57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1", - "extra.sha512_fingerprint": "E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87", - "extra.signature_algorithm": "sha1WithRSAEncryption", - "extra.subject_common_name": "usg20w_C86C870287EC", - "extra.tag": "ssl-freak", - "extra.transfer_encoding": "chunked", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 12577, - "source.geolocation.cc": "AT", - "source.geolocation.city": "BADEN", - "source.geolocation.region": "NIEDEROSTERREICH", - "source.ip": "198.51.100.224", - "source.port": 443, - "source.reverse_dns": "198-51-100-224.example.net", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2018-04-23T13:25:26+00:00" - }] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_poodle.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_poodle.py deleted file mode 100644 index 41535e67a4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_poodle.py +++ /dev/null @@ -1,91 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_ssl_poodle.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'SSL POODLE Vulnerable Servers', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ssl_poodle-test-geo.csv", - } -EVENTS = [{'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'ssl-poodle', - 'extra.browser_error': 'x509: unknown error', - 'extra.browser_trusted': False, - 'extra.cert_expiration_date': '2034-06-20 00:00:42', - 'extra.cert_expired': False, - 'extra.cert_issue_date': '2014-06-25 00:00:42', - 'extra.cert_length': 1024, - 'extra.cert_serial_number': '53AA112A', - 'extra.cert_valid': True, - 'extra.cipher_suite': 'TLS_RSA_WITH_RC4_128_SHA', - 'extra.content_type': 'text/html', - 'extra.handshake': 'TLSv1.0', - 'extra.http_code': 200, - 'extra.http_date': '2018-08-08T00:51:44+00:00', - 'extra.http_reason': 'OK', - 'extra.http_response_type': 'HTTP/1.1', - 'extra.issuer_common_name': 'usg20_107BEF394BA5', - 'extra.key_algorithm': 'rsaEncryption', - 'extra.md5_fingerprint': '33:E3:61:3F:5D:AA:96:99:38:A5:D6:F1:11:C7:ED:FC', - 'extra.self_signed': True, - 'extra.sha1_fingerprint': '04:FA:DE:1D:BD:4A:05:25:61:FB:F3:D6:64:74:66:44:01:22:D7:C3', - 'extra.sha256_fingerprint': '16:25:9F:C7:A1:8D:64:1F:D9:25:42:BF:87:5C:4F:F3:63:14:97:21:EC:B6:67:10:F2:CA:52:37:C9:FE:49:2E', - 'extra.sha512_fingerprint': '0B:2D:48:8C:4B:55:8B:F3:AB:F8:45:ED:E0:A0:63:F4:84:2F:4C:19:DC:A8:6F:7D:6A:AF:61:D7:98:AA:58:0F:CB:CA:87:D2:C3:0B:C5:DF:49:A7:84:7C:47:58:89:7D:92:B6:7B:98:7D:B1:64:4B:DC:DD:BE:9D:11:2A:D1:AE', - 'extra.signature_algorithm': 'sha1WithRSAEncryption', - 'extra.ssl_poodle': True, - 'extra.ssl_version': 2, - 'extra.subject_common_name': 'usg20_107BEF394BA5', - 'extra.tag': 'ssl-poodle', - 'extra.transfer_encoding': 'chunked', - 'feed.name': 'SSL POODLE Vulnerable Servers', - 'protocol.application': 'https', - 'source.asn': 65540, - 'source.geolocation.cc': 'AT', - 'source.geolocation.city': 'VIENNA', - 'source.geolocation.region': 'WIEN', - 'source.ip': '203.0.113.85', - 'source.port': 8443, - 'source.reverse_dns': 'example.com', - 'time.source': '2018-08-08T00:51:42+00:00', - "time.observation": "2015-01-01T00:00:00+00:00", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - '__type': 'Event', - }, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_stun.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_stun.py deleted file mode 100644 index 7fd5f6ec21..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_stun.py +++ /dev/null @@ -1,146 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_stun.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible-Session-Traversal-Utilities-for-NAT', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_stun-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-stun', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'extra.amplification': 5.4, - 'extra.fingerprint': '0xfaedd06e', - 'extra.magic_cookie': '2112a442', - 'extra.mapped_address': '192.168.0.1', - 'extra.mapped_family': '01', - 'extra.mapped_port': 3243, - 'extra.message_length': 88, - 'extra.message_type': '0101', - 'extra.response_size': 108, - 'extra.software': "Coturn-4.5.1.1 'dan Eider'", - 'extra.tag': 'stun', - 'extra.transaction_id': '000000000000000000000000', - 'extra.xor_mapped_address': '192.168.0.1', - 'extra.xor_mapped_family': '01', - 'extra.xor_mapped_port': 3243, - 'feed.name': 'Accessible-Session-Traversal-Utilities-for-NAT', - 'protocol.application': 'session traversal utilities for nat', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 3478, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-stun', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'extra.amplification': 5.4, - 'extra.fingerprint': '0x21128641', - 'extra.magic_cookie': '2112a442', - 'extra.mapped_address': '51.77.39.195', - 'extra.mapped_family': '01', - 'extra.mapped_port': 45877, - 'extra.message_length': 88, - 'extra.message_type': '0101', - 'extra.response_size': 108, - 'extra.software': "Coturn-4.5.1.1 'dan Eider'", - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'stun', - 'extra.transaction_id': '000000000000000000000000', - 'extra.xor_mapped_address': '192.168.0.2', - 'extra.xor_mapped_family': '01', - 'extra.xor_mapped_port': 45877, - 'feed.name': 'Accessible-Session-Traversal-Utilities-for-NAT', - 'protocol.application': 'session traversal utilities for nat', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 3478, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-stun', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'extra.amplification': 4.8, - 'extra.magic_cookie': '2112a442', - 'extra.mapped_address': '192.168.0.3', - 'extra.mapped_family': '01', - 'extra.mapped_port': 16321, - 'extra.message_length': 76, - 'extra.message_type': '0101', - 'extra.response_size': 96, - 'extra.software': "ApolloProxy-1.20.1.28 'sunflower'", - 'extra.tag': 'stun', - 'extra.transaction_id': '000000000000000000000000', - 'extra.xor_mapped_address': '188.68.240.32', - 'extra.xor_mapped_family': '01', - 'extra.xor_mapped_port': 16321, - 'feed.name': 'Accessible-Session-Traversal-Utilities-for-NAT', - 'protocol.application': 'session traversal utilities for nat', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 3478, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_synfulknock.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_synfulknock.py deleted file mode 100644 index 9b7e1fd3d9..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_synfulknock.py +++ /dev/null @@ -1,117 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_synfulknock.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'SYNful Knock', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_synfulknock-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-synfulknock', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.ack_number' : 791102, - 'extra.raw_packet' : '3cfdfec601e4700f6a9a2000080045000034c3780000f706789442099555b869f7ee0050b20800000000000c123e8012200002aa0000020405b40101040201030305', - 'extra.tag' : 'synfulknock', - 'extra.tcp_flags' : '4608', - 'extra.window_size' : 8192, - 'feed.name' : 'SYNful Knock', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 18885, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'JERSEY CITY', - 'source.geolocation.region' : 'NEW JERSEY', - 'source.ip' : '66.9.0.0', - 'source.port' : 80, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T09:18:23+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-synfulknock', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.ack_number' : 791102, - 'extra.raw_packet' : '90e2baaf0b84700f6a9a200008004500003434100000f2064382d58337d2b8698b720050916200000000000c123e8012200059d50000020405b40101040201030305', - 'extra.tag' : 'synfulknock', - 'extra.tcp_flags' : '4608', - 'extra.window_size' : 8192, - 'feed.name' : 'SYNful Knock', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 35805, - 'source.geolocation.cc' : 'GE', - 'source.geolocation.city' : 'TBILISI', - 'source.geolocation.region' : 'TBILISI', - 'source.ip' : '213.131.0.0', - 'source.port' : 80, - 'source.reverse_dns' : 'host-213-131-55-210-customer.wanex.net', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T09:19:17+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-synfulknock', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.ack_number' : 791102, - 'extra.raw_packet' : '90e2bab9cfd4700f6a9a20000800450000340f1d0000ea068bdad5b2e6914a522f360050eb5200000000000c123e801220001b4a0000020405b40101040201030305', - 'extra.tag' : 'synfulknock', - 'extra.tcp_flags' : '4608', - 'extra.window_size' : 8192, - 'feed.name' : 'SYNful Knock', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 29256, - 'source.geolocation.cc' : 'SY', - 'source.geolocation.city' : 'DAMASCUS', - 'source.geolocation.region' : 'DIMASHQ', - 'source.ip' : '213.178.0.0', - 'source.port' : 80, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T09:27:39+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_telnet.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_telnet.py deleted file mode 100644 index 66408db4c5..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_telnet.py +++ /dev/null @@ -1,87 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_telnet.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible Telnet', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_telnet-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Accessible Telnet', - "classification.identifier": "open-telnet", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.banner": "|MikroTik v6.5|Login:", - "extra.tag": "telnet-alt", - "protocol.application": "telnet", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 20255, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.145", - "source.port": 5678, - "source.reverse_dns": "example.local", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T12:27:34+00:00" - }, - {'__type': 'Event', - 'feed.name': 'Accessible Telnet', - "classification.identifier": "open-telnet", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.banner": "|MikroTik v6.45.3 (stable)|Login:", - "extra.tag": "telnet-alt", - "protocol.application": "telnet", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 20255, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.145", - "source.port": 5678, - "source.reverse_dns": "example.local", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T12:27:40+00:00" - }] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_tftp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_tftp.py deleted file mode 100644 index 3cf3688f97..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_tftp.py +++ /dev/null @@ -1,121 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_tftp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open TFTP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2019-03-25T00:00:00+00:00", - "extra.file_name": "2019-03-25-scan_tftp-test-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-tftp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 1.57, - 'extra.error': 'Not defined', - 'extra.errormessage': 'Get not supported', - 'extra.opcode': '5', - 'extra.size': 22, - 'extra.tag': 'tftp', - 'feed.name': 'Open TFTP', - 'protocol.application': 'tftp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 35067, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-tftp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 1.36, - 'extra.error': 'File not found', - 'extra.errorcode': '1', - 'extra.errormessage': 'File not found', - 'extra.opcode': '5', - 'extra.size': 19, - 'extra.tag': 'tftp', - 'feed.name': 'Open TFTP', - 'protocol.application': 'tftp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 56709, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-tftp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 1.5, - 'extra.error': 'Access violation', - 'extra.errorcode': '2', - 'extra.errormessage': 'Access violation', - 'extra.opcode': '5', - 'extra.size': 21, - 'extra.tag': 'tftp', - 'feed.name': 'Open TFTP', - 'protocol.application': 'tftp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 32785, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ubiquiti.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ubiquiti.py deleted file mode 100644 index 396bff1e33..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ubiquiti.py +++ /dev/null @@ -1,124 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ubiquiti.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open Ubiquiti', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2018-03-04T00:00:00+00:00", - "extra.file_name": "2019-03-25-scan_ubiquiti-test-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'accessible-ubiquiti-discovery-service', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 37.0, - 'extra.essid': 'Kachine-Meta-Lidia-Tereixa', - 'extra.firmwarerev': 'XS5.ar2313.v3.5.4494.091109.1459', - 'extra.mac_address': '00156db98c3a', - 'extra.model': 'NS5', - 'extra.radio_name': 'kachine.meta.lidia.tereixa', - 'extra.response_size': 148, - 'extra.tag': 'ubiquiti,iot', - 'feed.name': 'Open Ubiquiti', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 10001, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-ubiquiti-discovery-service', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 39.0, - 'extra.essid': 'Adana-Mason-Lanikai-Ozaner', - 'extra.firmwarerev': 'XM.ar7240.v5.6.3.28591.151130.1749', - 'extra.mac_address': '00156d7c9188', - 'extra.model': 'LM5', - 'extra.model_full': 'NanoStation Loco M5', - 'extra.radio_name': 'adana.mason.lanikai.ozaner', - 'extra.response_size': 156, - 'extra.tag': 'ubiquiti,iot', - 'feed.name': 'Open Ubiquiti', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 10001, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-ubiquiti-discovery-service', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 36.25, - 'extra.essid': 'Tailynn-Kadija-Noreen-Dinkar', - 'extra.firmwarerev': 'XW.ar934x.v5.6.5.29033.160515.2108', - 'extra.mac_address': '0418d6000fd5', - 'extra.model': 'P2B-400', - 'extra.model_full': 'PowerBeam M2 400', - 'extra.radio_name': 'tailynn.kadija.noreen.dinkar', - 'extra.response_size': 145, - 'extra.tag': 'ubiquiti,iot', - 'feed.name': 'Open Ubiquiti', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 10001, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_vnc.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_vnc.py deleted file mode 100644 index 457ec4425a..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_vnc.py +++ /dev/null @@ -1,86 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_vnc.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible VNC', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_vnc-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Accessible VNC', - "classification.identifier": "open-vnc", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.banner": "RFB 003.889", - "extra.product": "Apple remote desktop vnc", - "protocol.application": "vnc", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 5678, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.53", - "source.port": 5678, - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T14:51:44+00:00" - }, - {'__type': 'Event', - 'feed.name': 'Accessible VNC', - "classification.identifier": "open-vnc", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.banner": "RFB 005.000", - "extra.naics": 517311, - "extra.product": "RealVNC Enterprise v5.3 or later", - "protocol.application": "vnc", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 5678, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.112", - "source.port": 5678, - "source.reverse_dns": "localhost.localdomain", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T14:51:44+00:00"}] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ws_discovery.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ws_discovery.py deleted file mode 100644 index 41ab55e584..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ws_discovery.py +++ /dev/null @@ -1,119 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ws_discovery.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible-WS-Discovery-Service', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_ws_discovery-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-ws-discovery', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 164.83, - 'extra.error': 'Validation constraint violation: SOAP message expected', - 'extra.raw_response': 'c2FtcGxlIHJlc3BvbnNlIGRhdGEK', - 'extra.response_size': 989, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'ws-discovery', - 'feed.name': 'Accessible-WS-Discovery-Service', - 'protocol.application': 'ws-discovery', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 3702, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-ws-discovery', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 183.6, - 'extra.error': 'Validation constraint violation: missing root element', - 'extra.raw_response': 'c2FtcGxlIHJlc3BvbnNlIGRhdGEK', - 'extra.response_size': 918, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'ws-discovery', - 'feed.name': 'Accessible-WS-Discovery-Service', - 'protocol.application': 'ws-discovery', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 3702, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-ws-discovery', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 197.8, - 'extra.error': 'Validation constraint violation: SOAP message expected', - 'extra.raw_response': 'c2FtcGxlIHJlc3BvbnNlIGRhdGEK', - 'extra.response_size': 989, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'ws-discovery', - 'feed.name': 'Accessible-WS-Discovery-Service', - 'protocol.application': 'ws-discovery', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 3702, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_xdmcp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_xdmcp.py deleted file mode 100644 index d17482e715..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_xdmcp.py +++ /dev/null @@ -1,117 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Sebastian Wagner -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_xdmcp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "ShadowServer XDMCP", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_xdmcp-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-xdmcp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 6.29, - 'extra.opcode': 'Willing', - 'extra.reported_hostname': 'node01.example.com', - 'extra.size': 44, - 'extra.status': 'Linux 3.0.101-100-default', - 'extra.tag': 'xdmcp', - 'feed.name': 'ShadowServer XDMCP', - 'protocol.application': 'xdmcp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 177, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-xdmcp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 6.86, - 'extra.opcode': 'Willing', - 'extra.reported_hostname': 'node02.example.com', - 'extra.size': 48, - 'extra.status': 'Linux 2.6.9-103.ELsmp', - 'extra.tag': 'xdmcp', - 'feed.name': 'ShadowServer XDMCP', - 'protocol.application': 'xdmcp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 47074, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-xdmcp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 6.57, - 'extra.opcode': 'Willing', - 'extra.reported_hostname': 'node03.example.com', - 'extra.size': 46, - 'extra.status': '1 user, load: 6,5, 6,6, 6,6', - 'extra.tag': 'xdmcp', - 'feed.name': 'ShadowServer XDMCP', - 'protocol.application': 'xdmcp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 177, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_special.py b/intelmq/tests/bots/parsers/shadowserver/test_special.py deleted file mode 100644 index abad86cacc..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_special.py +++ /dev/null @@ -1,106 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/special.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Special', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-special-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'special', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.status' : 'likely compromised', - 'feed.name' : 'Special', - 'malware.name' : 'cyclops-blink', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'special', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.status' : 'likely compromised', - 'feed.name' : 'Special', - 'malware.name' : 'cyclops-blink', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'special', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.source.sector' : 'Professional, Scientific, and Technical Services', - 'extra.status' : 'likely compromised', - 'feed.name' : 'Special', - 'malware.name' : 'cyclops-blink', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_testdata.py b/intelmq/tests/bots/parsers/shadowserver/test_testdata.py deleted file mode 100644 index 19cbdd7d77..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_testdata.py +++ /dev/null @@ -1,81 +0,0 @@ -# SPDX-FileCopyrightText: 2017 Sebastian Wagner -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import csv -import json -import os -import os.path -import unittest -import pathlib - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot -from intelmq.bots.parsers.shadowserver.parser_json import ShadowserverJSONParserBot - -def csvtojson(csvfile): - datalist = [] - - with open(csvfile) as fop: - reader = csv.DictReader(fop, restval="") - - for row in reader: - datalist.append(row) - - return json.dumps(datalist, indent=4) - -CSVREPORTS = {} -JSONREPORTS = {} -testdata = pathlib.Path(__file__).parent / 'testdata' -for filename in testdata.glob('*.csv'): - EXAMPLE_FILE = filename.read_text() - shortname = filename.stem - CSVREPORTS[shortname] = {"raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": f"2019-01-01-{shortname}-test-test.csv", - } - JSONREPORTS[shortname] = {"raw": utils.base64_encode(csvtojson(filename)), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": f"2019-01-01-{shortname}-test-test.json", - } - - -def generate_feed_function(feedname, reports): - def test_feed(self): - """ Test if no errors happen for feed %s. """ % feedname - self.input_message = reports[feedname] - self.run_bot() - return test_feed - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - -class TestShadowserverJSONParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverJSONParserBot - -for key in CSVREPORTS: - setattr(TestShadowserverParserBot, 'test_feed_%s' % key, generate_feed_function(key, CSVREPORTS)) -for key in JSONREPORTS: - setattr(TestShadowserverJSONParserBot, 'test_feed_%s' % key, generate_feed_function(key, JSONREPORTS)) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv deleted file mode 100644 index cfadcbb2d2..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","hostname","source","reason","asn","geo","region","city","naics","sic","sector","tag" -"2019-09-04 07:00:19","198.123.245.134",host.local,"Alien Vault","Malicious Host AA",5678,"XX","LOCATION","LOCATION",517311,0,0, -"2019-09-04 07:00:19","198.123.245.171",,"Alien Vault","Malicious Host AA",5678,"XX","LOCATION","LOCATION",517311,0,, -"2019-09-04 07:00:19","198.123.245.0/24",,"Alien Vault","Malicious Host AA",5678,"XX","LOCATION","LOCATION",517311,0,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv.license deleted file mode 100644 index 476908eebe..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2020 Thomas Hungenberg -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/botnet_drone.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/botnet_drone.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/botnet_drone.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/caida_ip_spoofer.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/caida_ip_spoofer.csv.license deleted file mode 100644 index 456b03316c..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/caida_ip_spoofer.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2020 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv deleted file mode 100644 index 117dd65607..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","port","hostname","tag","application","asn","geo","region","city","url","http_host","category","system","detected_since","server","redirect_target","naics","sic","sector","cc_url","family" -"2017-01-16 00:43:48","203.0.113.1",80,"example.com","hacked-webserver-stealrat-t1","http",64496,"AT","WIEN","VIENNA","/header.php","example.com","spam","WINNT","2015-05-09 05:51:12","Microsoft-IIS/7.5",,0,0,,, -"2018-04-09 15:43:41","203.0.113.1","80","","phishing","http","64496","AT","STEIERMARK","GRAZ","/","example.com","phishing","","","","","0","0","",, -"2022-02-07 21:52:29","66.249.0.0",,"66-249-0-0.example.com","magecart",,1234,"US","CALIFORNIA","MOUNTAIN VIEW",,,"stealer",,,,,519130,,"Communications, Service Provider, and Hosting Service","https://lolfree.pw/ads.txt", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv.license deleted file mode 100644 index f8f131c2ce..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/darknet.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/darknet.csv.license deleted file mode 100644 index f8f131c2ce..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/darknet.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/ddos_amplification.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/ddos_amplification.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/ddos_amplification.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv deleted file mode 100644 index 22cfdd69e6..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","device_vendor","device_type","device_model" -"2022-01-10 00:01:42","88.84.0.0","tcp",10443,,"ssl,vpn",2116,"NO","TROMS OG FINNMARK","TROMVIK",517311,,,"Fortinet","firewall","FortiGate" -"2022-01-10 00:01:42","170.231.0.0","tcp",10443,,"ssl,vpn",27843,"PE","METROPOLITANA DE LIMA","LIMA",,,,"Fortinet","firewall","FortiGate" -"2022-01-10 00:01:42","96.60.0.0","tcp",10443,"96-60-66-218.example.com","ssl,vpn",4181,"US","WISCONSIN","MILWAUKEE",517311,,,"Fortinet","firewall","FortiGate" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/drone_brute_force.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/drone_brute_force.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/drone_brute_force.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv deleted file mode 100644 index 3114c26b15..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","domain_source","public_source","infection","family","tag","application","version","event_id","dst_network","dst_netmask","attack","duration","attack_src_ip","attack_src_port","domain","domain_transaction_id","gcip","http_method","http_path","http_postdata","http_usessl","ip_header_ack","ip_header_acknum","ip_header_dont_fragment","ip_header_fin","ip_header_identity","ip_header_psh","ip_header_rst","ip_header_seqnum","ip_header_syn","ip_header_tos","ip_header_ttl","ip_header_urg","number_of_connections","packet_length","packet_randomized","http_agent" -"2010-02-10 00:00:00",tcp,192.168.0.1,38055,64512,ZZ,Region,City,node01.example.com,0,,,,,172.16.0.1,443,65534,ZZ,Region,City,node01.example.net,0,"",,,ddos-participant,,,https,,,,,,,,,www.example.com,,,GET,/??=GovpfOoaWYlk,,,,,,,,,,,,,,,,,, -"2010-02-10 00:00:01",udp,192.168.0.2,53,64512,ZZ,Region,City,node02.example.com,0,,,,,172.16.0.2,53,65534,ZZ,Region,City,node02.example.net,0,"",,,ddos-participant,,,dns,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -"2010-02-10 00:00:02",udp,192.168.0.3,53,64512,ZZ,Region,City,node03.example.com,0,,Microsoft,email,Exchange,172.16.0.3,53,65534,ZZ,Region,City,node03.example.net,0,"",,,ddos-participant,,,dns,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv.license deleted file mode 100644 index 9f58c89ef0..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2023 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv deleted file mode 100644 index 17ff15ee6c..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv +++ /dev/null @@ -1,7 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","service","start_time","end_time","client_version","username","password","payload_url","payload_md5" -"2021-03-27 00:00:00","tcp","141.98.1.2",30123,209588,"NL","NOORD-HOLLAND","AMSTERDAM",,,,,,,"162.250.1.2",22,26832,"CA","QUEBEC","MONTREAL",,,,"CAPRICA-EU","ssh-brute-force",,,"ssh",,,,"2021-03-27T00:00:00.521730Z","2021-03-27T00:00:01.710968Z","b'SSH-2.0-Go'",,,, -"2021-03-27 00:00:00","tcp","5.188.3.4",55690,57172,"NL","NOORD-HOLLAND","AMSTERDAM",,518210,,,,,"162.250.3.4",22,26832,"CA","QUEBEC","MONTREAL",,,,"CAPRICA-EU","ssh-brute-force",,,"ssh",,,,"2021-03-27T00:00:00.520927Z","2021-03-27T00:00:01.670993Z","b'SSH-2.0-Go'",,,, -"2021-03-27 00:00:00","tcp","45.14.5.6",38636,44220,"RO","BIHOR","ORADEA",,,,,,,"82.118.5.6",23,204957,"PL","MAZOWIECKIE","WARSAW",,,,"CAPRICA-EU","telnet-brute-force",,,"telnet",,,,"2021-03-27T00:00:00.781774Z","2021-03-27T00:00:00.857244Z",,,,, -"2021-03-27 00:00:00","tcp","5.188.6.7",56385,49453,"NL","NOORD-HOLLAND","AMSTERDAM",,518210,,,,,"102.16.6.7",22,37054,"MG","ANTANANARIVO","ANTANANARIVO",,,"Communications, Service Provider, and Hosting Service","CAPRICA-EU","ssh-brute-force",,,"ssh",,,,"2021-03-27T00:00:00.163870Z","2021-03-27T00:00:02.896640Z","b'SSH-2.0-Go'",,,, -"2021-03-27 00:00:00","tcp","45.14.7.8",35802,44220,"RO","BIHOR","ORADEA",,,,,,,"82.118.7.8",23,204957,"PL","MAZOWIECKIE","WARSAW",,,,"CAPRICA-EU","telnet-brute-force",,,"telnet",,,,"2021-03-27T00:00:00.781272Z","2021-03-27T00:00:00.856606Z",,,,, -"2021-03-27 00:00:00","tcp","5.188.9.10",33289,49453,"NL","NOORD-HOLLAND","AMSTERDAM",,518210,,,,,"60.234.9.10",22,9790,"NZ","WELLINGTON","LOWER HUTT",,,"Communications, Service Provider, and Hosting Service","CAPRICA-EU","ssh-brute-force",,,"ssh",,,,"2021-03-27T00:00:00.044871Z","2021-03-27T00:00:00.077322Z","b'SSH-2.0-Go'",,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv.license deleted file mode 100644 index 8b9580cf15..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv deleted file mode 100644 index dc78c1c1aa..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv +++ /dev/null @@ -1,9 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","count" -"2021-03-07 00:00:00","tcp","61.3.1.2",4717,9829,"IN","KERALA","CHENGANNUR",,518210,,,,,,23,,,,,,,,,"mirai",,"mirai",,,, -"2021-03-07 00:00:00","tcp","211.218.3.4",4405,4766,"KR","GANGWON-DO","PYEONGCHANG-EUP",,517311,,,,,,23,,,,,,,,,"mirai",,"mirai",,,, -"2021-03-07 00:00:00","tcp","45.225.5.6",59777,266915,"BR","BAHIA","VITORIA DA CONQUISTA","static-45-225-x-x.example.net",,,,,,,23,,,,,,,,,"mirai",,"mirai",,,, -"2021-03-07 00:00:00","tcp","125.122.7.8",8460,4134,"CN","ZHEJIANG SHENG","HANGZHOU",,517311,,,,,,23,,,,,,,,,"mirai",,"mirai",,,, -"2021-03-07 00:00:00","tcp","219.77.9.10",21867,4760,"HK","HONG KONG","HONG KONG","n219077092196.example.com",517311,,,,,,5555,,,,,,,,,"mirai",,"mirai",,,, -"2021-03-07 00:00:00","tcp","24.137.11.12",4680,14638,"PR","PUERTO RICO","SAN JUAN","dynamic.libertypr.net",,,,,,,5555,,,,,,,,,"mirai",,"mirai",,,, -"2021-03-07 00:00:00","tcp","119.182.13.14",13175,4837,"CN","SHANDONG SHENG","JINING",,517311,,,,,,23,,,,,,,,,"mirai",,"mirai",,,, -"2021-03-07 00:00:00","tcp","27.198.15.16",56133,4837,"CN","SHANDONG SHENG","JINAN",,517311,,,,,,23,,,,,,,,,"mirai",,"mirai",,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv.license deleted file mode 100644 index f4e16ec677..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Birger Schacht -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv deleted file mode 100644 index f41cb508f7..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","domain_source","public_source","infection","family","tag","application","version","event_id","dst_network","dst_netmask","attack","duration","attack_src_ip","attack_src_port","domain","domain_transaction_id","gcip","http_method","http_path","http_postdata","http_usessl","ip_header_ack","ip_header_acknum","ip_header_dont_fragment","ip_header_fin","ip_header_identity","ip_header_psh","ip_header_rst","ip_header_seqnum","ip_header_syn","ip_header_tos","ip_header_ttl","ip_header_urg","number_of_connections","packet_length","packet_randomized","http_agent" -"2010-02-10 00:00:00",,192.168.0.1,61234,64512,ZZ,Region,City,node01.example.com,0,"Communications, Service Provider, and Hosting Service",,,,172.16.0.1,88,65534,ZZ,Region,City,node01.example.net,0,,,,ddos,mirai,mirai,mirai,,,121.12.110.28/32,32,atk10,30,,,,,,,,,,,,,,,,,,,,,,,1440, -"2010-02-10 00:00:01",,192.168.0.2,61234,64512,ZZ,Region,City,node02.example.com,0,"Communications, Service Provider, and Hosting Service",,,,172.16.0.2,80,65534,ZZ,Region,City,node02.example.net,0,,,,ddos,mirai,mirai,mirai,,,180.97.183.94/32,32,atk10,30,,,,,,,,,,,,,,,,,,,,,,,1440, -"2010-02-10 00:00:02",,192.168.0.3,6379,64512,ZZ,Region,City,node03.example.com,0,"Communications, Service Provider, and Hosting Service",,,,172.16.0.3,,65534,ZZ,Region,City,node03.example.net,0,"Communications, Service Provider, and Hosting Service",,,ddos,mirai,mirai,mirai,,,104.237.138.135/32,32,atk7,10,,,,,,,,,,,,,,,,,,,,,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv deleted file mode 100644 index a7d0bc4f1d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv +++ /dev/null @@ -1,6 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","request","count","bytes","end_time","duration","avg_pps","max_pps" -"2021-03-28 00:00:02",,"107.141.1.2",,7018,"US","CALIFORNIA","VISALIA","107-141-x-x.lightspeed.frsnca.sbcglobal.net",517311,"Communications, Service Provider, and Hosting Service",,,,,389,,,,,,,,"CISPA","ddos-amplification",,,,,,,,,"2021-03-28 00:20:22",,, -"2021-03-28 00:00:02",,"74.59.3.4",,5769,"CA","QUEBEC","CHICOUTIMI","modemcablex-x-59-74.mc.videotron.ca",517311,"Communications, Service Provider, and Hosting Service",,,,,389,,,,,,,,"CISPA","ddos-amplification",,,,,,,,,"2021-03-28 00:13:50",,, -"2021-03-28 00:00:02",,"65.131.5.6",,209,"US","WYOMING","CASPER","65-131-x-x.chyn.qwest.net",517311,"Communications, Service Provider, and Hosting Service",,,,,389,,,,,,,,"CISPA","ddos-amplification",,,,,,,,,,,, -"2021-03-28 00:00:02",,"104.162.7.8",,12271,"US","NEW YORK","KINGSTON","cpe-104-162-x-x.hvc.res.rr.com",517311,"Communications, Service Provider, and Hosting Service",,,,,389,,,,,,,,"CISPA","ddos-amplification",,,,,,,,,,,, -"2021-03-28 00:00:02",,"37.120.178.9.10",,197540,"DE","NIEDERSACHSEN","GIFHORN","v22020111328131649.ultrasrv.de",,,,,,,389,,,,,,,,"CISPA","ddos-amplification",,,,,,,,,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv.license deleted file mode 100644 index 8b9580cf15..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv deleted file mode 100644 index 0e5b1e5e9c..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","protocol","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","device_vendor","device_type","device_model","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","domain_source","public_source","infection","family","tag","application","version","event_id","dst_network","dst_netmask","attack","duration","attack_src_ip","attack_src_port","domain","domain_transaction_id","gcip","http_method","http_path","http_postdata","http_usessl","ip_header_ack","ip_header_acknum","ip_header_dont_fragment","ip_header_fin","ip_header_identity","ip_header_psh","ip_header_rst","ip_header_seqnum","ip_header_syn","ip_header_tos","ip_header_ttl","ip_header_urg","number_of_connections","packet_length","packet_randomized" -"2010-02-10 00:00:00",,172.16.0.1,80,65534,ZZ,Region,City,node01.example.net,0,,,,,192.168.0.1,61234,64512,ZZ,Region,City,node01.example.com,0,"Communications, Service Provider, and Hosting Service",,,ddos,mirai,mirai,mirai,,,115.238.198.85/32,32,atk0,30,,,,,,,,,,,,,,,,,,,,,,,1440, -"2010-02-10 00:00:01",,172.16.0.2,43437,65534,ZZ,Region,City,node02.example.net,0,Information,,,,192.168.0.2,61234,64512,ZZ,Region,City,node02.example.com,0,"Communications, Service Provider, and Hosting Service",,,ddos,mirai,mirai,mirai,,,52.184.50.250/32,32,atk0,30,,,,,,,,,,,,,,,,,,,,,,,1440, -"2010-02-10 00:00:02",,172.16.0.3,80,65534,ZZ,Region,City,node03.example.net,0,,,,,192.168.0.3,61234,64512,ZZ,Region,City,node03.example.com,0,"Communications, Service Provider, and Hosting Service",,,ddos,mirai,mirai,mirai,,,211.99.102.216/32,32,atk10,30,,,,,,,,,,,,,,,,,,,,,,,1440, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv deleted file mode 100644 index d9448bd83d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","pattern","http_url","http_agent","http_request_method","url_scheme","session_tags","vulnerability_enum","vulnerability_id","vulnerability_class","vulnerability_score","vulnerability_severity","vulnerability_version","threat_framework","threat_tactic_id","threat_technique_id","target_vendor","target_product","target_class","file_md5","file_sha256","request_raw","body_raw" -"2021-08-01 00:24:08","tcp","191.23.45.67",36455,1234,"EE","HARJUMAA","TALLINN","191-23-45-67-host.example.com",518210,"Communications, Service Provider, and Hosting Service",,,,"109.87.65.43",80,5678,"UK","WINDSOR AND MAIDENHEAD","MAIDENHEAD",,518210,,"CAPRICA-EU","http-scan",,,,"3.1.3-dev",,"unknown","/js/ueditor/wwwroot/way-board.cgi",,,,,,,,,,,,,,,,,,,"GET /js/ueditor/wwwroot/way-board.cgi HTTP/1.0rnAccept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8rnAccept-Encoding: gzip, deflaternAccept-Language: en-US,en;q=0.5rnConnection: closernDnt: 1rnHost: 109.87.65.43rnOrigin: http://109.87.65.43rnReferer: http://109.87.65.43/rnUser-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.104 Safari/537.36 Core/1.53.3084.400 QQBrowser/9.6.11346.400", -"2021-08-01 05:21:59","tcp","45.67.89.123",58610,12345,"EE","HARJUMAA","TALLINN",,,,,,,"82.41.20.10",8080,23456,"UA","KHARKIVS'KA OBLAST'","KHARKIV",,,,"CAPRICA-EU","http-scan",,,,,,,"/","Mozilla/5.0 (Windows NT 5.1; rv:9.0.1) Gecko/20100101 Firefox/9.0.1","GET","http",,,,,,,,,,,,,,,,"R0VUIC8gSFRUUC8xLjENCkhvc3Q6IDgyLjQxLjIwLjEwOjgwODANCkFjY2VwdDogdGV4dC9odG1sLGFwcGxpY2F0aW9uL3hodG1sK3htbCxhcHBsaWNhdGlvbi94bWw7cT0wLjksKi8qO3E9MC44DQpBY2NlcHQtRW5jb2Rpbmc6IGRlZmxhdGUsIGd6aXAsIGlkZW50aXR5DQpBY2NlcHQtTGFuZ3VhZ2U6IGVuLVVTO3E9MC42LGVuO3E9MC40DQpVc2VyLUFnZW50OiBNb3ppbGxhLzUuMCAoV2luZG93cyBOVCA1LjE7IHJ2OjkuMC4xKSBHZWNrby8yMDEwMDEwMSBGaXJlZm94LzkuMC4xDQoNCg==", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv.license deleted file mode 100644 index c1900637ff..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Mikk Margus Möll -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv deleted file mode 100644 index 174360bbdc..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv +++ /dev/null @@ -1,7 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","network","routedspoof","session","nat" -"2021-03-28 00:42:59","tcp","98.191.250.0",,22898,"US","OKLAHOMA","OKLAHOMA CITY","ip-98.191.250.0.atlinkservices.com",517311,,,,,,,,,,,,,,"caida","ip-spoofer",,,,"ipv4",,"98.191.250.0/24","received",1112907,"True" -"2021-03-28 01:36:22","tcp","191.7.16.0",,262485,"BR","RIO DE JANEIRO","NOVA IGUACU",,,,,,,,,,,,,,,,"caida","ip-spoofer",,,,"ipv4",,"191.7.16.0/24","received",1112914,"False" -"2021-03-28 02:10:58","tcp","202.53.160.0",,23923,"BD","DHAKA","DHAKA",,,,,,,,,,,,,,,,"caida","ip-spoofer",,,,"ipv4",,"202.53.160.0/24","received",1112931,"True" -"2021-03-28 03:41:51","tcp","87.121.75.0",,134697,"AU","QUEENSLAND","BRISBANE",,,,,,,,,,,,,,,,"caida","ip-spoofer",,,,"ipv4",,"87.121.75.0/24","received",1112953,"True" -"2021-03-28 06:07:17","tcp","189.201.194.0",,262944,"MX","COAHUILA","SALTILLO","ip-189-201-194-0.slw.spectro.mx",,,,,,,,,,,,,,,"caida","ip-spoofer",,,,"ipv4",,"189.201.194.0/24","received",1113015,"True" -"2021-03-28 06:59:53","tcp","197.15.48.0",,37671,"TN","TUNIS","TUNIS",,517311,,,,,,,,,,,,,,"caida","ip-spoofer",,,,"ipv4",,"197.15.48.0/24","received",1113035,"True" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv.license deleted file mode 100644 index f4e16ec677..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Birger Schacht -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv deleted file mode 100644 index eb0cbbab95..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv +++ /dev/null @@ -1,7 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id" -"2021-06-07 00:00:00","tcp","190.229.1.2",52955,7303,"AR","BUENOS AIRES","CASEROS",,517311,,,,,"168.63.134.179",16464,8075,"HK","HONG KONG","HONG KONG",,334111,"Information","MSDCU","b68-zeroaccess-2-32bit","zeroaccess","b68-zeroaccess-2-32bit",,, -"2021-06-07 00:00:00","tcp","96.20.3.4",16464,5769,"CA","QUEBEC","LAVAL",,517311,"Communications, Service Provider, and Hosting Service",,,,"52.169.3.4",16464,8075,"IE","DUBLIN","DUBLIN",,334111,"Information","MSDCU","b68-zeroaccess-2-32bit","zeroaccess","b68-zeroaccess-2-32bit",,, -"2021-06-07 00:00:00","tcp","187.222.5.6",55049,8151,"MX","CIUDAD DE MEXICO","MEXICO CITY",,517311,,,,,"168.63.134.179",16464,8075,"HK","HONG KONG","HONG KONG",,334111,"Information","MSDCU","b68-zeroaccess-2-32bit","zeroaccess","b68-zeroaccess-2-32bit",,, -"2021-06-07 00:00:00","tcp","75.84.7.8",64190,20001,"US","CALIFORNIA","NORTH HOLLYWOOD",,517311,"Communications, Service Provider, and Hosting Service",,,,"52.169.7.8",16464,8075,"IE","DUBLIN","DUBLIN",,334111,"Information","MSDCU","b68-zeroaccess-2-32bit","zeroaccess","b68-zeroaccess-2-32bit",,, -"2021-06-07 00:00:00","tcp","24.15.9.10",60373,7922,"US","ILLINOIS","HOMER GLEN",,517311,"Communications, Service Provider, and Hosting Service",,,,"104.40.6.5",16464,8075,"US","CALIFORNIA","SAN FRANCISCO",,334111,"Information","MSDCU","b68-zeroaccess-2-32bit","zeroaccess","b68-zeroaccess-2-32bit",,, -"2021-06-07 00:00:00","tcp","124.101.11.12",50386,4713,"JP","FUKUOKA","FUKUOKA",,517311,"Communications, Service Provider, and Hosting Service",,,,"23.99.101.165",16465,8075,"HK","HONG KONG","HONG KONG",,334111,"Information","MSDCU","b68-zeroaccess-2-64bit","zeroaccess","b68-zeroaccess-2-64bit",, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv.license deleted file mode 100644 index f4e16ec677..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Birger Schacht -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv deleted file mode 100644 index c56d1f218b..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv +++ /dev/null @@ -1,6 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","http_url","http_host","http_agent","forwarded_by","ssl_cipher","http_referer" -"2021-06-07 00:00:00","tcp","31.206.1.2",49245,8386,"TR","ANTALYA","KEPEZ",,,,,,,"40.121.206.97",80,8075,"US","VIRGINIA","ASHBURN",,334111,"Information","MSDCU","necurs","necurs","necurs",,,,"/locator.php","40.121.206.97",,,, -"2021-06-07 00:00:00","tcp","177.140.3.4",35919,28573,"BR","SAO PAULO","SAO PAULO",,517312,,,,,"204.95.99.204",443,8075,"US","WASHINGTON","REDMOND",,334111,"Information","MSDCU","caphaw","caphaw","caphaw",,,,"/index.php","3fo8jrthz3y.rgk.cc","Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.0.6103)",,,"null" -"2021-06-07 00:00:01","tcp","180.190.5.6",49264,132199,"PH","CEBU","MANDAUE",,517311,,,,,"40.121.206.97",80,8075,"US","VIRGINIA","ASHBURN",,334111,"Information","MSDCU","necurs","necurs","necurs",,,,"/locator.php","40.121.206.97",,,, -"2021-06-07 00:00:01","tcp","197.157.7.8",55307,37129,"KE","NAIROBI CITY","NAIROBI",,,,,,,"40.121.206.97",80,8075,"US","VIRGINIA","ASHBURN",,334111,"Information","MSDCU","necurs","necurs","necurs",,,,"/news/stream.php","40.121.206.97",,,, -"2021-06-07 00:00:01","tcp","174.114.9.10",59000,812,"CA","ONTARIO","OTTAWA",,517311,"Communications, Service Provider, and Hosting Service",,,,"40.121.206.97",80,8075,"US","VIRGINIA","ASHBURN",,334111,"Information","MSDCU","necurs","necurs","necurs",,,,"/locator.php","40.121.206.97",,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv.license deleted file mode 100644 index f4e16ec677..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Birger Schacht -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv deleted file mode 100644 index c5126c843a..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id" -"2021-03-04 00:00:00","tcp","190.113.1.2",17409,12252,"PE","METROPOLITANA DE LIMA","LIMA",,,,,,,"178.162.1.2",4455,28753,"DE","HESSEN","FRANKFURT AM MAIN",,518210,"Communications, Service Provider, and Hosting Service","eset","victorygate.b","victorygate.b",,,, -"2021-03-04 00:00:00","tcp","35.205.9.10",44696,15169,"BE","BRUXELLES-CAPITALE","BRUSSELS","x.x.205.35.bc.googleusercontent.com",519130,"Communications, Service Provider, and Hosting Service",,,,"148.81.9.10",80,1887,"PL","MAZOWIECKIE","WARSAW",,,,,"virut","virut",,,, -"2021-03-04 00:00:00","tcp","35.197.11.12",36968,15169,"US","OREGON","THE DALLES","x.x.197.35.bc.googleusercontent.com",519130,"Communications, Service Provider, and Hosting Service",,,,"148.81.111.11.12",80,1887,"PL","MAZOWIECKIE","WARSAW",,,,,"virut","virut",,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv.license deleted file mode 100644 index f4e16ec677..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Birger Schacht -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv deleted file mode 100644 index 3e85690d85..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","infection","family","tag","query_type","query","count" -"2022-01-06 00:00:02","udp","217.110.0.0",29614,8220,"DE","HESSEN","FRANKFURT AM MAIN",,518210,"Communications, Service Provider, and Hosting Service",,,,"calypso","calypso","msexchange","A","YolkIsh.COM",1 -"2022-01-06 00:00:02","udp","209.66.0.0",46189,40934,"US","VIRGINIA","ASHBURN",,518210,,,,,"orcus","orcus","rat","A","verble.rocks",1 -"2022-01-06 00:00:02","udp","217.110.0.0",3590,8220,"DE","HESSEN","FRANKFURT AM MAIN",,518210,"Communications, Service Provider, and Hosting Service",,,,"calypso","calypso","msexchange","A","RAwFuNS.COM",1 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv.license deleted file mode 100644 index 662bb20b71..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv deleted file mode 100644 index 4514f248ed..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv +++ /dev/null @@ -1,6 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","http_url","http_host","http_agent","forwarded_by","ssl_cipher","http_referer" -"2021-03-04 00:00:00","tcp","103.196.1.2",60902,134707,"PH","NUEVA ECIJA","DEL PILAR",,,,,,,"184.105.1.2",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,,"andromeda","avalanche-andromeda",,,,,"differentia.ru",,,, -"2021-03-04 00:00:00","tcp","5.14.3.4",55002,8708,"RO","CONSTANTA","CONSTANTA",,517311,"Communications, Service Provider, and Hosting Service",,,,"184.105.3.4",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,,"andromeda","avalanche-andromeda",,,,,"differentia.ru",,,, -"2021-03-04 00:00:00","tcp","49.145.5.6",31350,9299,"PH","CEBU","CEBU",,517311,,,,,"184.105.5.6",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,,"andromeda","avalanche-andromeda",,,,,"disorderstatus.ru",,,, -"2021-03-04 00:00:00","tcp","200.44.7.8",28063,8048,"VE","CARABOBO","VALENCIA",,517311,,,,,"184.105.7.8",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,,"andromeda","avalanche-andromeda",,,,,"differentia.ru",,,, -"2021-03-04 00:00:00","tcp","187.189.9.10",45335,17072,"MX","CHIHUAHUA","JUAREZ",,,,,,,"184.105.9.10",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,,"andromeda","avalanche-andromeda",,,,,"differentia.ru",,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv.license deleted file mode 100644 index f4e16ec677..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Birger Schacht -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv deleted file mode 100644 index 23a3cb2b68..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv +++ /dev/null @@ -1,6 +0,0 @@ -"timestamp","protocol","http_referer_ip","http_referer_port","http_referer_asn","http_referer_geo","http_referer_region","http_referer_city","http_referer_hostname","http_referer_naics","http_referer_sector","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","http_url","http_host","http_referer" -"2021-03-04 00:00:02","tcp","178.162.203.211",80,28753,"DE","HESSEN","FRANKFURT AM MAIN","12106.mobapptrack.com",518210,"Communications, Service Provider, and Hosting Service","85.17.31.82",80,60781,"NL","NOORD-HOLLAND","AMSTERDAM",,518210,"Communications, Service Provider, and Hosting Service",,,"kovter","kovter",,,1614816002,"GET /favicon.ico HTTP/1.1","12106.mobapptrack.com","http://12106.mobapptrack.com/click/redirect?feed_id=12106&sub_id=7&q=8A5491983C8FBE7743E2D2C36E45EBC4-18307118D2626C9BD756B3F09D14BB910E381EE4" -"2021-03-04 00:00:11","tcp","59.106.1.2",80,9370,"JP","OSAKA","OSAKA","x.noizm.com",518210,"Communications, Service Provider, and Hosting Service","178.162.1.2",80,28753,"DE","HESSEN","FRANKFURT AM MAIN",,518210,"Communications, Service Provider, and Hosting Service",,,"sunburst","sunburst",,,1614816011,"GET /animalally.com HTTP/1.1","freescanonline.com","http://x.noizm.com/jump.php?u=http://freescanonline.com/animalally.com" -"2021-03-04 00:00:12","tcp","142.250.3.4",80,15169,"US","CALIFORNIA","MOUNTAIN VIEW","x.blogspot.com",519130,"Communications, Service Provider, and Hosting Service","178.162.1.2",80,28753,"DE","HESSEN","FRANKFURT AM MAIN",,518210,"Communications, Service Provider, and Hosting Service",,,"kovter","kovter",,,1614816012,"GET /getjs?r=0.6393021999392658 HTTP/1.1","rxrtb.bid","http://x.blogspot.com/" -"2021-03-04 00:00:13","tcp","34.232.5.6",80,14618,"US","VIRGINIA","ASHBURN","www.example.com",454110,"Retail Trade","5.79.71.225",80,60781,"NL","NOORD-HOLLAND","AMSTERDAM",,518210,"Communications, Service Provider, and Hosting Service",,,"sunburst","sunburst",,,1614816013,"GET /personalationmall.com HTTP/1.1","freescanonline.com","http://www.example.com/teams/default.asp?u=EKL&t=c&s=lacrosse&p=remote&url=http://freescanonline.com/personalationmall.com" -"2021-03-04 00:01:26","tcp","210.172.7.8",80,2516,"JP","HOKKAIDO","SAPPORO","x.communes.jp",517312,"Communications, Service Provider, and Hosting Service","5.79.1.2",80,60781,"NL","NOORD-HOLLAND","AMSTERDAM",,518210,"Communications, Service Provider, and Hosting Service",,,"sunburst","sunburst",,,1614816086,"GET /raftcomply.com HTTP/1.1","freescanonline.com","http://x.communes.jp/?url=http://freescanonline.com/raftcomply.com" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv.license deleted file mode 100644 index c1900637ff..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Mikk Margus Möll -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv deleted file mode 100644 index 016d2f912b..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","http_url","http_host","http_agent","forwarded_by","ssl_cipher","http_referer" -"2022-03-02 09:14:19","tcp","2001:448a:1082:4d9b:7491:bf9e:3d5f:a634",49431,7713,"ID","JAKARTA RAYA","JAKARTA",,,,,,,"2001:470:1:332::fe",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,"boaxxe","boaxxe","3ve",,,,"GET /QKMSvF2hl11j%2fbMkyPbF5EpHYhd6VWTG4u19K3Rt7JGU3lMYRqpq8wPYEuOGKKeidKW3pefVfKSjBnL0cXizZbmuWWu8AQNRqw5g9Ny5vZtiv638XKoWwCLuUOTISTV%2fLcpcS1%2f22NjWqgXkHGISAuyVtafqyCC%2f5cA0eYg9Me8VzAIFDdTArogQOdYhElf2xluhEFPsstGQ%2bwrM4VmKHJpzyjD7Y%2fN%2bQV3wnZNdVkEVk1k2iKBJkotYv3ajgYWr56xxCbY5vE1IpZBRNhhaUDNZo0kJgi%2b6knXZ4m7JHt%2fGtJeP%2bNTxHSUL2ELlTIiT3ENlPYD6FdH6ZBxT1OneW%2f0ih%2fcN7vctb5B5Qwa1ez7ZjN2QxgBYkFDDHHTs42ej5eF2BysWAQDSUr%2fcySyGxcfPveIpfQEdrynGKR6z3OYqkFnP%2bYRDQp2rt1qt0FwCB4L9cg05TQlSSTJVGfPDrtcqjvKY4c9hWwSHtE8jMRpeCYO4Es%2bWgwr5DjzMicmuZo%2f4Ycr16jpN7xlDJdJ8iCFZxbSGgVC7ksVlGE8wlfWPI4KTuX5U5s61eNWPTlAC%2fOGb8grtw%2ffzizoIX9D6ZUMvslGLQIp%2fvNmNQkZy8HhNoV6Lns%2figITP%2fpN0H8h9HjUTl9qn65xFOEVpc0motSy8alcTPtTRKq5Jvc4Ao0x3N%2fvCB1v4Epx7XC0UpFbw8TrYEvAczEfGsGM HTTP/1.1","devps.net","Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET4.0C; .NET4.0E)",,, -"2022-03-02 09:15:10","tcp","2001:448a:1082:4d9b:7491:bf9e:3d5f:a634",49460,7713,"ID","JAKARTA RAYA","JAKARTA",,,,,,,"2001:470:1:332::ef",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,"boaxxe","boaxxe","3ve",,,,"GET /QKMSvF2hl11j%2fbMkyPbF5EpHYhd6VWTG4u19K3Rt7JGU3lMYRqpq8wPYEuOGKKeidKW3pefVfKSjBnL0cXizZbmuWWu8AQNRqw5g9Ny5vZtiv638XKoWwCLuUOTISTV%2fLcpcS1%2f22NjWqgXkHGISAuyVtafqyCC%2f5cA0eYg9Me8VzAIFDdTArogQOdYhElf2xluhEFPsstGQ%2bwrM4VmKHJpzyjD7Y%2fN%2bQV3wnZNdVkEVk1k2iKBJkotYv3ajgYWr56xxCbY5vE1IpZBRNhhaUDNZo0kJgi%2b6knXZ4m7JHt%2fGtJeP%2bNTxHSUL2ELlTIiT3ENlPYD6FdH6ZBxT1OneW%2f0ih%2fcN7vctb5B5Qwa1ez7ZjN2QxgBYkFDDHHTs42ej5eF2BysWAQDSUr%2fcySyGxcfPveIpfQEdrynGKR6z3OYqkFnP%2bYRDQp2rt1qt0FwCB4L9cg05TQlSSTJVGfPDrtcqjvKY4c9hWwSHtE8jMRpeCYO4Es%2bWgwr5DjzMicmuZo%2f4Ycr16jpN7xlDJdJ8iCFZxbSGgVC7ksVlGE8wlfWPI4KTuX5U5s61eNWPTlAC%2fOGb8grtw%2ffzizoIX9D6ZUMvslGLQIp%2fvNmNQkZy8HhNoV6Lns%2figITP%2fpN0H8h9HjUTl9qn65xFOEVpc0motSy8alcTPtTRKq5Jvc4Ao0x3N%2fvCB1v4Epx7XC0UpFbw8TrYEvAczEfGsGM HTTP/1.1","devps.net","Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET4.0C; .NET4.0E)",,, -"2022-03-02 14:15:10","tcp","2603:8080:b20a:dc00:f06e:8304:71f6:27e2",62932,11427,"US","TEXAS","GARLAND",,517311,"Communications, Service Provider, and Hosting Service",,,,"2001:470:1:332::fe",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,"boaxxe","boaxxe","3ve",,,,"GET /WMoUNCvuKGzdqSCeQcadP1%2f0B%2f3bzpOmyKBU85Z25HVOhvDQUPFl%2fk8uOcLewS%2b1BsuHXalRAOIgGOYYs2igj6UX8FkdCAmDewWPvfDhPD45nwd2tx1lLf2IoIfuOtIpGR6bN5Q6hGpSBgfERqCa0ImHcwfcZ2EdO%2fWvg7R8H6SLcTiuUC0I4pzvlWt1CRLgLdIEU1hZ0nnFHIHhchb6D7ITEgBQ2chQDxy5TJMrGjm4Dac6dKl%2ft5uYhRhSjAHkLLtgrJjsqVtVbelTAkt5kdcqLlO09m1SH%2fvtAb%2fOvR2DbhBss7%2f64DG7g6cAnghNA6JrFn1uW7sw%2bnKH8koKQwzUjdSsbrQAvmg4r0KDDW8Diq64gfDzxFWkzCLOYifc%2fwlinXPCl7aJiNCoieDC1U98RNQg%2f5td4SZmJnDQ2%2f96CPbFeSpCez5WD1rCjrxLj1h2cqzIgkydEWACceWP9ztxc4QaObzEcgOGxbRckWC7H2aaLeT8jaYEYdKi1pwEKChSL3YdEt4ZIb2IFrWwzNaXEpQzFXf07f902OEdI9vVA1ZdEOBPG6rAIkzMdebfprfVyhKEWtrCd3Skg3COUFtRQks5jzG1nv4sVGijTfSgyn6xE9Taka668Nycik6nmHy8Huj3oC01j3tee%2f1Z3eI6tV7lgM5d3uFJ84slRGHUCwMfVozOGmZRwNo%2fz%2bA HTTP/1.1","devps.net","Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko",,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv.license deleted file mode 100644 index 662bb20b71..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/hp_http_scan.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/hp_http_scan.csv.license deleted file mode 100644 index f8f131c2ce..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/hp_http_scan.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/hp_ics_scan.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/hp_ics_scan.csv.license deleted file mode 100644 index f8f131c2ce..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/hp_ics_scan.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv deleted file mode 100644 index ccafbab3f1..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","url","host","ip","asn","geo","region","city","naics","sector","tag","source","sha256","application" -"2022-01-07 00:02:07","http://41.86.0.0:50008/Mozi.m","41.86.0.0","41.86.0.0",37203,"LR","MONTSERRADO","MONROVIA",,,"CVE-2016-10372",,"12013662c71da69de977c04cd7021f13a70cf7bed4ca6c82acbc100464d4b0ef","http" -"2022-01-07 00:03:14","http://42.225.0.0:38173/Mozi.m","42.225.0.0","42.225.0.0",4837,"CN","HENAN SHENG","ZHUMADIAN",517311,,"CVE-2018-10562",,,"http" -"2022-01-07 00:10:26","http://211.52.0.0:53029/Mozi.m","211.52.0.0","211.52.0.0",4766,"KR","CHUNGCHEONGNAM-DO","SAGOK-MYEON",517311,,"CVE-2018-10562",,,"http" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/outdated_dnssec_key.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/outdated_dnssec_key.csv.license deleted file mode 100644 index f8f131c2ce..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/outdated_dnssec_key.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv deleted file mode 100644 index 965d763a3c..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","url","host","ip","asn","geo","region","city","naics","sector","source" -"2022-02-01 08:00:07","https://priceless-pare.example.net/Postal-/acec6/","priceless-pare.example.net","172.245.0.0",64512,"US","NEW YORK","BUFFALO",518210,"Communications, Service Provider, and Hosting Service","openphish.com" -"2022-02-01 08:00:07","https://mailyahooattt.example.net/","mailyahooattt.example.net","199.34.0.0",64512,"US","CALIFORNIA","SAN FRANCISCO",,"Professional, Scientific, and Technical Services","openphish.com" -"2022-02-01 08:00:07","https://www.example.net/viewer/vbid-730ec2b1-omsttuer","www.example.net","216.58.0.0",64512,"US","UTAH","DRAPER",519130,"Communications, Service Provider, and Hosting Service","openphish.com" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv deleted file mode 100644 index d5baa730fe..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","http","http_code","http_reason","content_type","connection","proxy_authenticate","via","server","content_length","transfer_encoding","http_date" -"2010-02-10 00:00:00",192.168.0.1,tcp,3128,node01.example.com,http-connect-proxy-closed,64512,ZZ,Region,City,0,0,HTTP/1.1,407,"Proxy Authentication Required",text/html;charset=utf-8,keep-alive,"Basic realm=\"\"Squid proxy-caching web server\"\"",,squid/4.10,3741,,"Wed, 10 Feb 2010 00:00:00 GMT" -"2010-02-10 00:00:01",192.168.0.2,tcp,3128,node02.example.com,http-connect-proxy-closed,64512,ZZ,Region,City,0,0,HTTP/1.1,407,"Proxy Authentication Required",text/html;charset=utf-8,keep-alive,"Basic realm=\"\"00:23:24:43:1c:34\"\"",,,3833,,"Wed, 10 Feb 2010 00:00:01 GMT" -"2010-02-10 00:00:02",192.168.0.3,tcp,3128,node03.example.com,http-connect-proxy-closed,64512,ZZ,Region,City,0,0,HTTP/1.1,407,"Proxy Authentication Required",text/html;charset=utf-8,keep-alive,"Basic realm=\"\"Proxy\"\"",,,179,,"Wed, 10 Feb 2010 00:00:02 GMT" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv deleted file mode 100644 index 4710af9742..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","asn","geo","md5","protocol","port","host","bytes_in","bytes_out" -"2022-01-10 00:00:03","40.119.6.228",8075,"US","b575ce6dcce6502a8431db5610135c25","udp",123,"time.windows.com",0,0 -"2022-01-10 00:00:03","8.252.70.126",3356,"US","c0d947f9a8685b0d9f3efdba966389c2","tcp",80,,0,0 -"2022-01-10 00:00:03","52.109.8.22",8075,"US","c0d947f9a8685b0d9f3efdba966389c2","tcp",443,,0,0 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv deleted file mode 100644 index 697cb6209a..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","md5hash","request","type","response","family","tag","source" -"2022-01-10 00:00:02","b575ce6dcce6502a8431db5610135c25","time.windows.com","A","40.119.6.228",,, -"2022-01-10 00:00:08","807679198a39c80d3ca07e60fd51b581","time.windows.com","A","40.119.6.228",,, -"2022-01-10 00:00:20","d97e973b9bf073bd3a217425259cea26","client-office365-tas.msedge.net","A","13.107.5.88",,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv deleted file mode 100644 index bbfe596a24..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","asn","geo","md5","url","user_agent","host","method" -"2022-01-10 00:01:13","23.196.47.89",20940,"US","37514b54e679a5313334e830ad780ec7","http://www.msftncsi.com/ncsi.txt","Microsoft NCSI","www.msftncsi.com","GET" -"2022-01-10 00:01:28","72.21.81.240",15133,"US","37514b54e679a5313334e830ad780ec7","http://www.download.windowsupdate.com/msdownload/update/v3/static/trustedr/en/authrootstl.cab","Microsoft-CryptoAPI/6.1","www.download.windowsupdate.com","GET" -"2022-01-10 00:08:24","23.56.4.57",20940,"US","e97ea2820c0d79f3f3ca241d4dcd1060","http://crl.microsoft.com/pki/crl/products/MicTimStaPCA_2010-07-01.crl","Microsoft-CryptoAPI/6.1","crl.microsoft.com","GET" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv deleted file mode 100644 index c0ff0bdf1e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","name","model","device","features","device_vendor","device_type","device_model","device_version","device_sector" -"2018-07-26 02:07:16","36.239.124.210","tcp",5555,"36-239-124-210.dynamic-ip.hinet.net","adb",3462,"TW","TAOYUAN COUNTY","TAOYUAN CITY",518210,737415,"hlteuc","SAMSUNG-SM-N900A","hlteatt",,,,,, -"2018-07-26 02:07:16","36.236.108.107","tcp",5555,"36-236-108-107.dynamic-ip.hinet.net","adb",3462,"TW","TAIPEI CITY","TAIPEI",518210,737415,"marlin","Pixel XL","marlin","cmd,shell_v2",,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv deleted file mode 100644 index c5494d4582..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","machine_type","afp_versions","uams","flags","server_name","signature","directory_service","utf8_servername","network_address" -"2019-09-04 05:05:53","198.13.34.22","tcp",548,"host.local","afp",6057,"AA","LOCATION","LOCATION",517311,0,"TimeCapsule8,119","AFP3.3,AFP3.2,AFP3.1","DHCAST128,DHX2,SRP,Recon1","SupportsCopyFile,SupportsChgPwd,SupportsServerMessages,SupportsServerSignature,SupportsTCP/IP,SupportsSrvrNotifications,SupportsReconnect,SupportsOpenDirectory,SupportsUTF8Servername,SupportsUUIDs,SupportsSuperClient","airport-time-capsule-de-jack","4338364e37364442463948350069672d",,"AirPort Time Capsule de jack","198.33.24.165:548,10.0.1.1:548,fe80:0008:0000:0000:6e70:9fff:fed4::548,fe80:0009:0000:0000:6e70:9fff:fed4::548,179.24.24.165 (DNS address)," -"2019-09-04 05:05:56","198.40.27.212","tcp",548,"host.local","afp",6057,"AA","LOCATION","LOCATION",517311,0,"TimeCapsule8,119","AFP3.3,AFP3.2,AFP3.1","DHCAST128,DHX2,SRP,Recon1","SupportsCopyFile,SupportsChgPwd,SupportsServerMessages,SupportsServerSignature,SupportsTCP/IP,SupportsSrvrNotifications,SupportsReconnect,SupportsOpenDirectory,SupportsUTF8Servername,SupportsUUIDs,SupportsSuperClient","time-capsule-del-jack","433836544b303147463948360069672d",,"Time Capsule del Jack","0.0.0.1:548,10.0.1.1:548,198.33.42.1:548,fe80:000b:0000:0000:dea4:caff:feba::548,fe80:000c:0000:0000:dea4:caff:feba::548,fe80:000d:0000:0000:4c7d:ffff:fec7::548,0.0.0.1 (DNS address)," diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv deleted file mode 100644 index 92f078af7b..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","channel","message_length","class","method","version_major","version_minor","capabilities","cluster_name","platform","product","product_version","mechanisms","locales","sector" -"2022-01-10 04:32:13","47.103.0.0","tcp",5672,,"amqp",37963,"CN","SHANGHAI SHI","SHANGHAI",518210,,0,509,10,10,0,9,"publisher_confirms,exchange_exchange_bindings,basic.nack,consumer_cancel_notify,connection.blocked,consumer_priorities,authentication_failure_close,per_consumer_qos","rabbit@iZuf63m0nnq9bwf7lhjxrkZ","Erlang/OTP","RabbitMQ","3.3.5","PLAIN AMQPLAIN","en_US", -"2022-01-10 04:32:13","141.95.0.0","tcp",5672,,"amqp",16276,"DE","SAARLAND","SAARBRUCKEN",518210,,0,509,10,10,0,9,"publisher_confirms,exchange_exchange_bindings,basic.nack,consumer_cancel_notify,connection.blocked,consumer_priorities,authentication_failure_close,per_consumer_qos,direct_reply_to","rabbit@mtk-breizh","Erlang/OTP 24.0.3","RabbitMQ","3.8.19","AMQPLAIN PLAIN","en_US", -"2022-01-10 04:32:13","54.234.0.0","tcp",5672,"ec2-54.234.0.0.compute-1.amazonaws.com","amqp",14618,"US","VIRGINIA","ASHBURN",454110,,0,509,10,10,0,9,"publisher_confirms,exchange_exchange_bindings,basic.nack,consumer_cancel_notify,connection.blocked,consumer_priorities,authentication_failure_close,per_consumer_qos,direct_reply_to","rabbit@1397a0e9629b","Erlang/OTP 24.2","RabbitMQ","3.9.11","PLAIN AMQPLAIN","en_US", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv deleted file mode 100644 index 9c43f8598b..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","machine_name","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,3283,node01.example.com,ard,64512,ZZ,Region,City,0,0,"Macmini (radio)",1006,201.20 -"2010-02-10 00:00:01",192.168.0.2,udp,3283,node02.example.com,ard,64512,ZZ,Region,City,0,0,biuro-rip-org-pl,1006,201.20 -"2010-02-10 00:00:02",192.168.0.3,udp,3283,node03.example.com,ard,64512,ZZ,Region,City,0,0,127.0.0.1,1006,201.20 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv deleted file mode 100644 index 7bd2b20e03..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","size","asn","geo","region","city","naics","sic","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,19,node01.example.com,chargen,,64512,ZZ,Region,City,0,0,,74,74.00 -"2010-02-10 00:00:01",192.168.0.2,udp,19,node02.example.com,chargen,,64512,ZZ,Region,City,0,0,,74,74.00 -"2010-02-10 00:00:02",192.168.0.3,udp,19,node03.example.com,chargen,,64512,ZZ,Region,City,0,0,Government,74,74.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv deleted file mode 100644 index 5182817c11..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic" -"2017-11-18 08:42:45","198.51.100.103","tcp",4786,"198-51-100-103.example.net","cisco-smart-install",8559,"AT","WIEN","VIENNA",0,0 -"2017-11-18 08:47:54","198.51.100.218","tcp",4786,,"cisco-smart-install",35609,"AT","WIEN","VIENNA",0,0 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv deleted file mode 100644 index 6d72dac539..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","response","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,5683,node01.example.com,coap,2,64512,ZZ,Region,City,0,0,",,",43,2.05 -"2010-02-10 00:00:01",192.168.0.2,udp,5683,node02.example.com,coap,2,64512,ZZ,Region,City,0,0,",,,,,,,,,",113,5.38 -"2010-02-10 00:00:02",192.168.0.3,udp,5683,node03.example.com,coap,1,64512,ZZ,Region,City,0,0,"`EsjAy************************************************************|CoAP RFC 7252 |************************************************************|This server is using the Eclipse Californium (Cf) CoAP framework|published under EPL+EDL: http://www.eclipse.org/californium/||(c) 2014, 2015, 2016 Institute for Pervasive Computing, ETH Zurich and others|************************************************************",454,113.50 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv deleted file mode 100644 index f4074f3ed9..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","server_version","couchdb_message","couchdb_version","git_sha","features","vendor","visible_databases","error","error_reason" -"2010-02-10 00:00:00",192.168.0.1,tcp,5984,node01.example.com,couchdb,64512,ZZ,Region,City,0,0,,"CouchDB/1.6.1 (Erlang OTP/18)",Welcome,1.6.1,,,"Ubuntu 16.04",_replicator;_users;test;shops;god,, -"2010-02-10 00:00:01",192.168.0.2,tcp,5984,node02.example.com,couchdb,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service","CouchDB/3.2.1 (Erlang OTP/23)",Welcome,3.2.1,244d428af,"access-ready,partitioned,pluggable-storage-engines,reshard,scheduler","The Apache Software Foundation",,, -"2010-02-10 00:00:02",192.168.0.3,tcp,5984,node03.example.com,couchdb,64512,ZZ,Region,City,0,0,"Retail Trade","CouchDB/3.2.1 (Erlang OTP/20)",Welcome,3.2.1,244d428af,"access-ready,partitioned,pluggable-storage-engines,reshard,scheduler","The Apache Software Foundation",,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv deleted file mode 100644 index 5aebed0500..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","http","http_code","http_reason","content_type","connection","www_authenticate","set_cookie","server","content_length","transfer_encoding","date","sector" -"2019-09-04 10:44:55","198.123.245.142","tcp",30005,,"cwmp",5678,"AA","LOCATION","LOCATION",517311,0,"HTTP/1.1",200,"OK","text/html","keep-alive",,,"DNVRS-Webs",5678,,"Wed, 04 Sep 2019 07:42:37 GMT", -"2019-09-04 11:06:50","198.123.245.162","tcp",5678,"localhost.localdomain","cwmp",5678,"AA","LOCATION","LOCATION",517311,0,"HTTP/1.1",404,"Not Found","text/html",,,,"RomPager/4.07 UPnP/1.0",,"chunked",, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv deleted file mode 100644 index c4bb32e573..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","db2_hostname","servername","size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,523,node01.example.com,db2,64512,ZZ,Region,City,0,0,NOWAK_SERWER,node01.example.com,298,14.90 -"2010-02-10 00:00:01",192.168.0.2,udp,523,node02.example.com,db2,64512,ZZ,Region,City,0,0,SPZOZ-DZIEWIN,node02.example.com,298,14.90 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv deleted file mode 100644 index 25e6f11d0e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","source_port","bytes","amplification","method" -"2010-02-10 00:00:00",192.168.0.1,tcp,80,node01.example.com,ddos-middlebox,64512,ZZ,Region,City,0,0,,49002,99,2,SYN+ACK:PSH -"2010-02-10 00:00:01",192.168.0.2,tcp,80,node02.example.com,ddos-middlebox,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",41200,99,2,SYN+ACK:PSH -"2010-02-10 00:00:02",192.168.0.3,tcp,80,node03.example.com,ddos-middlebox,64512,ZZ,Region,City,0,0,,47492,99,2,SYN+ACK:PSH diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv deleted file mode 100644 index 05b8078835..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv +++ /dev/null @@ -1,101 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","dns_version","asn","geo","region","city","min_amplification","p0f_genre","p0f_detail","naics","sic","sector" -"2018-04-14 00:14:34","198.51.100.179","udp",53,"198-51-100-189.example.net","openresolver","dnsmasq-2.66",25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:14:36","198.51.100.8","udp",53,"198-51-100-111.example.net","openresolver","dnsmasq-2.51",25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:14:34","198.51.100.179","udp",53,"198-51-100-189.example.net","openresolver","dnsmasq-2.66",25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:14:36","198.51.100.8","udp",53,"198-51-100-111.example.net","openresolver","dnsmasq-2.51",25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:14:36","198.51.100.158","udp",53,,"openresolver",,8447,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:14:37","198.51.100.7","udp",53,"198-51-100-7.example.net","openresolver","9.9.4-rpz2.13269.14-P2",13292,"AT","STEIERMARK","EISENERZ","4.6190",,,0,0, -"2018-04-14 00:14:38","198.51.100.167","udp",53,"198-51-100-167.example.net","openresolver","ZyWALL DNS",6830,"AT","KARNTEN","VILLACH","4.6667",,,0,0, -"2018-04-14 00:14:40","198.51.100.10","udp",53,"198-51-100-10.example.net","openresolver",,6830,"AT","WIEN","VIENNA","1.3810",,,518210,737415, -"2018-04-14 00:14:41","198.51.100.191","udp",53,"198-51-100-63.example.net","openresolver",,25255,"AT","TIROL","LIENZ","4.6190",,,0,0, -"2018-04-14 00:14:43","198.51.100.25","udp",53,"198-51-100-187.example.net","openresolver","p.4.0",25255,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:14:54","198.51.100.174","udp",53,"198-51-100-174.example.net","openresolver",,8445,"AT","SALZBURG","SALZBURG","6.4048",,,0,0, -"2018-04-14 00:14:54","198.51.100.181","udp",53,"198-51-100-181.example.net","openresolver","PowerDNS Recursor tele2/sil",8437,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:14:54","198.51.100.80","udp",53,"198-51-100-80.example.net","openresolver",,1901,"AT","WIEN","VIENNA","1.3810",,,518210,737415, -"2018-04-14 00:14:57","198.51.100.43","udp",53,"198-51-100-43.example.net","openresolver","vi2zcnsat10, Customer DNS",6830,"AT","STEIERMARK","GRAZ","1.3810",,,0,0, -"2018-04-14 00:14:58","198.51.100.124","udp",53,"198-51-100-124.example.net","openresolver","dnsmasq-2.47",28919,"AT","TIROL","EIBERG","3.8095",,,0,0, -"2018-04-14 00:15:00","198.51.100.60","udp",53,"198-51-100-60.example.net","openresolver",,24992,"AT","VORARLBERG","DORNBIRN","3.4762",,,0,0, -"2018-04-14 00:15:00","198.51.100.201","udp",53,"198-51-100-201.example.net","openresolver",,1853,"AT","STEIERMARK","GRAZ","4.6190",,,0,0, -"2018-04-14 00:15:01","198.51.100.82","udp",53,"198-51-100-82.example.net","openresolver","9.6-ESV-R7-P2",20811,"AT","TIROL","INNSBRUCK","4.6190",,,0,0, -"2018-04-14 00:15:01","198.51.100.105","udp",53,"198-51-100-105.example.net","openresolver",,6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:15:02","198.51.100.173","udp",53,"198-51-100-173.example.net","openresolver",,8445,"AT","NIEDEROSTERREICH","WALD","1.3810",,,0,0, -"2018-04-14 00:15:03","198.51.100.82","udp",53,"198-51-100-82.example.net","openresolver","PowerDNS Recursor tele2/sil",1257,"AT","NIEDEROSTERREICH","MODLING","1.3810",,,518111,737401, -"2018-04-14 00:15:05","198.51.100.39","udp",53,,"openresolver",,8437,"AT","VORARLBERG","LUSTENAU","1.3810",,,0,0, -"2018-04-14 00:15:09","198.51.100.33","udp",53,,"openresolver","dnsmasq-2.55",8447,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:15:09","198.51.100.248","udp",53,"198-51-100-248.example.net","openresolver",,39912,"AT","NIEDEROSTERREICH","HOLLABRUNN","3.8095",,,0,0, -"2018-04-14 00:15:10","198.51.100.119","udp",53,"198-51-100-172.example.net","openresolver","dnsmasq-2.62",25255,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:15:12","198.51.100.135","udp",53,"198-51-100-135.example.net","openresolver","no access.",43848,"AT","NIEDEROSTERREICH","WIESELBURG","3.8095",,,0,0, -"2018-04-14 00:15:15","198.51.100.64","udp",53,"198-51-100-64.example.net","openresolver",,6830,"AT","VORARLBERG","UBERSAXEN","1.3810",,,0,0, -"2018-04-14 00:15:17","198.51.100.80","udp",53,"198-51-100-80.example.net","openresolver",,42473,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:15:18","198.51.100.60","udp",53,"198-51-100-60.example.net","openresolver","198-51-100-60.example.net",35369,"AT","OBEROSTERREICH","LINZ","1.3810",,,0,0, -"2018-04-14 00:15:21","198.51.100.50","udp",53,,"openresolver","ZyWALL DNS",8447,"AT","STEIERMARK","TAUPLITZ","4.6667",,,0,0, -"2018-04-14 00:15:23","198.51.100.93","udp",53,,"openresolver","Microsoft DNS 6.1.7601 (1DB15D39)",8447,"AT","NIEDEROSTERREICH","SCHWADORF","1.3810",,,0,0, -"2018-04-14 00:15:24","198.51.100.33","udp",53,,"openresolver",,8447,"AT","STEIERMARK","FURSTENFELD","4.6190",,,0,0, -"2018-04-14 00:15:31","198.51.100.45","udp",53,,"openresolver","dnsmasq-2.52",8245,"AT","BURGENLAND","EISENSTADT","1.3810",,,0,0, -"2018-04-14 00:15:34","198.51.100.13","udp",53,"198-51-100-13.example.net","openresolver",,8447,"AT","WIEN","VIENNA","6.4048",,,518210,737415, -"2018-04-14 00:15:36","198.51.100.190","udp",53,,"openresolver",,8447,"AT","BURGENLAND","PINKAFELD","1.3810",,,0,0, -"2018-04-14 00:15:41","198.51.100.104","udp",53,,"openresolver",,21013,"AT","OBEROSTERREICH","LEONDING","2.6667",,,0,0, -"2018-04-14 00:15:42","198.51.100.101","udp",53,"198-51-100-101.example.net","openresolver",,8447,"AT","STEIERMARK","KAINACH BEI VOITSBERG","1.3810",,,0,0, -"2018-04-14 00:15:44","198.51.100.62","udp",53,"198-51-100-62.example.net","openresolver",,1901,"AT","OBEROSTERREICH","GMUNDEN","1.3810",,,518210,737415, -"2018-04-14 00:15:46","198.51.100.186","udp",53,"198-51-100-186.example.net","openresolver",,31239,"AT","WIEN","VIENNA","6.4048",,,0,0, -"2018-04-14 00:15:46","198.51.100.197","udp",53,,"openresolver",,8447,"AT","OBEROSTERREICH","KIRCHDORF AN DER KREMS","4.6190",,,0,0, -"2018-04-14 00:15:49","198.51.100.16","udp",53,,"openresolver",,8447,"AT","OBEROSTERREICH","LAAKIRCHEN","4.6190",,,0,0, -"2018-04-14 00:15:50","198.51.100.62","udp",53,"198-51-100-62.example.net","openresolver",,6830,"AT","NIEDEROSTERREICH","WIENER NEUSTADT","4.6190",,,0,0, -"2018-04-14 00:15:53","198.51.100.7","udp",53,"198-51-100-7.example.net","openresolver",,198950,"AT","TIROL","REUTTE","4.6190",,,518210,737415, -"2018-04-14 00:15:53","198.51.100.177","udp",53,"198-51-100-177.example.net","openresolver","Microsoft DNS 6.1.7601 (1DB1446A)",12605,"AT","OBEROSTERREICH","LINZ","1.3810",,,0,0, -"2018-04-14 00:15:57","198.51.100.47","udp",53,,"openresolver",,8447,"AT","NIEDEROSTERREICH","KOTTINGBRUNN","1.3810",,,0,0, -"2018-04-14 00:15:59","198.51.100.95","udp",53,"198-51-100-67.example.net","openresolver","GNS DNS Version 3",57169,"AT","WIEN","VIENNA","1.3810",,,0,0,"Information Technology" -"2018-04-14 00:16:02","198.51.100.104","udp",53,"198-51-100-104.example.net","openresolver",,6830,"AT","OBEROSTERREICH","BAD WIMSBACH-NEYDHARTING","1.3810",,,0,0, -"2018-04-14 00:16:04","198.51.100.106","udp",53,,"openresolver",,8447,"AT","STEIERMARK","GRAZ","1.3810",,,0,0, -"2018-04-14 00:16:05","198.51.100.204","udp",53,"198-51-100-204.example.net","openresolver",,12605,"AT","OBEROSTERREICH","LINZ","4.6190",,,0,0, -"2018-04-14 00:16:05","198.51.100.111","udp",53,"198-51-100-111.example.net","openresolver",,8447,"AT","OBEROSTERREICH","LINZ","1.3810",,,518210,737415, -"2018-04-14 00:16:06","198.51.100.131","udp",53,"198-51-100-139.example.net","openresolver","p.4.0",25255,"AT","OBEROSTERREICH","TRAUN","1.3810",,,0,0, -"2018-04-14 00:16:10","198.51.100.240","udp",53,"198-51-100-240.example.net","openresolver",,6830,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:16:13","198.51.100.9","udp",53,"198-51-100-42.example.net","openresolver",,13026,"AT","STEIERMARK","LEIBNITZ","6.4048",,,0,0, -"2018-04-14 00:16:15","198.51.100.231","udp",53,"198-51-100-74.example.net","openresolver",,25255,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:16:17","198.51.100.228","udp",53,"198-51-100-227.example.net","openresolver","u.1.0",25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:16:19","198.51.100.152","udp",53,"198-51-100-152.example.net","openresolver",,34694,"AT","TIROL","WORGL","4.6190",,,0,0, -"2018-04-14 00:16:21","198.51.100.88","udp",53,,"openresolver",,8447,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:16:22","198.51.100.97","udp",53,"198-51-100-97.example.net","openresolver",,8447,"AT","TIROL","INNSBRUCK","1.3810",,,518210,737415, -"2018-04-14 00:16:23","198.51.100.208","udp",53,"198-51-100-208.example.net","openresolver","dnsmasq-2.62",8447,"AT","TIROL","OTZTAL-BAHNHOF","1.3810",,,0,0, -"2018-04-14 00:16:33","198.51.100.113","udp",53,"198-51-100-121.example.net","openresolver","dnsmasq-2.62",25255,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:16:35","198.51.100.34","udp",53,"198-51-100-44.example.net","openresolver",,25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:16:37","198.51.100.236","udp",53,,"openresolver",,8447,"AT","NIEDEROSTERREICH","ST. ANDRAE-WOERDERN","4.6190",,,0,0, -"2018-04-14 00:16:40","198.51.100.46","udp",53,"198-51-100-46.example.net","openresolver",,21013,"AT","OBEROSTERREICH","LEONDING","2.6667",,,0,0, -"2018-04-14 00:16:45","198.51.100.72","udp",53,"198-51-100-5.example.net","openresolver",,25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:16:50","198.51.100.179","udp",53,"198-51-100-179.example.net","openresolver",,31125,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:16:50","198.51.100.107","udp",53,"198-51-100-107.example.net","openresolver","dnsmasq-2.66",18845,"AT","WIEN","VIENNA","1.3810",,,0,0,"Information Technology" -"2018-04-14 00:16:51","198.51.100.188","udp",53,,"openresolver","9.9.4-RedHat-9.9.4-51.el7_4.2",49322,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:16:54","198.51.100.232","udp",53,"198-51-100-232.example.net","openresolver",,6830,"AT","SALZBURG","SALZBURG","1.3810",,,0,0, -"2018-04-14 00:16:55","198.51.100.102","udp",53,"198-51-100-102.example.net","openresolver","ZyWALL DNS",6830,"AT","KARNTEN","WERNBERG","3.4762",,,0,0, -"2018-04-14 00:16:59","198.51.100.162","udp",53,"198-51-100-162.example.net","openresolver",,8445,"AT","SALZBURG","SALZBURG","1.3810",,,0,0, -"2018-04-14 00:17:00","198.51.100.110","udp",53,"198-51-100-110.example.net","openresolver",,31543,"AT","TIROL","SOLDEN","4.6190",,,0,0, -"2018-04-14 00:17:02","198.51.100.193","udp",53,"198-51-100-193.example.net","openresolver",,8447,"AT","STEIERMARK","FOHNSDORF","1.3810",,,0,0, -"2018-04-14 00:17:06","198.51.100.45","udp",53,"198-51-100-45.example.net","openresolver",,61201,"AT","KARNTEN","KLAGENFURT AM WORTHERSEE","1.3810",,,0,0, -"2018-04-14 00:17:06","198.51.100.219","udp",53,"198-51-100-219.example.net","openresolver",,8437,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:10","198.51.100.47","udp",53,"198-51-100-47.example.net","openresolver","unsupported query",8412,"AT","NIEDEROSTERREICH","WIENER NEUSTADT","3.8095",,,0,0, -"2018-04-14 00:17:13","198.51.100.87","udp",53,"198-51-100-87.example.net","openresolver","dnsmasq-2.40",6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:16","198.51.100.121","udp",53,"198-51-100-121.example.net","openresolver",,8447,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:20","198.51.100.115","udp",53,,"openresolver",,8447,"AT","TIROL","WAIDRING","1.3810",,,0,0, -"2018-04-14 00:17:22","198.51.100.235","udp",53,,"openresolver",,8447,"AT","OBEROSTERREICH","GRIESKIRCHEN","1.3810",,,0,0, -"2018-04-14 00:17:33","198.51.100.154","udp",53,,"openresolver",,8447,"AT","STEIERMARK","GRAZ","4.6190",,,0,0, -"2018-04-14 00:17:36","198.51.100.36","udp",53,"198-51-100-36.example.net","openresolver","BIND",12605,"AT","OBEROSTERREICH","LINZ","4.6190",,,0,0, -"2018-04-14 00:17:38","198.51.100.100","udp",53,"198-51-100-100.example.net","openresolver","dnsmasq-2.40",6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:41","198.51.100.181","udp",53,"198-51-100-181.example.net","openresolver","Microsoft DNS 6.0.6002 (17724D35)",6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:41","198.51.100.242","udp",53,"198-51-100-242.example.net","openresolver","Microsoft DNS 6.0.6002 (17724D35)",34767,"AT","NIEDEROSTERREICH","WIENER NEUSTADT","3.2857",,,0,0, -"2018-04-14 00:17:42","198.51.100.38","udp",53,,"openresolver","PowerDNS Recursor tele2/sil",8437,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:43","198.51.100.132","udp",53,"198-51-100-132.example.net","openresolver","PowerDNS Recursor tele2/sil",8437,"AT","STEIERMARK","GRAZ","1.3810",,,0,0, -"2018-04-14 00:17:49","198.51.100.166","udp",53,"198-51-100-166.example.net","openresolver","9.8.4-rpz2+rl005.12-P1",13292,"AT","STEIERMARK","KINDBERG","4.6190",,,0,0, -"2018-04-14 00:17:49","198.51.100.212","udp",53,"198-51-100-212.example.net","openresolver","dnsmasq-2.40",6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:51","198.51.100.225","udp",53,,"openresolver",,8220,"AT","WIEN","VIENNA","1.3810",,,518210,737415, -"2018-04-14 00:17:53","198.51.100.161","udp",53,"198-51-100-161.example.net","openresolver",,6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:54","198.51.100.12","udp",53,,"openresolver",,8447,"AT","NIEDEROSTERREICH","LANGENLOIS","1.3810",,,0,0, -"2018-04-14 00:17:55","198.51.100.113","udp",53,"198-51-100-113.example.net","openresolver",,6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:57","198.51.100.175","udp",53,"198-51-100-175.example.net","openresolver","PowerDNS Recursor tele2/sil",1257,"AT","NIEDEROSTERREICH","MODLING","1.3810",,,518111,737401, -"2018-04-14 00:17:59","198.51.100.107","udp",53,"198-51-100-107.example.net","openresolver",,50719,"AT","STEIERMARK","TIESCHEN","3.8095",,,0,0, -"2018-04-14 00:17:59","198.51.100.51","udp",53,"198-51-100-68.example.net","openresolver","dnsmasq-2.66",25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:18:04","198.51.100.131","udp",53,,"openresolver","ZyWALL DNS",8447,"AT","TIROL","OBERPERFUSS","3.4762",,,0,0, -"2018-04-14 00:18:05","198.51.100.138","udp",53,"198-51-100-138.example.net","openresolver","unsupported query",8412,"AT","NIEDEROSTERREICH","WIENER NEUSTADT","3.8095",,,0,0, -"2018-04-14 00:18:06","198.51.100.62","udp",53,"198-51-100-62.example.net","openresolver","viezcnsat13, Customer DNS",6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:18:07","198.51.100.109","udp",53,"198-51-100-109.example.net","openresolver",,1901,"AT","OBEROSTERREICH","LINZ","6.9524",,,518210,737415, -"2018-04-14 00:18:10","198.51.100.205","udp",53,"198-51-100-205.example.net","openresolver",,8437,"AT","WIEN","VIENNA","1.3810",,,0,0, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv deleted file mode 100644 index 535dc4ea8e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","sector","http","http_code","http_reason","content_type","server","date","experimental","api_version","arch","go_version","os","kernel_version","git_commit","min_api_version","build_time","pkg_version" -"2010-02-10 00:00:00",192.168.0.1,tcp,2375,node01.example.com,docker,18.05.0-ce,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",HTTP/1.1,200,OK,"application/json; charset=UTF-8","Docker/18.05.0-ce (linux)","Fri, 06 May 2022 14:06:30 GMT",false,1.37,amd64,go1.9.5,linux,3.10.0-514.26.2.el7.x86_64,f150324,1.12,2018-05-09T22:18:36.000000000+00:00, -"2010-02-10 00:00:01",192.168.0.2,tcp,2375,node02.example.com,docker,1.13.1,64512,ZZ,Region,City,0,0,,HTTP/1.1,200,OK,application/json,"Docker/1.13.1 (linux)","Fri, 06 May 2022 14:08:07 GMT",false,1.26,amd64,go1.10.3,linux,3.10.0-693.2.2.el7.x86_64,7d71120/1.13.1,1.12,2022-03-02T15:25:43.414574467+00:00,docker-1.13.1-209.git7d71120.el7.centos.x86_64 -"2010-02-10 00:00:02",192.168.0.3,tcp,2375,node03.example.com,docker,18.05.0-ce,64512,ZZ,Region,City,0,0,,HTTP/1.1,200,OK,"application/json; charset=UTF-8","Docker/18.05.0-ce (linux)","Fri, 06 May 2022 14:08:06 GMT",false,1.37,amd64,go1.9.5,linux,3.10.0-514.26.2.el7.x86_64,f150324,1.12,2018-05-09T22:18:36.000000000+00:00, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv deleted file mode 100644 index 60c7119733..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","device_vendor","device_type","device_model","device_version","device_id","device_serial","machine_name","manufacturer","method","http_port","internal_port","video_input_channels","alarm_input_channels","video_output_channels","alarm_output_channels","remote_video_input_channels","mac_address","ipv4_address","ipv4_gateway","ipv4_subnet_mask","ipv4_dhcp_enable","ipv6_address","ipv6_link_local","ipv6_gateway","ipv6_dhcp_enable","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,37810,node01.example.com,dvrdhcpdiscover,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",General,IPC,BCS-TIP3401IR-E-V,2.800.106F004.0.R,,6J0E022PAG35073,6J0E022PAG35073,General,client.notifyDevInfo,80,37777,1,0,0,0,0,38:c4:e8:03:b3:e2,192.168.0.1,192.168.0.240,255.255.255.0,0,fd09:4ab5:dae9:b078::1,fe80::3ac4:e8ff:fe03:b3e2/64,fd09:4ab5:dae9:b078::ff,0,794,794.00 -"2010-02-10 00:00:01",192.168.0.2,udp,37810,node02.example.com,dvrdhcpdiscover,64512,ZZ,Region,City,0,0,,Private,HCVR,HCVR,3.210.1.4,,2K0488CPAGS0ND6,HCVR,Private,client.notifyDevInfo,80,37777,3,0,0,0,9,3c:ef:8c:18:a5:07,192.168.0.2,192.168.0.240,255.255.255.0,0,fd09:4ab5:dae9:b078::2,fe80::3eef:8cff:fe18:a507/64,fd09:4ab5:dae9:b078::ff,,761,761.00 -"2010-02-10 00:00:02",192.168.0.3,udp,37810,node03.example.com,dvrdhcpdiscover,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",General,HCVR,BCS-XVR0401-IV,4.000.0000002.11,,5L034FAPAZA0E30,XVR,General,client.notifyDevInfo,80,37777,4,0,0,0,0,38:c4:e8:02:74:da,192.168.0.3,192.168.0.240,255.255.255.0,0,fd09:4ab5:dae9:b078::3,fe80::3ac4:e8ff:fe02:74da/64,fd09:4ab5:dae9:b078::ff,,711,711.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv deleted file mode 100644 index c681a8595d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","ok","name","cluster_name","http_code","build_hash","build_timestamp","build_snapshot","lucene_version","tagline","sector" -"2010-02-10 00:00:00",192.168.0.1,tcp,9200,node01.example.com,elasticsearch,2.3.5,64512,ZZ,Region,City,0,0,,"Red Skull",elasticsearch,,90f439ff60a3c0f497f91663701e64ccd01edbb4,2016-07-27T10:36:52Z,false,5.5.0,"You Know, for Search","Communications, Service Provider, and Hosting Service" -"2010-02-10 00:00:01",192.168.0.2,tcp,9200,node02.example.com,elasticsearch,7.17.0,64512,ZZ,Region,City,0,0,,allinonepod,docker-cluster,,bee86328705acaa9a6daede7140defd4d9ec56bd,,false,8.11.1,"You Know, for Search","Communications, Service Provider, and Hosting Service" -"2010-02-10 00:00:02",192.168.0.3,tcp,9200,node03.example.com,elasticsearch,7.15.0,64512,ZZ,Region,City,0,0,,f547c2952610,docker-cluster,,79d65f6e357953a5b3cbcc5e2c7c21073d89aa29,,false,8.9.0,"You Know, for Search","Communications, Service Provider, and Hosting Service" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv deleted file mode 100644 index 4e375a9b42..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv +++ /dev/null @@ -1,8 +0,0 @@ -"timestamp","ip","port","hostname","tag","asn","geo","region","city","naics","sic","sector","version","servername","url" -"2021-05-14 00:11:30","12.237.1.2",443,"afs-exch-cas2.xxx.com","exchange;cve-2021-26855",7018,"US","CALIFORNIA","TURLOCK",517311,,"Communications, Service Provider, and Hosting Service","15.2.721","AFS-EXCH2019", -"2021-05-14 00:11:37","98.153.3.4",443,"rrcs-98-153-x-x.west.biz.rr.com","exchange;webshell",20001,"US","CALIFORNIA","LOS ANGELES",517311,,"Communications, Service Provider, and Hosting Service","15.0.847","SSAMAIL", -"2021-05-14 00:11:38","206.210.5.6",443,"webmail.xxx.com","exchange;webshell",17054,"US","PENNSYLVANIA","PITTSBURGH",518210,,,"15.0.1178","OMNYXEXCH02", -"2021-05-14 00:11:38","12.33.7.8",443,"mail.xxx.org","exchange;cve-2021-26855",7018,"US","ARKANSAS","LITTLE ROCK",921120,,"Communications, Service Provider, and Hosting Service","15.1.2176","MHASVR02", -"2021-05-14 00:11:38","41.204.9.10",443,"mail.xxx.mg","exchange;cve-2021-26855",21042,"MG","ANTANANARIVO","ANTANANARIVO",,,,,"SABMHQE0232", -"2021-05-14 00:11:38","62.33.11.12",443,,"exchange;cve-2021-26855",20485,"RU","ALTAYSKIY KRAY","BARNAUL",,,,"15.2.659","PV-SRV04", -"2021-05-14 00:11:43","199.33.13.14",443,"mail.xxx.tv","exchange;cve-2021-26855",26481,"US","CALIFORNIA","LOS ANGELES",,,,"15.1.1779","MAIL", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv.license deleted file mode 100644 index f4e16ec677..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Birger Schacht -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv deleted file mode 100644 index 912e73d841..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","banner","handshake","cipher_suite","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serial_number","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","cert_valid","self_signed","cert_expired","validation_level","auth_tls_response","auth_ssl_response","tlsv13_support","tlsv13_cipher","jarm","device_vendor","device_type","device_model","device_version","device_sector" -"2019-03-06 06:37:00","61.126.3.70","tcp",21,"arcus-net.co.jp","ftp",4713,"JP","TOKYO","TOKYO",517311,737401,"220 FTP Server ready.|","TLSv1.2","TLS_RSA_WITH_AES_128_CBC_SHA",2048,"*.bizmw.com","GlobalSign Organization Validation CA - SHA256 - G2","Jan 14 08:04:50 2015 GMT","Jan 14 08:04:50 2020 GMT","D9:98:3F:2E:F9:D1:BE:9A:10:1E:DE:51:2C:C1:DF:01:18:0A:20:65","1121DC7421AB7924C3B1D396AEA3707E9E29",2,"sha256WithRSAEncryption","rsaEncryption","NTT Communications Corporation",,"JP","Tokyo","Minato-ku",,,,,,,,"GlobalSign nv-sa",,"BE",,,,,,,,,,"27:4A:8A:3A:A7:DF:82:D0:43:03:0E:6F:48:30:30:C9:24:77:11:1A:08:EF:F7:B9:74:0C:CE:40:87:03:D2:51","E5:93:8B:72:84:0F:35:52:8E:7A:6C:E3:EF:36:90:4C:F2:86:A7:4D:B2:DD:C0:C6:23:83:18:EF:DD:86:34:92:91:57:22:29:75:45:71:8B:3A:CD:F1:27:A9:CA:5F:70:5E:AC:15:A5:E6:63:FD:6F:BB:C5:E2:45:99:73:E9:E6","D1:A7:BC:96:78:1D:16:D0:24:A8:62:7C:3A:95:5A:4A","N","N","N","OV","234 AUTH TLS successful",,,,,,,,, -"2019-03-06 06:37:00","62.48.156.65","tcp",21,"dial-62-48-156-65.ptprime.net","ftp",15525,"PT","LISBOA","FRIELAS",0,0,"220-================================================================| PT Empresas| Acesso Reservado| Acesso nao autorizado punido por lei: 109/91; 67/98| ----------------------------------------------------------------| HENNES & MAURITZ LDA - 149093| SITE: PT303 - Cascais Shopping| MORADA: | NIR: EWS1822940| ================================================================|220 FTP server ready, 1 active clients of 4 simultaneous clients allowed.|",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"500 Syntax error, command unrecognized.","500 Syntax error, command unrecognized.",,,,,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv deleted file mode 100644 index 26f8ccbcf0..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","port","hostname","version","asn","geo","region","city","naics","sic","server_type","clusterid","total_disk","used_disk","free_disk","livenodes","namenodeaddress","volumeinfo" -"2017-09-13 02:06:05","199.116.235.200",50070,,"2.7.3, rbaa91f7c6bc9cb92be5982de4719c1c8af91ccff",15296,"CA","ALBERTA","CALGARY",0,0,"namenode","CID-64471a53-60cb-4302-9832-92f321f111fe",41567956992,53248,25160089600,"edmonton:50010",, -"2017-09-13 02:07:48","104.43.235.92",50075,,"2.7.1.2.4.0.0-169",8075,"US","IOWA","DES MOINES",334111,357101,"datanode","CID-771bae52-9e4f-4ec4-bc1a-c867585751f0",,,,,"sandbox.hortonworks.com","/hadoop/hdfs/data/current" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv.license deleted file mode 100644 index f8f131c2ce..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv deleted file mode 100644 index a7e3eb7074..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","http","http_code","http_reason","content_type","connection","www_authenticate","set_cookie","server","content_length","transfer_encoding","http_date" -"2018-04-19 00:02:26","75.74.78.113","tcp",8080,"c-75-74-78-113.hsd1.fl.comcast.net","http",7922,"US","FLORIDA","MIAMI",518111,737401,"HTTP/1.1",200,"OK","text/html",,,,"lighttpd",,"chunked","Thu, 19 Apr 2018 00:02:28 GMT" -"2018-04-19 00:02:26","88.162.174.130","tcp",8080,"sto95-3-88-162-174-130.fbx.proxad.net","http",12322,"FR",,"SAINT-OUEN-LAUMONE",518210,737415,"HTTP/1.1",200,"OK","text/html",,,,,17729,,"Thu, 19 Apr 2018 02:02:28 GMT" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv deleted file mode 100644 index b1f2330f1f..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","http","http_code","http_reason","content_type","connection","proxy_authenticate","via","server","content_length","transfer_encoding","http_date" -"2010-02-10 00:00:00",192.168.0.1,tcp,3128,node01.example.com,http-connect-proxy,64512,ZZ,Region,City,0,0,HTTP/1.1,200,"Connection established",,,,,,,,"Wed, 10 Feb 2010 00:00:00 GMT" -"2010-02-10 00:00:01",192.168.0.2,tcp,3128,node02.example.com,http-connect-proxy,64512,ZZ,Region,City,0,0,HTTP/1.1,200,"Connection established",,,,"HTTP/1.1 s_proxy_den1",,,,"Wed, 10 Feb 2010 00:00:01 GMT" -"2010-02-10 00:00:02",192.168.0.3,tcp,3128,node03.example.com,http-connect-proxy,64512,ZZ,Region,City,0,0,HTTP/1.1,200,"Connection established",,,,"HTTP/1.1 s_proxy_yvr",,,,"Wed, 10 Feb 2010 00:00:02 GMT" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv deleted file mode 100644 index 195342533e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","http","http_code","http_reason","content_type","connection","www_authenticate","set_cookie","server","content_length","transfer_encoding","http_date","version","build_date","detail" -"2010-02-10 00:00:00",192.168.0.1,tcp,8080,node01.example.com,"basic-auth,http",64512,ZZ,Region,City,0,0,HTTP/1.1,401,Unauthorized,"text/html; charset=utf-8",,"Basic realm=\"\"OpenWebif\"\"",TWISTED_SESSION=5473ad3faa3de66685fb3a53bffb390b4fcec2039893009a06caf38e1bec8aa8,TwistedWeb/19.7.0,149,,"Wed, 10 Feb 2010 00:00:00 GMT",,, -"2010-02-10 00:00:01",192.168.0.2,tcp,80,node02.example.com,"basic-auth,http",64512,ZZ,Region,City,0,0,HTTP/1.1,401,Unauthorized,"text/html; charset=utf-8",,"Basic realm=\"\"OpenWebif\"\"",TWISTED_SESSION=d2460d37b7fdbdd6c27dd74423ead5704e553d4f2c230672313edc5602059e33,TwistedWeb/19.7.0,149,,"Wed, 10 Feb 2010 00:00:01 GMT",,, -"2010-02-10 00:00:02",192.168.0.3,tcp,443,node03.example.com,git-config-file,64512,ZZ,Region,City,0,0,,,,,,,,,,,"Wed, 10 Feb 2010 00:00:02 GMT",,,"repositoryformatversion = 0;filemode = false;bare = false;logallrefupdates = true;symlinks = false;ignorecase = true" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv deleted file mode 100644 index d327f1f3ba..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","device_vendor","device_type","device_model","device_version","device_id","response_size","raw_response" -2022-03-02 00:34:22,192.168.0.1,tcp,502,host1.example.net,modbus,64512,ZZ,REGION,CITY,0,0,Sector,Vendor 1,device_type,device_model,device_version,0,5,dGVzdDE= -2022-03-02 00:34:22,192.168.0.2,tcp,502,host2.example.net,modbus,64513,ZZ,REGION,CITY,0,0,Sector,Vendor 2,device_type,device_model,device_version,0,5,dGVzdDI= -2022-03-02 00:34:22,192.168.0.3,tcp,502,host3.example.net,modbus,64514,ZZ,REGION,CITY,0,0,Sector,Vendor 3,device_type,device_model,device_version,0,5,dGVzdDM= diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv deleted file mode 100644 index 87a98157ff..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv +++ /dev/null @@ -1,96 +0,0 @@ -"timestamp","ip","port","hostname","tag","ipmi_version","asn","geo","region","city","none_auth","md2_auth","md5_auth","passkey_auth","oem_auth","defaultkg","permessage_auth","userlevel_auth","usernames","nulluser","anon_login","error","deviceid","devicerev","firmwarerev","version","manufacturerid","manufacturername","productid","productname","naics","sic","sector" -"2016-07-24 00:09:42","198.51.100.4",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:43","198.51.100.182",623,,"ipmi","2.0",28753,"DE","HESSEN","FRANKFURT AM MAIN","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:42","198.51.100.4",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:43","198.51.100.182",623,,"ipmi","2.0",28753,"DE","HESSEN","FRANKFURT AM MAIN","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:43","198.51.100.176",623,,"ipmi","1.5",2914,"DE","BAYERN","MUNICH","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:43","198.51.100.221",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:44","198.51.100.176",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:45","198.51.100.174",623,,"ipmi","1.5",2914,"DE","HESSEN","FRANKFURT AM MAIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:45","198.51.100.167",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:46","198.51.100.60",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:09:47","198.51.100.7",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:48","198.51.100.24",623,,"ipmi","2.0",24940,"DE","BAYERN","GUNZENHAUSEN","no","yes","yes","yes","yes","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:49","198.51.100.86",623,,"ipmi","1.5",2914,"DE","HESSEN","FRANKFURT AM MAIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:49","198.51.100.231",623,,"ipmi","1.5",20686,"DE","BAYERN","HAPPURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:49","198.51.100.197",623,,"ipmi","2.0",3320,"DE","BERLIN","BERLIN","no","no","yes","yes","yes","default","enabled","enabled","yes","no","yes",,,,,,,,,,541690,874899, -"2016-07-24 00:09:49","198.51.100.87",623,,"ipmi","1.5",3209,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:09:49","198.51.100.6",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:51","198.51.100.193",623,,"ipmi","2.0",15598,"DE","BAYERN","NUREMBERG","no","yes","yes","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:51","198.51.100.63",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:52","198.51.100.179",623,,"ipmi","2.0",3320,"DE","BAYERN","DENKLINGEN","no","no","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,518210,737415, -"2016-07-24 00:09:53","198.51.100.112",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:53","198.51.100.189",623,,"ipmi","2.0",30134,"DE","HESSEN","FRANKFURT AM MAIN","no","no","no","no","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0,"Communications" -"2016-07-24 00:09:54","198.51.100.44",623,"198-51-100-44.example.net","ipmi","2.0",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE","no","yes","yes","no","no","default","disabled","disabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:54","198.51.100.215",623,,"ipmi","1.5",2914,"DE","BAYERN","MUNICH","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:55","198.51.100.231",623,"198-51-100-231.example.net","ipmi","2.0",6805,"DE","HAMBURG","HAMBURG","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:55","198.51.100.234",623,,"ipmi","2.0",31103,"DE","THURINGEN","ERFURT","no","no","yes","no","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:55","198.51.100.165",623,,"ipmi","2.0",31400,"DE","RHEINLAND-PFALZ","FREINSHEIM","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:56","198.51.100.170",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:09:56","198.51.100.66",623,,"ipmi","2.0",41412,"DE","BAYERN","REGENSBURG","no","yes","yes","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:56","198.51.100.150",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:57","198.51.100.222",623,,"ipmi","2.0",34309,"DE","BERLIN","BERLIN","no","yes","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:57","198.51.100.19",623,,"ipmi","2.0",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:58","198.51.100.83",623,,"ipmi","1.5",3209,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:10:00","198.51.100.61",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:00","198.51.100.94",623,,"ipmi","1.5",2914,"DE","HESSEN","FRANKFURT AM MAIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:01","198.51.100.242",623,,"ipmi","2.0",28753,"DE","HESSEN","FRANKFURT AM MAIN","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:03","198.51.100.251",623,,"ipmi","2.0",553,"DE","BADEN-WURTTEMBERG","HEIDELBERG","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:10:03","198.51.100.41",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:04","198.51.100.160",623,"198-51-100-160.example.net","ipmi","1.5",2914,"DE","BAYERN","MUNICH","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:04","198.51.100.243",623,,"ipmi","1.5",2914,"DE","BAYERN","MUNICH","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:05","198.51.100.190",623,,"ipmi","1.5",2914,"DE","HESSEN","FRANKFURT AM MAIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:05","198.51.100.29",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:05","198.51.100.224",623,,"ipmi","2.0",13301,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:06","198.51.100.143",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","HEMER","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:07","198.51.100.120",623,,"ipmi","2.0",13003,"DE","SACHSEN","LEIPZIG","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:07","198.51.100.196",623,,"ipmi","1.5",20686,"DE","BAYERN","HAPPURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:07","198.51.100.123",623,,"ipmi","1.5",2914,"DE","HESSEN","FRANKFURT AM MAIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:08","198.51.100.122",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:08","198.51.100.192",623,,"ipmi","2.0",34171,"DE","BERLIN","BERLIN","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,518210,737415, -"2016-07-24 00:10:08","198.51.100.146",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:08","198.51.100.127",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:09","198.51.100.112",623,,"ipmi","2.0",24940,"DE","BAYERN","GUNZENHAUSEN","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:10:09","198.51.100.45",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:09","198.51.100.46",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","NEUSS","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:10","198.51.100.202",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:11","198.51.100.6",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:11","198.51.100.34",623,,"ipmi","2.0",3320,"DE","HESSEN","LEUN","no","yes","yes","no","yes","default","enabled","enabled","yes","no","no",,,,,,,,,,518210,737415, -"2016-07-24 00:10:12","198.51.100.210",623,,"ipmi","2.0",3320,"DE","BADEN-WURTTEMBERG","AALEN","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,541690,874899, -"2016-07-24 00:10:12","198.51.100.97",623,,"ipmi","2.0",42730,"DE","BERLIN","BERLIN","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:12","198.51.100.172",623,,"ipmi","1.5",2914,"DE","HESSEN","FRANKFURT AM MAIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:13","198.51.100.20",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:13","198.51.100.181",623,,"ipmi","2.0",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE","no","yes","yes","no","no","default","disabled","disabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:13","198.51.100.244",623,,"ipmi","2.0",28753,"DE","HESSEN","FRANKFURT AM MAIN","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:13","198.51.100.85",623,,"ipmi","2.0",34309,"DE","HESSEN","FRANKFURT AM MAIN","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:13","198.51.100.150",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:13","198.51.100.154",623,,"ipmi","2.0",196763,"DE","SAARLAND","ST. INGBERT","no","yes","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:13","198.51.100.83",623,,"ipmi","2.0",31342,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:14","198.51.100.6",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:14","198.51.100.228",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:14","198.51.100.150",623,,"ipmi","2.0",44066,"DE","BAYERN","MUNICH","no","yes","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:15","198.51.100.71",623,,"ipmi","2.0",44066,"DE","BAYERN","MUNICH","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:15","198.51.100.239",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:10:17","198.51.100.46",623,"198-51-100-53.example.net","ipmi","2.0",29083,"DE","BRANDENBURG","MAHLOW","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:17","198.51.100.78",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:18","198.51.100.164",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,812990,489999, -"2016-07-24 00:10:18","198.51.100.142",623,,"ipmi","2.0",34568,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:18","198.51.100.85",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:19","198.51.100.173",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:19","198.51.100.180",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:19","198.51.100.119",623,,"ipmi","2.0",12843,"DE","RHEINLAND-PFALZ","SPEYER","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:19","198.51.100.183",623,,"ipmi","1.5",12348,"DE","BAYERN","NUREMBERG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:20","198.51.100.108",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:20","198.51.100.221",623,"198-51-100-156.example.net","ipmi","2.0",24940,"DE","BAYERN","GUNZENHAUSEN","no","yes","yes","yes","yes","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:10:21","198.51.100.200",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:21","198.51.100.162",623,,"ipmi","1.5",30766,"DE","HESSEN","BENSHEIM","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:21","198.51.100.140",623,,"ipmi","2.0",31400,"DE","RHEINLAND-PFALZ","FREINSHEIM","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:21","198.51.100.121",623,,"ipmi","2.0",34549,"DE","HESSEN","FRANKFURT AM MAIN","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:22","198.51.100.33",623,,"ipmi","2.0",47215,"DE","NORDRHEIN-WESTFALEN","GUTERSLOH","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:22","198.51.100.203",623,,"ipmi","2.0",201011,"DE","BAYERN","NUREMBERG","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:23","198.51.100.16",623,,"ipmi","2.0",28753,"DE","HESSEN","FRANKFURT AM MAIN","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:24","198.51.100.166",623,,"ipmi","2.0",24940,"DE","BAYERN","GUNZENHAUSEN","no","no","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:25","198.51.100.135",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:25","198.51.100.154",623,"198-51-100-154.example.net","ipmi","1.5",2914,"DE","HESSEN","FRANKFURT AM MAIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:25","198.51.100.237",623,,"ipmi","2.0",12586,"DE","HESSEN","FRANKFURT AM MAIN","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:25","198.51.100.45",623,,"ipmi","2.0",13301,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv deleted file mode 100644 index a585db6eb6..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv +++ /dev/null @@ -1,2 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","ipp_version","cups_version","printer_uris","printer_name","printer_info","printer_more_info","printer_make_and_model","printer_firmware_name","printer_firmware_string_version","printer_firmware_version","printer_organization","printer_organization_unit","printer_uuid","printer_wifi_ssid","device_vendor","device_type","device_model","device_version","device_sector" -"2020-06-08 11:30:14","123.45.67.89","tcp",631,"some.host.com","ipp",12345,"AA","REGION","CITY",517311,0,"IPP/2.1","CUPS/2.0","ipp://123.45.67.89:631/ipp/print","NPI3F0D22","HP Color LaserJet MFP M277dw","http://123.45.67.89:631/hp/device/info_config_AirPrint.html?tab=Networking&menu=AirPrintStatus","HP Color LaserJet MFP M277dw",20191203,20191203,20191203,"org","unit","urn:uuid:456e4238-4a44-4643-4c42-10e1813f0a18","wifissid",,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv.license deleted file mode 100644 index 476908eebe..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2020 Thomas Hungenberg -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv deleted file mode 100644 index cef6b027c6..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","initiator_spi","responder_spi","next_payload","exchange_type","flags","message_id","next_payload2","domain_of_interpretation","protocol_id","spi_size","notify_message_type" -"2019-09-04 00:17:25","198.123.245.42","udp",500,"example.local","isakmp-vulnerable",5678,"AA","LOCATION","LOCATION",517311,0,"3e35c70729dfedef","253acab7cbfda607",11,05,00,00000000,00,00,,0,14 -"2019-09-04 00:17:28","198.123.245.67","udp",500,"example.local","isakmp-vulnerable",20255,"AA","LOCATION","LOCATION",0,0,"3e35c70729dfedef","b274460e7adc1bf0",11,05,00,00000000,00,00,,0,14 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv deleted file mode 100644 index ab71b9a15d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","sector","http","http_code","http_reason","content_type","server","date","major","minor","git_version","git_commit","git_tree_state","build_date","go_version","compiler","platform","handshake","cipher_suite","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serial_number","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","cert_valid","self_signed","cert_expired","validation_level","browser_trusted","browser_error","raw_cert","raw_cert_chain" -"2010-02-10 00:00:00",192.168.0.1,tcp,6443,node01.example.com,kubernetes,,64512,ZZ,Region,City,0,0,,HTTP/1.1,200,OK,application/json,,"Tue, 10 May 2022 14:24:13 GMT",1,20,v1.20.13,2444b3347a2c45eb965b182fb836e1f51dc61b70,clean,2021-11-17T13:00:29Z,go1.15.15,gc,linux/amd64,TLSv1.2,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,2,sha256WithRSAEncryption,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,N,Y,unknown,N,"x509: failed to load system roots and no roots provided",, -"2010-02-10 00:00:01",192.168.0.2,tcp,6443,node02.example.com,kubernetes,,64512,ZZ,Region,City,0,0,"Retail Trade",HTTP/1.1,200,OK,application/json,,"Tue, 10 May 2022 14:24:12 GMT",1,23,v1.23.3+e419edf,6f5a5295923a614a4202a7ad274b38b69f9ca8c0,clean,2022-02-25T06:26:46Z,go1.17.5,gc,linux/amd64,TLSv1.2,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,2,sha256WithRSAEncryption,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,N,Y,unknown,N,"x509: failed to load system roots and no roots provided",, -"2010-02-10 00:00:02",192.168.0.3,tcp,6443,node03.example.com,kubernetes,,64512,ZZ,Region,City,0,0,,HTTP/1.1,200,OK,application/json,,"Tue, 10 May 2022 14:24:12 GMT",1,16+,v1.16.9-aliyun.1,4f7ea78,,2020-05-08T07:29:59Z,go1.13.9,gc,linux/amd64,TLSv1.2,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,2,sha256WithRSAEncryption,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,N,Y,unknown,N,"x509: failed to load system roots and no roots provided",, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv deleted file mode 100644 index 54121fd3b7..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","size","configuration_naming_context","current_time","default_naming_context","dns_host_name","domain_controller_functionality","domain_functionality","ds_service_name","forest_functionality","highest_committed_usn","is_global_catalog_ready","is_synchronized","ldap_service_name","naming_contexts","root_domain_naming_context","schema_naming_context","server_name","subschema_subentry","supported_capabilities","supported_control","supported_ldap_policies","supported_ldap_version","supported_sasl_mechanisms","amplification" -"2010-02-10 00:00:00",192.168.0.1,tcp,389,node01.example.com,ldap-tcp,64512,ZZ,Region,City,0,0,0,"CN=Configuration,DC=ad,DC=example,DC=com",,"DC=ad,DC=example,DC=com",node01.example.com,7,,"CN=Configuration,DC=ad,DC=example,DC=com",2,,,,node01.example.com,"DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com","DC=example,DC=com","CN=Schema,CN=Configuration,DC=example,DC=com","CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com","CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com",,1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|,MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent,3|2,GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5, -"2010-02-10 00:00:01",192.168.0.2,tcp,389,node02.example.com,ldap-tcp,64512,ZZ,Region,City,0,0,0,"CN=Configuration,DC=ad,DC=example,DC=com",20220821124435.0Z,"DC=ad,DC=example,DC=com",node02.example.com,7,7,"CN=Configuration,DC=ad,DC=example,DC=com",7,25029662,TRUE,TRUE,node02.example.com,"DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com","DC=example,DC=com","CN=Schema,CN=Configuration,DC=example,DC=com","CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com","CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com",1.2.840.113556.1.4.800|1.2.840.113556.1.4.1670|1.2.840.113556.1.4.1791|1.2.840.113556.1.4.1935|1.2.840.113556.1.4.2080|1.2.840.113556.1.4.2237,1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|1.2.840.113556.1.4.2206|1.2.840.113556.1.4.2211|1.2.840.113556.1.4.2239|1.2.840.113556.1.4.2255|1.2.840.113556.1.4.2256|1.2.840.113556.1.4.2309,MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent,3|2,GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5, -"2010-02-10 00:00:02",192.168.0.3,tcp,389,node03.example.com,ldap-tcp,64512,ZZ,Region,City,0,0,0,"CN=Configuration,DC=ad,DC=example,DC=com",20220821124539.0Z,"DC=ad,DC=example,DC=com",node03.example.com,,,"CN=Configuration,DC=ad,DC=example,DC=com",,,,,node03.example.com,"DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com","DC=example,DC=com","CN=Schema,CN=Configuration,DC=example,DC=com","CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com","CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com",,1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv deleted file mode 100644 index 3cd5021c54..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","size","configuration_naming_context","current_time","default_naming_context","dns_host_name","domain_controller_functionality","domain_functionality","ds_service_name","forest_functionality","highest_committed_usn","is_global_catalog_ready","is_synchronized","ldap_service_name","naming_contexts","root_domain_naming_context","schema_naming_context","server_name","subschema_subentry","supported_capabilities","supported_control","supported_ldap_policies","supported_ldap_version","supported_sasl_mechanisms","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,389,node01.example.com,ldap-udp,64512,ZZ,Region,City,0,0,3038,"CN=Configuration,DC=ad,DC=example,DC=com",20220821044533.0Z,"DC=ad,DC=example,DC=com",node01.example.com,7,7,"CN=Configuration,DC=ad,DC=example,DC=com",7,222537,TRUE,TRUE,node01.example.com,"DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com","DC=example,DC=com","CN=Schema,CN=Configuration,DC=example,DC=com","CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com","CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com",1.2.840.113556.1.4.800|1.2.840.113556.1.4.1670|1.2.840.113556.1.4.1791|1.2.840.113556.1.4.1935|1.2.840.113556.1.4.2080|1.2.840.113556.1.4.2237,1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|1.2.840.113556.1.4.2206|1.2.840.113556.1.4.2211|1.2.840.113556.1.4.2239|1.2.840.113556.1.4.2255|1.2.840.113556.1.4.2256|1.2.840.113556.1.4.2309|1.2.840.113556.1.4.2330|1.2.840.113556.1.4.2354,MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent,3|2,GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5,58.42 -"2010-02-10 00:00:01",192.168.0.2,udp,389,node02.example.com,ldap-udp,64512,ZZ,Region,City,0,0,3062,"CN=Configuration,DC=ad,DC=example,DC=com",20220821044948.0Z,"DC=ad,DC=example,DC=com",node02.example.com,7,7,"CN=Configuration,DC=ad,DC=example,DC=com",7,1478714,TRUE,TRUE,node02.example.com,"DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com","DC=example,DC=com","CN=Schema,CN=Configuration,DC=example,DC=com","CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com","CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com",1.2.840.113556.1.4.800|1.2.840.113556.1.4.1670|1.2.840.113556.1.4.1791|1.2.840.113556.1.4.1935|1.2.840.113556.1.4.2080|1.2.840.113556.1.4.2237,1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|1.2.840.113556.1.4.2206|1.2.840.113556.1.4.2211|1.2.840.113556.1.4.2239|1.2.840.113556.1.4.2255|1.2.840.113556.1.4.2256|1.2.840.113556.1.4.2309|1.2.840.113556.1.4.2330|1.2.840.113556.1.4.2354,MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent,3|2,GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5,58.88 -"2010-02-10 00:00:02",192.168.0.3,udp,389,node03.example.com,ldap-udp,64512,ZZ,Region,City,0,0,36,"CN=Configuration,DC=ad,DC=example,DC=com",,"DC=ad,DC=example,DC=com",node03.example.com,,,"CN=Configuration,DC=ad,DC=example,DC=com",,,,,node03.example.com,"DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com","DC=example,DC=com","CN=Schema,CN=Configuration,DC=example,DC=com","CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com","CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com",,,,,,0.69 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv deleted file mode 100644 index 4a97121e75..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","mdns_name","mdns_ipv4","mdns_ipv6","services","workstation_name","workstation_ipv4","workstation_ipv6","workstation_info","http_name","http_ipv4","http_ipv6","http_ptr","http_info","http_target","http_port","spotify_name","spotify_ipv4","spotify_ipv6","opc_ua_discovery" -"2010-02-10 00:00:00",192.168.0.1,udp,5353,node01.example.com,mdns,64512,ZZ,Region,City,0,0,,,,"_smb._tcp.local.; _device-info._tcp.local.; _http._tcp.local.; _dacp._tcp.local.;",,192.168.0.1,fd09:4ab5:dae9:b078::1,,,192.168.0.1,fd09:4ab5:dae9:b078::1,,,,,,,, -"2010-02-10 00:00:01",192.168.0.2,udp,5353,node02.example.com,mdns,64512,ZZ,Region,City,0,0,,,,_home-assistant._tcp.local.;,,192.168.0.2,fd09:4ab5:dae9:b078::2,,,192.168.0.2,fd09:4ab5:dae9:b078::2,,,,,,,, -"2010-02-10 00:00:02",192.168.0.3,udp,5353,node03.example.com,"mdns,iot",64512,ZZ,Region,City,0,0,,,,"_webdav._tcp.local.; _adisk._tcp.local.; _smb._tcp.local.; _http._tcp.local.; _dacp._tcp.local.; _afpovertcp._tcp.local.; _device-info._tcp.local.;",,192.168.0.3,fd09:4ab5:dae9:b078::3,,snmeijer.local.,192.168.0.3,fd09:4ab5:dae9:b078::3,snmeijer._http._tcp.local.,"\"\"vendor=Synology\"\" \"\"model=DS218+\"\" \"\"serial=17A0PCN482002\"\" \"\"version_major=6\"\" \"\"version_minor=2\"\" \"\"version_build=25556\"\" \"\"admin_port=5000\"\" \"\"secure_admin_port=5001\"\" \"\"mac_address=00:11:32:80:fd:b5\"\"",snmeijer.local.,5000,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv deleted file mode 100644 index 6a1d445e7a..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","pid","pointer_size","uptime","time","curr_connections","total_connections","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,50260,node01.example.com,memcached,1.4.15,64512,ZZ,Region,City,0,0,1010,64,32908114,"2022-08-21 10:34:06",243,6106,"Communications, Service Provider, and Hosting Service",1144,81.71 -"2010-02-10 00:00:01",192.168.0.2,udp,11211,node02.example.com,memcached,1.4.13,64512,ZZ,Region,City,0,0,5316,64,9618498,"2022-08-21 10:39:21",9,2962,"Communications, Service Provider, and Hosting Service",1053,75.21 -"2010-02-10 00:00:02",192.168.0.3,udp,11211,node03.example.com,memcached,1.2.6,64512,ZZ,Region,City,0,0,1460,32,1375159,"2022-08-21 10:39:39",2,534,,442,31.57 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv deleted file mode 100644 index 1228dcfc60..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv +++ /dev/null @@ -1,11 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","gitversion","sysinfo","opensslversion","allocator","javascriptengine","bits","maxbsonobjectsize","ok","visible_databases","sector" -"2016-07-24 00:40:07","198.51.100.203","tcp",27017,"198-51-100-203.example.net","mongodb","2.4.5",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,"a2ddc68ba7c9cee17bfe69ed840383ec3506602b","Linux ip-198-51-100-100 198.51.100.103-2.ec2.v1.2.fc8xen #1 SMP Fri Nov 20 17:48:28 EST 2009 x86_64 BOOST_LIB_VERSION=1_49",,"tcmalloc","V8",64,16777216,1,"local | countly | admin", -"2016-07-24 00:40:07","198.51.100.42","tcp",27017,"198-51-100-208.example.net","mongodb","2.6.12",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,"d73c92b1c85703828b55c2916a5dd4ad46535f6a","Linux build5.ny.cbi.10gen.cc 2.6.32-431.3.1.el6.x86_64 #1 SMP Fri Jan 3 21:39:27 UTC 2014 x86_64 BOOST_LIB_VERSION=1_49",,"tcmalloc","V8",64,16777216,1,"none visible","Information Technology" -"2016-07-24 00:40:07","198.51.100.225","tcp",27017,"198-51-100-225.example.net","mongodb","3.0.6",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,"1ef45a23a4c5e3480ac919b28afcba3c615488f2","Linux ip-198-51-100-100 3.4.43-43.43.amzn1.x86_64 #1 SMP Mon May 6 18:04:41 UTC 2013 x86_64 BOOST_LIB_VERSION=1_49","OpenSSL 1.0.0-fips 29 Mar 2010","tcmalloc","V8",64,16777216,1,"bluu | local","Communications" -"2016-07-24 00:40:07","198.51.100.144","tcp",27017,"198-51-100-144.example.net","mongodb","2.2.2",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,"d1b43b61a5308c4ad0679d34b262c5af9d664267","Linux ip-198-51-100-100 198.51.100.252-2.ec2.v1.2.fc8xen #1 SMP Fri Nov 20 17:48:28 EST 2009 x86_64 BOOST_LIB_VERSION=1_49",,,,64,16777216,1,"errbit_production | DELETED_BECAUSE_YOU_DIDNT_PASSWORD_PROTECT_YOUR_MONGODB | admin | local", -"2016-07-24 00:40:07","198.51.100.68","tcp",27017,,"mongodb","3.2.6",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,"05552b562c7a0b3143a729aaa0838e558dc49b25","deprecated",,"tcmalloc","mozjs",64,16777216,1,"none visible", -"2016-07-24 00:40:07","198.51.100.101","tcp",27017,,"mongodb","3.0.9",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,"20d60d3491908f1ae252fe452300de3978a040c7","Linux ip-198-51-100-100 3.13.0-24-generic #46-Ubuntu SMP Thu Apr 10 19:11:08 UTC 2014 x86_64 BOOST_LIB_VERSION=1_49","OpenSSL 1.0.1f 6 Jan 2014","tcmalloc","V8",64,16777216,1,"none visible", -"2016-07-24 00:40:07","198.51.100.53","tcp",27017,"198-51-100-162.example.net","mongodb","3.2.6",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,"05552b562c7a0b3143a729aaa0838e558dc49b25","deprecated",,"tcmalloc","mozjs",64,16777216,1,"none visible", -"2016-07-24 00:40:07","198.51.100.206","tcp",27017,"198-51-100-206.example.net","mongodb","2.4.10",34011,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,"e3d78955d181e475345ebd60053a4738a4c5268a","Linux bs-linux32.10gen.cc 198.51.100.34-2.fc8xen #1 SMP Fri Feb 15 12:39:36 EST 2008 i686 BOOST_LIB_VERSION=1_49",,"system","V8",32,16777216,1,"sharelatex | test1 | local | tmp | lococms_production", -"2016-07-24 00:40:10","198.51.100.157","tcp",27017,"198-51-100-157.example.net","mongodb","2.0.6",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,"nogitversion","Linux biber 2.6.32-5-amd64 #1 SMP Mon Feb 25 00:26:11 UTC 2013 i686 BOOST_LIB_VERSION=1_49",,,,32,16777216,1,"none visible", -"2016-07-24 00:40:10","198.51.100.173","tcp",27017,"198-51-100-173.example.net","mongodb","2.6.12",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,"nogitversion","FreeBSD 101amd64-default-job-24 10.1-RELEASE-p33 FreeBSD 10.1-RELEASE-p33 amd64 BOOST_LIB_VERSION=1_49","OpenSSL 1.0.1l-freebsd 15 Jan 2015","system","V8",64,16777216,1,"none visible", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv deleted file mode 100644 index cfe4f00614..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv +++ /dev/null @@ -1,2 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","anonymous_access","raw_response","hex_code","code","cipher_suite","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serialNumber" -"2022-02-07 12:56:53","18.220.0.0","tcp",8883,"18-220-0-0.example.com","mqtt",12345,"US","OHIO","COLUMBUS",454110,,"N",20020005,05,"Connection Refused, not authorized","TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",2048,"*.tracesafe.io","Sectigo RSA Domain Validation Secure Server CA","2020-08-12 00:00:00","2022-11-14 00:00:00","70:84:F1:6D:28:DA:B6:E6:27:60:13:8B:2C:93:52:B6:7B:4B:13:7B","D2:D7:54:52:EB:86:4E:2D:34:4D:FC:CE:CD:CF:39:41:E1:06:5C:8B:B8:54:E6:0C:DF:FD:6E:E3:F1:B5:41:00","17:57:FB:88:9D:BE:A7:F0:29:A5:31:FC:79:DF:F7:8A:1C:D6:4A:DF:1B:4A:DC:BF:05:E7:E8:2F:79:9A:FA:FE:F7:E8:66:22:CB:B9:4C:72:F7:FB:6C:1D:59:8C:54:63:70:05:DE:7F:3C:2F:BA:B8:37:18:CE:29:6F:11:E8:AB","DE:2C:98:30:27:2E:7D:C9:ED:A3:9D:AF:9E:CE:14:CC","085699743A23114C9B6B8DC975A8AF42",2,"sha256WithRSAEncryption","rsaEncryption",,,,,,,,,,,,,"Sectigo Limited",,"GB","Greater Manchester","Salford",,,,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv.license deleted file mode 100644 index 476908eebe..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2020 Thomas Hungenberg -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv deleted file mode 100644 index e0ab4b9298..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","raw_response","hex_code","code","cipher_suite","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serialNumber" -"2022-01-10 00:59:34","47.106.0.0","tcp",8883,,"mqtt,mqtt-anon",37963,"CN","GUANGDONG SHENG","SHENZHEN",518210,,20020000,00,"Connection Accepted","TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",2048,"Server","RootCA","2020-05-08 08:07:05","2030-05-06 08:07:05","70:1A:1E:1F:EC:5F:7E:A9:12:32:B2:C9:8A:C9:EE:91:8E:0B:82:45","85:26:A2:F2:A2:50:CD:96:33:19:A6:2D:12:2E:97:6B:D3:06:3C:11:EA:01:B4:B7:25:2A:B7:4F:0A:8F:45:40","72:50:07:30:9A:6F:CB:FD:E2:80:69:02:65:62:77:16:C3:B4:0C:98:44:4E:D4:2C:AC:6B:AF:F8:9E:AB:51:C2:FA:A8:72:A3:45:DF:81:09:50:08:18:EB:03:34:FC:92:33:A7:12:46:FE:90:20:91:86:C5:4D:89:48:86:4C:CD","AB:A8:E0:2C:EF:AE:BF:9D:DD:FA:70:BA:2F:F2:CA:5C",02,2,"sha256WithRSAEncryption","rsaEncryption","EMQ",,"CN","hangzhou",,,,,,,,,"EMQ",,"CN","hangzhou",,,,,,,, -"2022-01-10 00:59:34","144.76.0.0","tcp",8883,,"mqtt,mqtt-anon",24940,"DE","SACHSEN-ANHALT","WERNIGERODE",518210,,20020000,00,"Connection Accepted","TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",2048,"example.com","R3","2021-12-06 13:48:04","2022-03-06 13:48:03","20:0E:AC:E7:AF:07:8D:D3:16:7C:63:D1:B9:12:AD:1D:2C:F0:46:86","DD:7A:4C:8A:1D:66:1D:7C:F5:17:04:5B:A0:B4:C4:E0:80:58:44:B4:DB:A7:5E:61:AE:43:9D:85:4C:9E:DC:83","55:B6:3D:56:A4:39:6E:99:B6:AF:72:AF:4D:3C:7C:C5:A8:C5:4F:A1:79:92:D0:46:8A:A2:9B:2A:48:0D:00:68:39:F0:B8:67:B4:E0:88:51:2A:D7:55:46:83:BD:ED:1E:09:6E:DB:3D:21:E2:AA:DB:42:6A:33:45:1A:2A:DB:4C","23:99:39:C6:77:D8:9F:55:90:FC:A5:FB:BA:72:8B:42","06B25BEAD1F43266ABCFCDDE408D3544D04B",2,"sha256WithRSAEncryption","rsaEncryption",,,,,,,,,,,,,"Lets Encrypt",,"US",,,,,,,,, -"2022-01-10 00:59:34","173.0.0.0","tcp",8883,"example.com","mqtt,mqtt-anon",5555,"US","CALIFORNIA","BURBANK",,,20020000,00,"Connection Accepted","TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA",2048,"foo.example.com","ClearView2Dev","2020-08-07 16:51:57","2030-08-05 16:51:57","32:4B:66:98:FA:5B:D2:D1:F2:53:83:21:19:11:5A:A9:BE:85:56:16","AE:0D:65:34:2F:51:F7:32:1E:DF:B1:DA:12:C7:6A:DE:42:B5:4B:FF:80:2C:E5:EF:99:F6:CC:01:4B:C9:77:68","44:C4:B8:19:FA:39:55:51:EC:E4:6D:C4:6D:0F:A5:46:BB:D5:F9:FD:A6:8D:DF:F3:2D:D2:92:6C:0B:D5:D3:25:CB:19:50:9D:A6:A4:D4:D3:2E:53:10:F5:8D:77:F7:90:F8:65:A7:79:AB:14:62:72:01:F3:EA:38:E2:68:C7:25","43:0D:A7:89:9E:76:8D:6E:D5:AD:95:CC:F2:91:87:56","A71541EFAE529B03",0,"sha256WithRSAEncryption","rsaEncryption","Sohonet",,,,"<",,,,,,,,"Sohonet","ClearView2Dev",,,,,,,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv deleted file mode 100644 index c12a6063eb..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","server_name","instance_name","tcp_port","named_pipe","response_size","amplification","sector" -"2010-02-10 00:00:00",192.168.0.1,udp,1434,node01.example.com,mssql,13.2.5026.0,64512,ZZ,Region,City,0,0,ERPOPTIMA,OPTIMA,49729,"\\\\ERPOPTIMA\\pipe\\MSSQL$OPTIMA\\sql\\query",310,310.00, -"2010-02-10 00:00:01",192.168.0.2,udp,1434,node02.example.com,mssql,13.0.1601.5,64512,ZZ,Region,City,0,0,SERWER,MSSQLSERVER,1433,,226,226.00,"Communications, Service Provider, and Hosting Service" -"2010-02-10 00:00:02",192.168.0.3,udp,1434,node03.example.com,mssql,10.50.2500.0,64512,ZZ,Region,City,0,0,ILONY,INSERTGT,49358,"\\\\ILONY\\pipe\\MSSQL$INSERTGT\\sql\\query",304,304.00, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv deleted file mode 100644 index 25fed2166b..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","sector","mysql_protocol_version","server_version","error_code","error_id","error_message","client_can_handle_expired_passwords","client_compress","client_connect_attrs","client_connect_with_db","client_deprecated_eof","client_found_rows","client_ignore_sigpipe","client_ignore_space","client_interactive","client_local_files","client_long_flag","client_long_password","client_multi_results","client_multi_statements","client_no_schema","client_odbc","client_plugin_auth","client_plugin_auth_len_enc_client_data","client_protocol_41","client_ps_multi_results","client_reserved","client_secure_connection","client_session_track","client_ssl","client_transactions","handshake","cipher_suite","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serial_number","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","cert_valid","self_signed","cert_expired","validation_level","browser_trusted","browser_error","raw_cert","raw_cert_chain" -"2010-02-10 00:00:00",192.168.0.1,tcp,3306,node01.example.com,mysql,,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",10,5.7.37-0ubuntu0.18.04.1,1,1,1,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,N,N,N,TLSv1.2,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,2,sha256WithRSAEncryption,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,,Y,unknown,,"x509: failed to load system roots and no roots provided",, -"2010-02-10 00:00:01",192.168.0.2,tcp,3306,node02.example.com,mysql,,64512,ZZ,Region,City,0,0,,10,5.7.30-0ubuntu0.18.04.1-log,1,1,1,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,N,N,N,TLSv1.2,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,2,sha256WithRSAEncryption,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,,Y,unknown,,"x509: failed to load system roots and no roots provided",, -"2010-02-10 00:00:02",192.168.0.3,tcp,3306,node03.example.com,mysql,,64512,ZZ,Region,City,0,0,"Retail Trade",10,8.0.23,1,1,1,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,N,N,N,TLSv1.2,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,2,sha256WithRSAEncryption,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,,Y,unknown,,"x509: failed to load system roots and no roots provided",, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv deleted file mode 100644 index e8a1108d5a..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","opcode","uptime","external_ip","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,5351,node01.example.com,nat-pmp,0,64512,ZZ,Region,City,0,0,128,291278940,192.168.0.1,,12,6.00 -"2010-02-10 00:00:01",192.168.0.2,udp,5351,node02.example.com,nat-pmp,0,64512,ZZ,Region,City,0,0,128,768416,192.168.0.2,,12,6.00 -"2010-02-10 00:00:02",192.168.0.3,udp,5351,node03.example.com,nat-pmp,0,64512,ZZ,Region,City,0,0,128,19629454,192.168.0.3,,12,6.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv deleted file mode 100644 index 932225b0b0..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","mac_address","asn","geo","region","city","workgroup","machine_name","username","naics","sic","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,137,node01.example.com,netbios,00-00-00-00-00-00,64512,ZZ,Region,City,,NBG6503,NBG6503,0,0,,229,4.58 -"2010-02-10 00:00:01",192.168.0.2,udp,137,node02.example.com,netbios,00-00-00-00-00-00,64512,ZZ,Region,City,PRACOWNIAELN.,NAS-OLD,NAS-OLD,0,0,,193,3.86 -"2010-02-10 00:00:02",192.168.0.3,udp,137,node03.example.com,netbios,00-25-90-F0-64-64,64512,ZZ,Region,City,HRSIGMA,HR-SRV01,,0,0,Government,157,3.14 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv deleted file mode 100644 index 4e91593565..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","port","hostname","tag","response","asn","geo","region","city","naics","sic","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,53413,node01.example.com,netis_vulnerability,Login:,64512,ZZ,Region,City,0,0,,18,18.00 -"2010-02-10 00:00:01",192.168.0.2,53413,node02.example.com,netis_vulnerability,Login:,64512,ZZ,Region,City,0,0,,18,18.00 -"2010-02-10 00:00:02",192.168.0.3,53413,node03.example.com,netis_vulnerability,Login:,64512,ZZ,Region,City,0,0,,18,18.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv deleted file mode 100644 index cc3cf6fc2f..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","version","clk_wander","clock","error","frequency","jitter","leap","mintc","noise","offset","peer","phase","poll","precision","processor","refid","reftime","rootdelay","rootdispersion","stability","state","stratum","system","tai","tc","naics","sic","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,123,node01.example.com,ntpversion,64512,ZZ,Region,City,4,,0xe6ac3809.363028e7,,2.018,0.977,0,,0.984,0.557,18986,,10,-10,unknown,81.15.252.130,0xe6ac35ba.2d2e8f2b,17.685,61.254,0.027,4,4,UNIX,,,0,0,,324,27.00 -"2010-02-10 00:00:01",192.168.0.2,udp,123,node02.example.com,ntpversion,64512,ZZ,Region,City,4,0.007,0xE6AC3806.7DF3B7A0,,-20.407,8.776,0,3,,-14.502,19244,,,-10,unknown,10.48.21.21,0xE6AC3431.B3B64790,32.25,105.778,,,8,UNIX,,10,0,0,"Transportation and Warehousing",328,27.33 -"2010-02-10 00:00:02",192.168.0.3,udp,123,node03.example.com,ntpversion,64512,ZZ,Region,City,4,0.001,0xE6AC380A.5A1CAD00,,-24.01,2.343,0,3,,0.49,51892,,,-10,unknown,172.28.0.1,0xE6AC3020.0C49BA80,7.749,81.612,,,4,UNIX,,10,0,0,,324,27.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv deleted file mode 100644 index dca5386d9e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","packets","size","asn","geo","region","city","naics","sic","sector","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,123,node01.example.com,2,664,64512,ZZ,Region,City,0,0,,55.33 -"2010-02-10 00:00:01",192.168.0.2,udp,123,node02.example.com,100,44000,64512,ZZ,Region,City,0,0,,3666.67 -"2010-02-10 00:00:02",192.168.0.3,udp,123,node03.example.com,100,44000,64512,ZZ,Region,City,0,0,,3666.67 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv deleted file mode 100644 index c32bc3d4d0..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","programs","mountd_port","exports","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,111,node01.example.com,portmapper,64512,ZZ,Region,City,0,0,"100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp; 100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp;",,"/mnt/export 192.168.0.0","Communications, Service Provider, and Hosting Service",148,3.70 -"2010-02-10 00:00:01",192.168.0.2,udp,111,node02.example.com,portmapper,64512,ZZ,Region,City,0,0,"100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp; 100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp;",,"/mnt/export 192.168.0.0","Communications, Service Provider, and Hosting Service",148,3.70 -"2010-02-10 00:00:02",192.168.0.3,udp,111,node03.example.com,portmapper,64512,ZZ,Region,City,0,0,"100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp; 100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp;",,"/mnt/export 192.168.0.0",Government,148,3.70 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv deleted file mode 100644 index 8c1d6f725a..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","sector","supported_protocols","protocol_error_code","protocol_error_file","protocol_error_line","protocol_error_message","protocol_error_routine","protocol_error_severity","protocol_error_severity_v","startup_error_code","startup_error_file","startup_error_line","startup_error_message","startup_error_routine","startup_error_severity","startup_error_severity_v","client_ssl","handshake","cipher_suite","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serial_number","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","cert_valid","self_signed","cert_expired","validation_level","browser_trusted","browser_error","raw_cert","raw_cert_chain" -"2010-02-10 00:00:00",192.168.0.1,tcp,5432,node01.example.com,postgres,,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",1.0-3.0,0A000,postmaster.c,1798,"unsupported frontend protocol 255.255: server supports 1.0 to 3.0",ProcessStartupPacket,FATAL,,28000,postmaster.c,1893,"no PostgreSQL user name specified in startup packet",ProcessStartupPacket,FATAL,,N,,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,,,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,,Y,,,,, -"2010-02-10 00:00:01",192.168.0.2,tcp,5432,node02.example.com,postgres,,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",1.0-3.0,0A000,postmaster.c,1798,"unsupported frontend protocol 255.255: server supports 1.0 to 3.0",ProcessStartupPacket,FATAL,,28000,postmaster.c,1893,"no PostgreSQL user name specified in startup packet",ProcessStartupPacket,FATAL,,N,,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,,,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,,Y,,,,, -"2010-02-10 00:00:02",192.168.0.3,tcp,5432,node03.example.com,postgres,,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",1.0-3.0,0A000,postmaster.c,1798,"unsupported frontend protocol 255.255: server supports 1.0 to 3.0",ProcessStartupPacket,FATAL,,28000,postmaster.c,1893,"no PostgreSQL user name specified in startup packet",ProcessStartupPacket,FATAL,,N,,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,,,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,,Y,,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv deleted file mode 100644 index 857699376e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","quote","asn","geo","region","city","naics","sic","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,17,node01.example.com,qotd,"_The secret of being miserable is to have leisure to bother about whether?? you are happy or not. The cure for it is occupation._?? George Bernard Shaw (1856-1950)?",64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",166,166.00 -"2010-02-10 00:00:01",192.168.0.2,udp,17,node02.example.com,qotd,"_Oh the nerves, the nerves; the mysteries of this machine called man!?? Oh the little that unhinges it, poor creatures that we are!_?? Charles Dickens (1812-70)?",64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",162,162.00 -"2010-02-10 00:00:02",192.168.0.3,udp,17,node03.example.com,qotd,"_Oh the nerves, the nerves; the mysteries of this machine called man!?? Oh the little that unhinges it, poor creatures that we are!_?? Charles Dickens (1812-70)?",64512,ZZ,Region,City,0,0,,162,162.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv deleted file mode 100644 index c9fb18896e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","version_field_1","version_field_2","version_field_3","version_field_4" -"2022-01-10 14:31:17","176.255.0.0","udp",443,"test1.example.com","quic",5607,"UK","LONDON","LONDON",517311,,"Q050",,"Q046","Q043" -"2022-01-10 14:31:17","24.244.0.0","udp",443,,"quic",6327,"CA","SASKATCHEWAN","MEACHAM",517311,,"Q050","Q046",,"Q043" -"2022-01-10 14:31:17","23.60.0.0","udp",443,"test3.example.com","quic",20940,"JP","OSAKA","OSAKA",517919,,,"Q050","Q046","Q043" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv deleted file mode 100644 index 76b388acaa..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv +++ /dev/null @@ -1,10 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic" -"2020-07-06 13:55:26","74.101.218.75","tcp",4899,"static-74-101-218-75.nycmny.fios.verizon.net","radmin","Radmin (Details Unknown)",701,"US","NEW YORK","BROOKLYN",517312, -"2020-07-06 13:55:27","192.162.189.171","tcp",4899,"rubin.an.ru","radmin","Radmin v3.X Radmin Authentication",56618,"RU","MURMANSKAYA OBLAST","MURMANSK",0, -"2020-07-06 13:55:27","111.197.143.69","tcp",4899,,"radmin","Radmin (Details Unknown)",4808,"CN","BEIJING SHI","BEIJING",517311, -"2020-07-06 13:55:27","121.147.215.220","tcp",4899,,"radmin","Radmin v3.X Radmin Authentication",4766,"KR","GWANGJU-GWANGYEOKSI","DAEIN-DONG",517311, -"2020-07-06 13:55:27","121.147.215.178","tcp",4899,,"radmin","Radmin v3.X Radmin Authentication",4766,"KR","GWANGJU-GWANGYEOKSI","DAEIN-DONG",517311, -"2020-07-06 13:55:27","183.230.5.219","tcp",4899,,"radmin","Radmin v3.X Radmin Authentication",9808,"CN","CHONGQING SHI","CHONGQING",517312, -"2020-07-06 13:55:27","85.93.154.74","tcp",4899,,"radmin","Radmin v3.X Radmin Authentication",34300,"RU","MOSKVA","MOSCOW",0, -"2020-07-06 13:55:27","81.246.135.247","tcp",4899,"247.135-246-81.adsl-dyn.isp.belgacom.be","radmin","Radmin v3.X Radmin Authentication",5432,"BE","ANTWERPEN","BRASSCHAAT",517311, -"2020-07-06 13:55:27","46.27.146.22","tcp",4899,"static-22-146-27-46.ipcom.comunitel.net","radmin","Radmin v3.X Radmin Authentication",12430,"ES","LAS PALMAS","LAS PALMAS DE GRAN CANARIA",517312, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv.license deleted file mode 100644 index 833024a759..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2020 sinus-x -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv deleted file mode 100644 index 4bac90f199..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","port","hostname","tag","handshake","asn","geo","region","city","rdp_protocol","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","naics","sic","sector","tlsv13_support","tlsv13_cipher","cve20190708_vulnerable","bluekeep_vulnerable","jarm" -"2019-09-04 15:45:51","198.123.245.178",5678,,"rdp",,5678,"AA","LOCATION","LOCATION","RDP",5678,"KABESRV.KABE.local","KABESRV.KABE.local","2019-04-29 02:22:06","2019-10-29 02:22:06","EC:BB:4D:DB:9F:0C:D3:FF:5B:49:EA:B1:56:62:B6:A7:5D:60:54:42","1EF2B37AF850C9BF4E88F18177001D6B",2,"sha256WithRSAEncryption","rsaEncryption","B7:C9:F4:07:D5:C0:75:1D:EA:0C:40:E7:26:39:C2:30:C6:13:83:7E:18:46:D8:E9:4C:45:3F:88:1B:0B:70:76","08:AC:75:FA:EB:A3:2B:44:15:DE:6D:A7:0B:C0:AE:17:94:F3:55:D9:EC:70:AC:5B:B7:94:79:F0:D7:84:83:89:CB:A9:11:E0:08:D7:54:4D:33:85:89:D2:A8:DD:9D:15:F4:CC:95:DE:6A:E3:DF:6B:FA:8B:27:E3:DA:16:AF:0A","BC:6E:C3:E2:98:22:EC:BA:5B:30:E2:53:FD:4A:9D:FF",517311,0,,,,"N","N" -"2019-09-04 15:45:51","198.123.245.233",5678,,"rdp",,5678,"AA","LOCATION","LOCATION","RDP",5678,"RAMBLA01.rambla.local","RAMBLA01.rambla.local","2019-04-16 06:15:20","2019-10-16 06:15:20","7A:67:1F:F8:87:C6:B0:AC:A9:84:15:B7:40:EC:CB:19:AA:E3:19:52","3FF3EBC5CF154BA54D128A8548C8AAF5",2,"sha1WithRSAEncryption","rsaEncryption","8F:CD:7D:C4:80:2D:8D:9B:06:A0:40:18:9F:ED:73:7A:BA:83:55:BE:1B:56:83:A2:97:DF:BB:B4:06:57:CB:F1","E8:9B:9A:93:69:B4:58:01:D8:46:C2:DC:01:20:1E:DD:93:E1:EB:E3:9D:6B:65:A0:C5:00:6C:A4:44:08:FE:A4:A6:19:FF:55:79:F2:AA:61:68:C8:1C:B0:CE:78:EB:84:DD:29:9D:64:2F:4E:25:31:3A:6C:B8:02:C9:AF:F5:1F","38:73:6A:B3:AA:41:69:C9:BA:E7:3D:D7:40:16:F8:AA",517311,0,"Information Technology",,,"N","N" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv deleted file mode 100644 index 73d0d55efd..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sessionid","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,3389,node01.example.com,rdpeudp,64512,ZZ,Region,City,0,0,05b28c0c,1232,77.00 -"2010-02-10 00:00:01",192.168.0.2,udp,3389,node02.example.com,rdpeudp,64512,ZZ,Region,City,0,0,053d355f,1232,77.00 -"2010-02-10 00:00:02",192.168.0.3,udp,3389,node03.example.com,rdpeudp,64512,ZZ,Region,City,0,0,0567a8cb,1232,77.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv deleted file mode 100644 index dc9760cf2d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv +++ /dev/null @@ -1,94 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","git_sha1","git_dirty_flag","build_id","mode","os","architecture","multiplexing_api","gcc_version","process_id","run_id","uptime","connected_clients","sector" -"2016-07-24 00:42:33","198.51.100.152","tcp",6379,,"redis","2.8.19",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"26069fb482f6334b","standalone","Linux 3.2.0-4-amd64 x86_64",,"epoll","4.7.2",2127,"d440b0b2fb3d1db655ad607e11e6f38011a0f599",27946314,50, -"2016-07-24 00:42:43","198.51.100.67","tcp",6379,"198-51-100-67.example.net","redis","3.0.6",12586,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",310556,25376, -"2016-07-24 00:42:43","198.51.100.125","tcp",6379,"198-51-100-125.example.net","redis","2.8.17",8972,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"5937320cdd80c1e4","standalone","Linux 2.6.32-5-amd64 x86_64",,"epoll","4.9.2",11573,"0d58143df099738a7ce9330ee5ec2367d11b1187",25888041,4, -"2016-07-24 00:42:43","198.51.100.203","tcp",6379,"198-51-100-203.example.net","redis","2.8.4",31103,"DE","THURINGEN","ERFURT",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.13.0-83-generic x86_64",,"epoll","4.8.2",3847,"4f7765dee91d8c4b1b24604cc5f0c29fca1a4f32",3068554,38, -"2016-07-24 00:42:43","198.51.100.240","tcp",6379,"198-51-100-30.example.net","redis","3.0.7",20473,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"869e89100d5ea8c2","standalone","Linux 3.13.0-87-generic x86_64",,"epoll","4.8.4",1011,"864c8d7df1e72c662a4edd77b6df6cd30161af6e",2476542,2,"Information Technology" -"2016-07-24 00:42:49","198.51.100.69","tcp",6379,"198-51-100-69.example.net","redis","3.0.6",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"315c8c74805fca88","standalone","Linux 3.2.0-98-generic x86_64",,"epoll","4.6.3",28961,"bc705102c854ea1818213e4740a3c6fd9b9f1716",4633191,1, -"2016-07-24 00:42:53","198.51.100.50","tcp",6379,"198-51-100-50.example.net","redis","3.0.7",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"6afb1e1f0d80abd0","standalone","Linux 2.6.32-5-amd64 x86_64",,"epoll","4.4.5",1717,"f729595b3642b48f3ac9e098bcccab1d6ef82e3e",6345372,3, -"2016-07-24 00:43:49","198.51.100.113","tcp",6379,,"redis","3.0.6",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",310623,24628, -"2016-07-24 00:43:49","198.51.100.228","tcp",6379,"198-51-100-131.example.net","redis","2.8.210",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"83ad777ec89a946b","standalone","Windows",,"winsock_IOCP",,1948,"f5d6ad26e423039636afaf3918ee7e6a7e0b5b68",2214134,4,"Information Technology" -"2016-07-24 00:43:59","198.51.100.155","tcp",6379,,"redis","3.0.7",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"f09a0843cc9876c3","standalone","Linux 3.13.0-79-generic x86_64",,"epoll","4.9.2",1,"5f4f5b7158f928cc96e3ae6af6092a163ace15eb",2897902,24, -"2016-07-24 00:43:59","198.51.100.171","tcp",6379,,"redis","3.0.6",12586,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",310633,25031, -"2016-07-24 00:44:09","198.51.100.230","tcp",6379,"198-51-100-230.example.net","redis","2.8.11",8560,"DE","BADEN-WURTTEMBERG","KARLSRUHE",0,0,00000000,0,"f26bfdf4b8265fc","standalone","Linux 3.8.0-29-generic x86_64",,"epoll","4.6.3",14551,"0d001175cf26cee88486d814b4f0c972a5aa89b9",21038337,9, -"2016-07-24 00:44:09","198.51.100.182","tcp",6379,"198-51-100-182.example.net","redis","3.0.7",197540,"DE","BADEN-WURTTEMBERG","KARLSRUHE",0,0,00000000,0,"fd24f54fec00684b","standalone","Linux 3.13.0-85-generic x86_64",,"epoll","4.8.4",949,"b11fdf2b95251b8e6c3e9e782409ef82fc8b89aa",8643389,11, -"2016-07-24 00:44:10","198.51.100.23","tcp",6379,"198-51-100-116.example.net","redis","2.8.4",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 4.2.0-27-generic x86_64",,"epoll","4.8.2",335,"90079d58e970a1ae94aa91bc0ea0236a0e55269c",4930922,2,"Information Technology" -"2016-07-24 00:44:19","198.51.100.51","tcp",6379,"198-51-100-51.example.net","redis","3.0.6",12586,"DE","BERLIN","BERLIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",310652,26257, -"2016-07-24 00:44:22","198.51.100.88","tcp",6379,,"redis","3.0.6",13301,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",310656,26371, -"2016-07-24 00:44:22","198.51.100.107","tcp",6379,"octopus-dev","redis","2.8.14",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"78be6d5e32e34139","standalone","Linux 2.6.32-042stab108.2 x86_64",,"epoll","4.8.2",21205,"b98a41b6ea690c207527587f60bff1f1d24236b4",9364864,4, -"2016-07-24 00:44:22","198.51.100.75","tcp",6379,,"redis","3.0.0",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"2b5201a6bfd5f75e","standalone","Linux 3.11.0-19-generic x86_64",,"epoll","4.8.2",832,"2bdcda8b3b59cef244785b58935d68daf48645be",6745479,5, -"2016-07-24 00:44:25","198.51.100.12","tcp",6379,,"redis","3.0.6",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.13.0-79-generic x86_64",,"epoll","4.8.4",899,"94550e510bf770aa315cc3983ce9958853c77cfe",7816856,9, -"2016-07-24 00:44:27","198.51.100.13","tcp",6379,"198-51-100-13.example.net","redis","3.0.7",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"6f8b503a2787e3a6","standalone","Linux 4.4.5-15.26.amzn1.x86_64 x86_64",,"epoll","4.9.2",1,"e050f40e755a739ffecdb2468e1333f371e2abca",7124048,6,"Communications" -"2016-07-24 00:44:29","198.51.100.12","tcp",6379,"198-51-100-12.example.net","redis","2.8.3",8972,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"992c97be25a6b6d2","standalone","Linux 2.6.32-042stab111.12 x86_64",,"epoll","4.4.5",12340,"d7cda18212cf4bcdfd7c42fff33e506a4e9a2614",16874891,8, -"2016-07-24 00:44:38","198.51.100.66","tcp",6379,"198-51-100-66.example.net","redis","3.2.1",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"ed627d97d5dc311e","standalone","Linux 4.2.0-19-generic x86_64",,"epoll","4.9.2",1,"4a6beb721ddbaa411f53e5268e6112127903cae3",2029470,3,"Chemical" -"2016-07-24 00:44:38","198.51.100.170","tcp",6379,,"redis","3.0.6",8881,"DE","SACHSEN","RADEBEUL",0,0,00000000,0,"1b14d17ce6fea422","standalone","Linux 4.2.6-1-pve x86_64",,"epoll","4.9.2",728,"c423ba856285690a2fae350b03514cec80db9d5e",1679635,1, -"2016-07-24 00:44:38","198.51.100.67","tcp",6379,"198-51-100-67.example.net","redis","2.8.23",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"8e819a849ea2d7f8","standalone","Linux 4.2.0-23-generic x86_64",,"epoll","4.9.2",1,"7ee1dc403540ff4d1fc0a80d9f0b2910857b6c1b",9451832,68,"Information Technology" -"2016-07-24 00:44:44","198.51.100.238","tcp",6379,,"redis","2.8.4",44066,"DE","BAYERN","MUNICH",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 2.6.32-19-pve x86_64",,"epoll","4.8.2",2207,"6a079396cc44c1aca745edab13f4014c394da3ab",10338949,3, -"2016-07-24 00:44:44","198.51.100.84","tcp",6379,"198-51-100-84.example.net","redis","3.0.2",51862,"DE","BADEN-WURTTEMBERG","KARLSRUHE",0,0,00000000,0,"4795df119e2d77fe","standalone","Linux 3.13.0-91-generic x86_64",,"epoll","4.7.2",1,"c120481a551c232b8e1a9cff20d9e0968a402dd9",1040551,7, -"2016-07-24 00:44:44","198.51.100.23","tcp",6379,"198-51-100-23.example.net","redis","3.0.6",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"98c227055d7fa7b6","standalone","Linux 3.10.0-327.10.1.el7.x86_64 x86_64",,"epoll","4.8.5",35198,"424b15e04ce09f26299ff19b252a920916d4e4be",8875355,2, -"2016-07-24 00:44:47","198.51.100.160","tcp",6379,"198-51-100-160.example.net","redis","2.8.210",6724,"DE","BERLIN","BERLIN",0,0,00000000,0,"83ad777ec89a946b","standalone","Windows",,"winsock_IOCP",,2284,"9bde76afda6f81acfb241ea5ee3a9e878ad53881",742778,2, -"2016-07-24 00:44:47","198.51.100.111","tcp",6379,"198-51-100-98.example.net","redis","3.2.1",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"e19bb8c3d1c28291","standalone","Linux 3.10.0-327.22.2.el7.x86_64 x86_64",,"epoll","5.3.0",1,"c951371f430c1d94299bfc93759f6940d8bfce78",208557,2, -"2016-07-24 00:44:48","198.51.100.227","tcp",6379,"198-51-100-227.example.net","redis","3.0.6",12586,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",310681,26496, -"2016-07-24 00:44:54","198.51.100.18","tcp",6379,"198-51-100-18.example.net","redis","2.8.9",8972,"DE","NORDRHEIN-WESTFALEN","COLOGNE",0,0,00000000,0,"52c7b9284559eb20","standalone","Linux 2.6.32-5-amd64 x86_64",,"epoll","4.4.5",31887,"e5b1da35862482c4df8d4fce635ec89a36476a4d",14393072,6, -"2016-07-24 00:44:54","198.51.100.248","tcp",6379,"198-51-100-248.example.net","redis","3.0.6",12586,"DE","BERLIN","BERLIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",310687,26112, -"2016-07-24 00:44:57","198.51.100.228","tcp",6379,"198-51-100-228.example.net","redis","3.0.7",8972,"DE","NORDRHEIN-WESTFALEN","COLOGNE",0,0,00000000,0,"5e03212a543f54f8","standalone","Linux 3.13.0-042stab116.1 x86_64",,"epoll","4.8.4",719,"537e3e824a45414c3199ef20201b4362b752eeb5",1263367,2, -"2016-07-24 00:45:04","198.51.100.227","tcp",6379,"198-51-100-227.example.net","redis","2.8.12",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"ff040dde4a39b4ff","standalone","Windows",,"winsock_IOCP","0.0.0",1872,"c78751c65793a9a72f6fb0318efa532eb4fc87de",277953,18,"Chemical" -"2016-07-24 00:45:07","198.51.100.132","tcp",6379,,"redis","3.0.5",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"30405cba8f6c2d55","standalone","Linux 3.13.0-57-generic x86_64",,"epoll","4.8.4",2500,"10b4084b930d5a77e5f09e89cf0b21702027bd60",10028956,695, -"2016-07-24 00:46:10","198.51.100.47","tcp",6379,"198-51-100-185.example.net","redis","3.0.7",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"6a943c0b5bf37fa1","standalone","Linux 4.4.0-24-generic x86_64",,"epoll","5.3.1",1023,"de9c9c0da3d971f689bd7366c1edc93a00fd1506",2791106,1, -"2016-07-24 01:23:27","198.51.100.246","tcp",6379,"198-51-100-190.example.net","redis","2.8.19",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"665519ce00ddac9b","standalone","Linux 2.6.32-43-pve x86_64",,"epoll","4.9.2",2310,"94595838457eddb30a60184a9db66212268e6f82",9481199,4, -"2016-07-24 01:23:29","198.51.100.187","tcp",6379,"198-51-100-63.example.net","redis","2.8.19",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"c0359e7aa3798aa2","standalone","Linux 3.10.0-229.7.2.el7.x86_64 x86_64",,"epoll","4.8.3",14050,"e67a19de4bd2dc485b98ca353eb6fdc65e8fed4a",14051444,10, -"2016-07-24 01:23:29","198.51.100.228","tcp",6379,"198-51-100-228.example.net","redis","2.8.4",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.13.0-32-generic x86_64",,"epoll","4.8.2",22837,"daf5dba760d3db12716c6dc1d0bfe6d5e7b33749",10916038,8, -"2016-07-24 01:23:43","198.51.100.180","tcp",6379,"198-51-100-180.example.net","redis","3.2.1",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"ed627d97d5dc311e","standalone","Linux 4.2.0-19-generic x86_64",,"epoll","4.9.2",1,"569881874d8d5e1508d584a3fd9dff0ac3515839",1677711,1,"Chemical" -"2016-07-24 01:23:56","198.51.100.5","tcp",6379,"198-51-100-207.example.net","redis","3.0.7",20473,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"869e89100d5ea8c2","standalone","Linux 3.13.0-87-generic x86_64",,"epoll","4.8.4",1011,"864c8d7df1e72c662a4edd77b6df6cd30161af6e",2479015,2,"Information Technology" -"2016-07-24 01:24:03","198.51.100.226","tcp",6379,"198-51-100-226.example.net","redis","3.0.5",8972,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"b33bc3e2f8ad13f6","standalone","Linux 2.6.32-573.12.1.el6.x86_64 x86_64",,"epoll","4.4.7",1801,"7f4bb7ed008cdbd665672e88d57fc55616b6dbf2",13189200,9, -"2016-07-24 01:24:14","198.51.100.253","tcp",6379,"198-51-100-136.example.net","redis","2.8.4",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.19.0-39-generic x86_64",,"epoll","4.8.2",28272,"13a889aa846c6302dc8f5453e35e051a6f359e9a",14046610,185, -"2016-07-24 01:24:28","198.51.100.206","tcp",6379,,"redis","3.0.6",13301,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",313061,26695, -"2016-07-24 01:24:35","198.51.100.73","tcp",6379,,"redis","3.0.2",28753,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"6be7fc9e6b88f79","standalone","Linux 2.6.32-71.29.1.el6.x86_64 x86_64",,"epoll","4.4.7",811,"bbd4dce247ab51d029a64243810aeb900e00d1d6",10082205,15, -"2016-07-24 01:24:35","198.51.100.83","tcp",6379,"198-51-100-174.example.net","redis","3.2.1",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"7e7b61a55b95e8e7","standalone","Linux 4.2.0-41-generic x86_64",,"epoll","4.8.4",1076,"48f5f780ca53553fc4c0bbdbb32a5cb06a0551cd",814255,88,"Information Technology" -"2016-07-24 01:25:30","198.51.100.182","tcp",6379,,"redis","3.0.7",31400,"DE","RHEINLAND-PFALZ","FREINSHEIM",0,0,00000000,0,"d9ceac045f7983a9","standalone","FreeBSD 10.1-RELEASE-p26 amd64",,"kqueue","4.2.1",957,"48f37d15b3f5169f11aa5d7194fdfccc7f8df20b",6364747,1, -"2016-07-24 01:25:30","198.51.100.211","tcp",6379,"198-51-100-118.example.net","redis","2.8.17",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"e4968abcd4b78b2e","standalone","Linux 3.13.0-36-generic x86_64",,"epoll","4.8.2",1643,"665565b1b1fb6e773039707a0f680bbc417186be",20180649,4,"Information Technology" -"2016-07-24 01:25:35","198.51.100.249","tcp",6379,,"redis","3.0.2",28753,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"6be7fc9e6b88f79","standalone","Linux 2.6.32-71.29.1.el6.x86_64 x86_64",,"epoll","4.4.7",811,"bbd4dce247ab51d029a64243810aeb900e00d1d6",10082265,15, -"2016-07-24 01:25:40","198.51.100.55","tcp",6379,,"redis","3.2.1",3320,"DE","NORDRHEIN-WESTFALEN","SOLINGEN",518210,737415,00000000,0,"e19bb8c3d1c28291","standalone","Linux 4.4.0-24-generic x86_64",,"epoll","5.3.0",1,"49687ba2a5be7f7b6cdf0c837e06307442f6a369",494739,1, -"2016-07-24 01:25:42","198.51.100.62","tcp",6379,"198-51-100-62.example.net","redis","3.0.7",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"2b87841ee28adfc3","standalone","Linux 3.13.0-042stab113.11 x86_64",,"epoll","4.8.4",525,"4045d68fd2e59a1135bb303206d7cd0439ba7ffd",6971251,4, -"2016-07-24 01:25:55","198.51.100.127","tcp",6379,"198-51-100-25.example.net","redis","2.8.4",20473,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.13.0-87-generic x86_64",,"epoll","4.8.2",11492,"3de3e977405eef9392a77db4a50d99a5caa2f2d9",2194103,3,"Information Technology" -"2016-07-24 01:26:08","198.51.100.92","tcp",6379,"198-51-100-92.example.net","redis","2.8.10",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"5fce0c4aab65e01","standalone","Linux 2.6.32-042stab113.11 x86_64",,"epoll","4.6.3",490,"15abe68a10b011972f50d0abb3bb18f1735994a5",7505621,4, -"2016-07-24 01:26:17","198.51.100.218","tcp",6379,,"redis","3.0.7",34011,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"dc142e699f115c40","standalone","Linux 3.2.60-grsec-x86_64 x86_64",,"epoll","4.7.3",8006,"53a093bd4d0a7b72b2d084ec3767d23b18b8b947",4024979,7, -"2016-07-24 01:26:29","198.51.100.168","tcp",6379,"198-51-100-168.example.net","redis","3.0.6",51167,"DE","BAYERN","MUNICH",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.13.0-37-generic x86_64",,"epoll","4.8.4",1279,"8218bd77a0dcb0e00bd77dbb9478115757c70ba5",2405965,1, -"2016-07-24 01:26:29","198.51.100.155","tcp",6379,"198-51-100-155.example.net","redis","3.0.7",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"d9155128f7b25ea0","standalone","Linux 3.19.0-25-generic x86_64",,"epoll","4.8.4",27030,"0ede623cb268643672abc04d0267f684a5ee7a0d",6880190,5,"Information Technology" -"2016-07-24 01:26:34","198.51.100.185","tcp",6379,,"redis","2.8.4",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.13.0-43-generic x86_64",,"epoll","4.8.2",1196,"ae80fcbb54017f521212caf257418885cd6836a0",5412584,5, -"2016-07-24 01:26:34","198.51.100.1","tcp",6379,"198-51-100-1.example.net","redis","3.2.0",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"5382f69a4e75566b","standalone","Linux 4.2.0-19-generic x86_64",,"epoll","4.9.2",1,"ff8990f109ff5b2d4e0eee47e5ebc66acc43f9e3",4615889,4,"Chemical" -"2016-07-24 01:26:39","198.51.100.51","tcp",6379,"198-51-100-164.example.net","redis","3.0.0",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"9526f4809583faaa","standalone","Linux 2.6.32-042stab113.21 x86_64",,"epoll","4.4.5",14528,"d7271feff55175f434ace92d199f332ad35776a9",7440370,16, -"2016-07-24 01:26:44","198.51.100.138","tcp",6379,,"redis","3.0.6",12586,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",313197,26452, -"2016-07-24 01:26:47","198.51.100.16","tcp",6379,,"redis","2.8.17",25074,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"5937320cdd80c1e4","standalone","Linux 2.6.32-43-pve x86_64",,"epoll","4.9.2",266,"e1d403f2daff849a64b178f74c672db6712f217a",351253,1, -"2016-07-24 01:26:54","198.51.100.171","tcp",6379,"198-51-100-171.example.net","redis","3.0.6",12586,"DE","BERLIN","BERLIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",313207,26601, -"2016-07-24 01:27:14","198.51.100.89","tcp",6379,"198-51-100-89.example.net","redis","3.0.6",12586,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",313227,26358, -"2016-07-24 01:27:24","198.51.100.65","tcp",6379,,"redis","3.0.7",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"869e89100d5ea8c2","standalone","Linux 3.13.0-85-generic x86_64",,"epoll","4.8.4",21575,"3ec40168300e14f5776d82a48ba873a3999caec1",1897530,1, -"2016-07-24 01:27:24","198.51.100.248","tcp",6379,"198-51-100-248.example.net","redis","3.0.6",12586,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",313237,25902, -"2016-07-24 01:27:33","198.51.100.17","tcp",6379,,"redis","2.8.17",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"43dd9e14444e6aea","standalone","Linux 3.16.0-4-amd64 x86_64",,"epoll","4.9.2",556,"3e8fc2878511cc72f79b765fca86cefe21346912",2607965,72, -"2016-07-24 01:27:33","198.51.100.134","tcp",6379,"198-51-100-134.example.net","redis","3.0.7",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"6f8b503a2787e3a6","standalone","Linux 3.16.0-4-amd64 x86_64",,"epoll","4.9.2",1,"b85b2419cf35dd81ff5b9ba6e8bf802cf1d439f6",128621,33, -"2016-07-24 01:27:42","198.51.100.186","tcp",6379,"198-51-100-186.example.net","redis","2.8.13",34011,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"d588bf57ea0dfa69","standalone","Linux 4.4.8-jb1 i686",,"epoll","4.6.3",2460,"97b8d49e62d340d94a38c96c5104abfcacbfa4cb",181557,1, -"2016-07-24 01:27:42","198.51.100.21","tcp",6379,"198-51-100-21.example.net","redis","2.8.19",34011,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"920d7eda78149e99","standalone","Linux 4.4.8-x86_64-jb1 x86_64",,"epoll","4.7.2",3722,"74dfd8a7d87cbb9ecc590ceafd438c85d5073903",183984,1, -"2016-07-24 01:27:43","198.51.100.128","tcp",6379,"198-51-100-203.example.net","redis","3.0.5",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"f3bd5bc2b8b4c486","standalone","Linux 2.6.32-573.8.1.el6.x86_64 x86_64",,"epoll","4.4.7",1968,"0d92b1323fea791ba4b0a43435a156b6ec0aac1c",2967611,2,"Information Technology" -"2016-07-24 01:27:44","198.51.100.216","tcp",6379,"198-51-100-229.example.net","redis","2.8.4",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.16.0-30-generic x86_64",,"epoll","4.8.2",1470,"e76cd0cf25eec5d254c880965189ae011a119220",302420,1, -"2016-07-24 01:27:53","198.51.100.242","tcp",6379,"198-51-100-242.example.net","redis","3.0.2",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"6a04b5ede30cd4cd","standalone","Linux 3.13.0-32-generic x86_64",,"epoll","4.8.4",29725,"1b7e8dc53dec8fb29a8a2d76f516fd3dcb8df652",5815739,7, -"2016-07-24 01:27:53","198.51.100.54","tcp",6379,"198-51-100-54.example.net","redis","2.8.4",8560,"DE","BADEN-WURTTEMBERG","KARLSRUHE",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.13.0-91-generic x86_64",,"epoll","4.8.2",2903,"0e02514dec6031018eb148b13a4a9639cab3e8aa",905886,1, -"2016-07-24 01:27:54","198.51.100.225","tcp",6379,"198-51-100-225.example.net","redis","3.0.6",12586,"DE","BERLIN","BERLIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",313267,25281, -"2016-07-24 01:27:57","198.51.100.38","tcp",6379,"198-51-100-38.example.net","redis","3.0.5",6724,"DE","BERLIN","BERLIN",0,0,00000000,0,"3b863f97501297e9","standalone","Linux 3.13.0-042stab111.12 x86_64",,"epoll","4.8.4",2088,"31a8cececad2e4a33310a741143d85cdef3479b4",11906868,10, -"2016-07-24 01:27:58","198.51.100.22","tcp",6379,"198-51-100-22.example.net","redis","2.8.9",51167,"DE","BAYERN","MUNICH",0,0,00000000,0,"2ac6afaedfd3ea15","standalone","Linux 3.13.0-86-generic x86_64",,"epoll","4.8.4",9082,"8e5d9d74c86a9f148a7012733eb52a21938c3c04",5833880,5, -"2016-07-24 01:28:05","198.51.100.106","tcp",6379,"198-51-100-106.example.net","redis","2.8.19",36351,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"9968db13395be4aa","standalone","Windows",,"winsock_IOCP","0.0.0",4372,"89716352a10cd53b5c10e6d5e6cd1d46f5f53a30",485031,4,"Information Technology" -"2016-07-24 01:28:06","198.51.100.130","tcp",6379,"198-51-100-130.example.net","redis","2.8.3",51167,"DE","BAYERN","MUNICH",0,0,00000000,0,"542faa6f897d2236","standalone","Linux 2.6.32-573.3.1.el6.x86_64 x86_64",,"epoll","4.4.7",25531,"9d7606a883f764e744d766b7bf0036ba61f7fb6e",496133,5, -"2016-07-24 01:28:08","198.51.100.37","tcp",6379,"198-51-100-37.example.net","redis","2.8.23",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"50630e46be5feb4f","standalone","Linux 3.13.0-74-generic x86_64",,"epoll","4.9.2",1,"62d16be721c3c62d6c4d080a9bdbe9502c57ca86",3481683,9,"Communications" -"2016-07-24 01:28:32","198.51.100.148","tcp",6379,"198-51-100-148.example.net","redis","3.0.5",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"83dc15dcf8ee3eb8","standalone","Linux 4.1.7-15.23.amzn1.x86_64 x86_64",,"epoll","4.8.3",2304,"883accf76dc364c60902b4eab7861dd1a7eac71d",10981957,10,"Communications" -"2016-07-24 01:28:49","198.51.100.247","tcp",6379,"198-51-100-247.example.net","redis","3.0.7",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"3e971e94fbe2eaa6","standalone","Linux 3.2.0-4-amd64 x86_64",,"epoll","4.7.2",2535,"d223aab0621cdd2e4ab752978ad3009ad3814d8b",7715188,57, -"2016-07-24 02:08:46","198.51.100.220","tcp",6379,"198-51-100-220.example.net","redis","3.0.6",51167,"DE","BAYERN","MUNICH",0,0,00000000,0,"1f8e4c92f1ca309","standalone","Linux 3.13.0-74-generic x86_64",,"epoll","4.8.4",3355,"dd517756bb6ee81e1929fa605972318b2baebb93",5211978,10, -"2016-07-24 02:08:46","198.51.100.239","tcp",6379,"198-51-100-239.example.net","redis","2.8.23",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"83a5616190c5a1aa","standalone","Linux 3.16.0-4-amd64 x86_64",,"epoll","4.9.2",711,"4117960b13fa313b823c79b0e9f188d8ec6aa3ac",10156283,6, -"2016-07-24 02:08:50","198.51.100.233","tcp",6379,,"redis","2.8.11",8560,"DE","BADEN-WURTTEMBERG","KARLSRUHE",0,0,00000000,0,"f26bfdf4b8265fc","standalone","Linux 3.8.0-29-generic x86_64",,"epoll","4.6.3",14551,"0d001175cf26cee88486d814b4f0c972a5aa89b9",21043417,9, -"2016-07-24 02:08:51","198.51.100.208","tcp",6379,"198-51-100-181.example.net","redis","3.0.6",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 4.2.0-38-generic x86_64",,"epoll","4.8.4",809,"14c5ec7f9669e42ea45a40ff26a6501d593695c0",2405839,19, -"2016-07-24 02:08:51","198.51.100.60","tcp",6379,"198-51-100-60.example.net","redis","3.0.7",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"4ed99bd9c45dfc14","standalone","Linux 3.13.0-57-generic x86_64",,"epoll","4.8.4",1144,"9e28c29ff40017e2fbe32fb97755caf801f95793",843538,2, -"2016-07-24 02:08:51","198.51.100.107","tcp",6379,"198-51-100-39.example.net","redis","3.2.0",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"82b2619163aabc80","standalone","Linux 4.2.0-25-generic x86_64",,"epoll","4.9.2",1,"98f6640bbde04b1214730937212e1fd4e58d03a8",2195657,12, -"2016-07-24 02:08:54","198.51.100.31","tcp",6379,,"redis","2.8.4",6724,"DE","BERLIN","BERLIN",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.13.0-042stab111.12 x86_64",,"epoll","4.8.2",1112,"9c4e55b5ebd06045c5d89d43fa202e219ec8b42c",8839783,7, -"2016-07-24 02:08:56","198.51.100.221","tcp",6379,,"redis","3.0.7",44066,"DE","BAYERN","MUNICH",0,0,00000000,0,"49f951dce0725d71","standalone","FreeBSD 10.0-RELEASE-p7 amd64",,"kqueue","4.2.1",932,"28c6af3c4dedcd9b71cf51a7ebc4e84899196aee",8000949,1, -"2016-07-24 02:09:01","198.51.100.155","tcp",6379,"198-51-100-155.example.net","redis","2.8.22",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"fcdf45e47686c89b","standalone","Linux 3.13.0-57-generic x86_64",,"epoll","4.8.4",7,"946ec6b96fe9925d2b677ce02b6c56097c5e69a8",8449694,6, -"2016-07-24 02:09:02","198.51.100.219","tcp",6379,"198-51-100-219.example.net","redis","2.8.4",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.13.0-74-generic x86_64",,"epoll","4.8.2",1047,"9b83d6a6e7a6ffe50e75dac88cdc5e06f6203c9c",966148,1,"Chemical" -"2016-07-24 02:09:02","198.51.100.193","tcp",6379,"198-51-100-193.example.net","redis","3.0.7",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"fd640d8ef55a22dd","standalone","Linux 4.2.0-42-generic x86_64",,"epoll","4.8.4",1397,"ed5ec17d78d089af53afd4abc339f7decf4641d4",651175,2,"Information Technology" -"2016-07-24 02:09:20","198.51.100.120","tcp",6379,"198-51-100-120.example.net","redis","3.2.1",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"ed627d97d5dc311e","standalone","Linux 3.16.0-4-amd64 x86_64",,"epoll","4.9.2",1,"f524508ad29334eee2fcf7bdda5c80b9f99d3dfe",987580,167, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv deleted file mode 100644 index a61e4573ec..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","module","motd","has_password" -"2010-02-10 00:00:00",192.168.0.1,tcp,873,node01.example.com,rsync,64512,ZZ,Region,City,0,0,"system|Backup system;system_full|Backup full system;mysql|Backup virtual mysql;netadmin|Backup virtual netadmin;",,N -"2010-02-10 00:00:01",192.168.0.2,tcp,873,node02.example.com,rsync,64512,ZZ,Region,City,0,0,"system|Backup system;system_full|Backup full system;mysql|Backup virtual mysql;netadmin|Backup virtual netadmin;",,N -"2010-02-10 00:00:02",192.168.0.3,tcp,873,node03.example.com,rsync,64512,ZZ,Region,City,0,0,"system|Backup system;system_full|Backup full system;mysql|Backup virtual mysql;netadmin|Backup virtual netadmin;",,N diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv deleted file mode 100644 index ee0a625e55..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","sip","sip_code","sip_reason","user_agent","sip_via","sip_to","sip_from","content_length","content_type","server","contact","cseq","call_id","allow","amplification","response_size" -"2010-02-10 00:00:00",192.168.0.1,udp,5060,node01.example.com,sip,64512,ZZ,Region,City,SIP/2.0,489,"Event Package Not Supported",,,,,0,,,,,,"INVITE,ACK,BYE,CANCEL,REGISTER",15.57,109 -"2010-02-10 00:00:01",192.168.0.2,udp,5060,node02.example.com,sip,64512,ZZ,Region,City,SIP/2.0,400,"Bad Request",,,,,364,text/plain,,,,,,62.57,438 -"2010-02-10 00:00:02",192.168.0.3,udp,5060,node03.example.com,sip,64512,ZZ,Region,City,SIP/2.0,400,"Bad Request",,,,,0,,,,,,,6.57,46 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv.license deleted file mode 100644 index 9f58c89ef0..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2023 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv deleted file mode 100644 index 256dd78f60..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","version","function","function_text","flags","next_extension_offset","xid","language_tag_length","language_tag","error_code","error_code_text","response_size","raw_response" -"2010-02-10 00:00:00",192.168.0.1,tcp,427,node01.example.com,slp,64512,ZZ,Region,City,0,0,,2,2,"Service reply",0x0000,0,5,2,en,5,"Unsupported SLP SPI",40,MDIwMjAwMDAxNDAwMDAwMDAwMDAwMDA1MDAwMjY1NmUwMDA1MDAwMA== -"2010-02-10 00:00:01",192.168.0.2,tcp,427,node02.example.com,slp,64512,ZZ,Region,City,0,0,,2,2,"Service reply",0x0000,0,5,2,en,5,"Unsupported SLP SPI",40,MDIwMjAwMDAxNDAwMDAwMDAwMDAwMDA1MDAwMjY1NmUwMDA1MDAwMA== -"2010-02-10 00:00:02",192.168.0.3,tcp,427,node03.example.com,slp,64512,ZZ,Region,City,0,0,,2,2,"Service reply",0x0000,0,5,2,en,5,"Unsupported SLP SPI",40,MDIwMjAwMDAxNDAwMDAwMDAwMDAwMDA1MDAwMjY1NmUwMDA1MDAwMA== diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv.license deleted file mode 100644 index 9f58c89ef0..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2023 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv deleted file mode 100644 index fc7fe2fff6..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","port","hostname","tag","asn","geo","region","city","naics","sic","smb_implant","arch","key","smbv1_support","smb_major_number","smb_minor_number","smb_revision","smb_version_string" -"2010-02-10 00:00:00",192.168.0.1,445,node01.example.com,smb,64512,ZZ,Region,City,0,0,N,,,N,2,1,0,"SMB 2.1" -"2010-02-10 00:00:01",192.168.0.2,445,node02.example.com,smb,64512,ZZ,Region,City,0,0,N,,,N,2,1,0,"SMB 2.1" -"2010-02-10 00:00:02",192.168.0.3,445,node03.example.com,smb,64512,ZZ,Region,City,0,0,N,,,N,2,1,0,"SMB 2.1" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv deleted file mode 100644 index 19eb560538..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","banner" -"2021-07-08 11:58:42","1.2.3.4","tcp",25,"smtp-server.invalid","smtp;21nails",12345,"EE","HARJUMAA","TALLINN",,,"220 smtp-server.invalid ESMTP Exim 4.80 Wed, 11 Jun 2021 10:00:00 +0300|" -"2021-07-08 11:58:44","5.6.7.8","tcp",25,"smtp-out.invalid","smtp;21nails",23456,"EE","HARJUMAA","TALLINN",,,"220 smtp-out.invalid, ESMTP EXIM 4.86_2|" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv.license deleted file mode 100644 index c1900637ff..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Mikk Margus Möll -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv deleted file mode 100644 index f489261c42..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","sysdesc","sysname","asn","geo","region","city","version","naics","sic","sector","device_vendor","device_type","device_model","device_version","device_sector","tag","community","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,161,node01.example.com,"Linux localhost 3.18.20 #1 SMP Mon Jul 9 14:11:21 CST 2018 armv7l",,64512,ZZ,Region,City,2,0,0,,,,,,,snmp,public,165,1.94 -"2010-02-10 00:00:01",192.168.0.2,udp,161,node02.example.com,"RouterOS CCR1009-8G-1S-1S+",,64512,ZZ,Region,City,2,0,0,,MikroTik,router,,,consumer,"snmp,iot",public,115,1.35 -"2010-02-10 00:00:02",192.168.0.3,udp,161,node03.example.com,,,64512,ZZ,Region,City,2,0,0,,,,,,,snmp,public,85,1.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv deleted file mode 100644 index c591a5c099..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector" -"2010-02-10 00:00:00",192.168.0.1,tcp,1080,node01.example.com,socks4,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service" -"2010-02-10 00:00:01",192.168.0.2,tcp,1080,node02.example.com,socks5,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service" -"2010-02-10 00:00:02",192.168.0.3,tcp,1080,node03.example.com,socks4,64512,ZZ,Region,City,0,0,"Retail Trade" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv deleted file mode 100644 index 460be32c50..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","header","asn","geo","region","city","systime","cache_control","location","server","search_target","unique_service_name","host","nts","nt","content_type","naics","sic","sector","server_port","instance","version","updated_at","resource_identifier","amplification","response_size" -"2010-02-10 00:00:00",192.168.0.1,udp,60194,node01.example.com,ssdp,"HTTP/1.1 200 OK",64512,ZZ,Region,City,"Sun, 21 Aug 2022 09:51:13 GMT",max-age=100,http://192.168.200.254:49152/description.xml,"Linux/2.6.26, UPnP/1.0, Portable SDK for UPnP devices/1.3.1",upnp:rootdevice,uuid:28802880-2880-1880-a880-001bc502f600::upnp:rootdevice,node01.example.com,,,,0,0,Government,,,,,,3.35,325 -"2010-02-10 00:00:01",192.168.0.2,udp,38732,node02.example.com,ssdp,"HTTP/1.1 200 OK",64512,ZZ,Region,City,,"max-age = 1800",http://95.160.216.14:52235/dmr/SamsungMRDesc.xml,"Linux/9.0 UPnP/1.0 PROTOTYPE/1.0",upnp:rootdevice,uuid:f144ca92-6816-94b5-b95f-b58180834044::upnp:rootdevice,node02.example.com,,,,0,0,,,,,,,2.71,263 -"2010-02-10 00:00:02",192.168.0.3,udp,57626,node03.example.com,ssdp,"HTTP/1.1 200 OK",64512,ZZ,Region,City,"Sun, 03 Jan 2016 21:37:50 GMT",max-age=1800,http://192.168.1.3:8008/ssdp/device-desc.xml,"Linux/3.10.79, UPnP/1.0, Portable SDK for UPnP devices/1.6.18",upnp:rootdevice,uuid:62fa0fc8-079d-d00f-2e22-59b49fb488f9::upnp:rootdevice,node03.example.com,,,,0,0,Government,,,,,,4.79,465 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv deleted file mode 100644 index 837adbad10..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","serverid_raw","serverid_version","serverid_software","serverid_comment","server_cookie","available_kex","available_ciphers","available_mac","available_compression","selected_kex","algorithm","selected_cipher","selected_mac","selected_compression","server_signature_value","server_signature_raw","server_host_key","server_host_key_sha256","rsa_prime","rsa_prime_length","rsa_generator","rsa_generator_length","rsa_public_key","rsa_public_key_length","rsa_exponent","rsa_modulus","rsa_length","dss_prime","dss_prime_length","dss_generator","dss_generator_length","dss_public_key","dss_public_key_length","dss_dsa_public_g","dss_dsa_public_p","dss_dsa_public_q","dss_dsa_public_y","ecdsa_curve25519","ecdsa_curve","ecdsa_public_key_length","ecdsa_public_key_b","ecdsa_public_key_gx","ecdsa_public_key_gy","ecdsa_public_key_n","ecdsa_public_key_p","ecdsa_public_key_x","ecdsa_public_key_y","ed25519_curve25519","ed25519_cert_public_key_nonce","ed25519_cert_public_key_bytes","ed25519_cert_public_key_raw","ed25519_cert_public_key_sha256","ed25519_cert_public_key_serial","ed25519_cert_public_key_type_id","ed25519_cert_public_key_type_name","ed25519_cert_public_key_keyid","ed25519_cert_public_key_principles","ed25519_cert_public_key_valid_after","ed25519_cert_public_key_valid_before","ed25519_cert_public_key_duration","ed25519_cert_public_key_sigkey_bytes","ed25519_cert_public_key_sigkey_raw","ed25519_cert_public_key_sigkey_sha256","ed25519_cert_public_key_sigkey_value","ed25519_cert_public_key_sig_raw","banner","userauth_methods","device_vendor","device_type","device_model","device_version","device_sector" -"2022-01-10 02:20:37","18.179.0.0","tcp",22,"ec2-18-179-0-0.ap-northeast-1.compute.amazonaws.com","ssh",16509,"JP","TOKYO","TOKYO",454110,,"SSH-2.0-OpenSSH_7.4","2.0","OpenSSH_7.4",,"bGjsifbPIDWT7tAu8BMjyg==","curve25519-sha256, curve25519-sha256@libssh.org, ecdh-sha2-nistp256, ecdh-sha2-nistp384, ecdh-sha2-nistp521, diffie-hellman-group-exchange-sha256, diffie-hellman-group16-sha512, diffie-hellman-group18-sha512, diffie-hellman-group-exchange-sha1, diffie-hellman-group14-sha256, diffie-hellman-group14-sha1, diffie-hellman-group1-sha1","chacha20-poly1305@openssh.com, aes128-ctr, aes192-ctr, aes256-ctr, aes128-gcm@openssh.com, aes256-gcm@openssh.com, aes128-cbc, aes192-cbc, aes256-cbc, blowfish-cbc, cast128-cbc, 3des-cbc","umac-64-etm@openssh.com, umac-128-etm@openssh.com, hmac-sha2-256-etm@openssh.com, hmac-sha2-512-etm@openssh.com, hmac-sha1-etm@openssh.com, umac-64@openssh.com, umac-128@openssh.com, hmac-sha2-256, hmac-sha2-512, hmac-sha1","none, zlib@openssh.com","curve25519-sha256@libssh.org","ecdsa-sha2-nistp256","aes128-ctr","hmac-sha2-256","none","AAAAIQCd+X/B/OEx+FrwJSlVecOvNMuS5w2vTRz0z4prM+5VBwAAACEArU60b9CHs/d5BgyaOd7vmFygTMK5SyL90bS8VIztX/4=","AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAABKAAAAIQCd+X/B/OEx+FrwJSlVecOvNMuS5w2vTRz0z4prM+5VBwAAACEArU60b9CHs/d5BgyaOd7vmFygTMK5SyL90bS8VIztX/4=","AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDSEHaLacthwB30rtA4xJgN3G9zXkCmm2WhV/TlNBrD20fuNQAZX7XciX2YkqIHtK2dWLBYwVCCqvl//zoM42kI=","a6e4e1c16ba25d51bcddc58a6e16797144575dd18d02d9dedf75093d2b15c557",,,,,,,,,,,,,,,,,,,,"1xx7ASut7BF4ED8b592bebZBMBKTCzOsmbH4cjwx/0U=","P-256",256,"WsY12Ko6k+ez671VdpiGvGUdBrDMU7D2O848PifSYEs=","axfR8uEsQkf4vOblY6RA8ncDfYEt6zOg9KE5RdiYwpY=","T+NC4v4af5uO5+tKfA+eFivOM1drMV7Oy7ZAaDe/UfU=","/////wAAAAD//////////7zm+q2nF56E87nKwvxjJVE=","/////wAAAAEAAAAAAAAAAAAAAAD///////////////8=","NIQdotpy2HAHfSu0DjEmA3cb3NeQKabZaFX9OU0GsPY=","0fuNQAZX7XciX2YkqIHtK2dWLBYwVCCqvl//zoM42kI=",,,,,,,,,,,,,,,,,,,,"publickey",,,,, -"2022-01-10 02:20:37","170.10.0.0","tcp",22,"170-10-0-0.example.com","ssh",11976,"US","TEXAS","MARSHALL",,,"SSH-2.0-ARRIS_0.50","2.0","ARRIS_0.50",,"Y4RQS9sdRgEFwNJKVP6bZg==","diffie-hellman-group1-sha1","aes128-cbc, 3des-cbc, aes256-cbc, twofish256-cbc, twofish-cbc, twofish128-cbc, blowfish-cbc","hmac-sha1-96, hmac-sha1, hmac-md5","none","diffie-hellman-group1-sha1","ssh-rsa","aes128-cbc","hmac-sha1","none","LQj+UTJEQqdb/p/c/19yVc63eo+rnedwXKjP6eNNxxijN2cFoOjVMeqT2QTBjyoN7yRWBU2EID+3y2jUYT8mCqmqfyUv1eEbiCfLVlUyQ0X/CY9I5DDb5l6yEjNkuH2xVNNV6R7GFRwyYKAsYzfy+i9o1OORlUh3tozkkPfA9z/NlA==","AAAAB3NzaC1yc2EAAACCLQj+UTJEQqdb/p/c/19yVc63eo+rnedwXKjP6eNNxxijN2cFoOjVMeqT2QTBjyoN7yRWBU2EID+3y2jUYT8mCqmqfyUv1eEbiCfLVlUyQ0X/CY9I5DDb5l6yEjNkuH2xVNNV6R7GFRwyYKAsYzfy+i9o1OORlUh3tozkkPfA9z/NlA==","AAAAB3NzaC1yc2EAAAADAQABAAAAgwCDq1kGqqwdQVryCNcoyDbBpnL/okvM2d9NmR0OjprcToCZ2TZ5WUZt2BGwPE1QLJYskjhv7GwlfQ4qhEqHDg35wMrkO7j9LTQC7KW3xisOLuUil4FmMxPkol6s39945zBGpjw0l/BmJnUDlutxavkdd84fppFMwXNp2vbjxV1SYVc9","d53fedbfe92e631264629882b2e85bfd213ca4b07b824cd31f8de1fcb8d0ddcb",,,,,,,65537,"g6tZBqqsHUFa8gjXKMg2waZy/6JLzNnfTZkdDo6a3E6Amdk2eVlGbdgRsDxNUCyWLJI4b+xsJX0OKoRKhw4N+cDK5Du4/S00Auylt8YrDi7lIpeBZjMT5KJerN/feOcwRqY8NJfwZiZ1A5brcWr5HXfOH6aRTMFzadr248VdUmFXPQ==",1040,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"publickey, password","Arris",,,, -"2022-01-10 02:20:37","72.17.0.0","tcp",22,"072-017-0-0.example.com","ssh",33363,"US","FLORIDA","ORLANDO",517311,,"SSH-1.99-Cisco-1.25","1.99","Cisco-1.25",,"Z2fOfWsrLlh76Y0bOqa1cw==","diffie-hellman-group-exchange-sha1, diffie-hellman-group14-sha1, diffie-hellman-group1-sha1","aes128-cbc, 3des-cbc, aes192-cbc, aes256-cbc","hmac-sha1, hmac-sha1-96, hmac-md5, hmac-md5-96","none","diffie-hellman-group14-sha1","ssh-rsa","aes128-cbc","hmac-sha1","none","lrzL2DY9fVvwYg6CgB75uf2s8CLo+rL8Mp9tU1Ja3sDfBzj9QJjVDykupiy8s3usHfxMrHS2v3DhTiZjz/b5K6tVTgUBTXL94JfM4lwB+3EbLggPzKnlm1jQgnnU9c+tb7RX3IhBqU9Yj1gqxhErv9NFotgajQOOLgY0Ua5C0Ee+AIaMlLaNZe3LTejMsNUZMN5tl+sEmtutMHkGQsmjJxiJ3feF+Pys0I2+ojiiAfzqlMYar/5xOPl4Dj+HO+h91xVQ1/8nQRBc082fM7+ZJtDbRLtt4G8srlB5gew26jqfVASc/ui5gx4+BR9DG9VH8w+rJWBGfhOAaWqLFE2M3YuEWkjEmQMR1SQK1WFQ/oNiWJO2K5L3rk2LcAmyR6nQMtClVxYZ7CQOwa3uFL+JNXp9AhiiAtVaqhrEK81NJrJNh/+egTBl5STphxIShXd4KI9wyvkGlCIvNIMO94iXPVaWUXXbsGnU03+dsUkBzGf0eJ4DePInCk/RtunlSmOsjGld+rpS9g0VRxPrzbQRWuhpkgpV+CldyrI3C/rOxJRs2vSAKXocRsGwhqEKseAJzHXmiZ5ncsaGKoeB5lUkWLwcKjyok2tHVCDlzDUpE4aA/JHNEhT48det9RqtjC71yz8m0PeK2ySI/I+Qb7eBgevgduBmt+OUxgvfKi2UB6s=","AAAAB3NzaC1yc2EAAAIAlrzL2DY9fVvwYg6CgB75uf2s8CLo+rL8Mp9tU1Ja3sDfBzj9QJjVDykupiy8s3usHfxMrHS2v3DhTiZjz/b5K6tVTgUBTXL94JfM4lwB+3EbLggPzKnlm1jQgnnU9c+tb7RX3IhBqU9Yj1gqxhErv9NFotgajQOOLgY0Ua5C0Ee+AIaMlLaNZe3LTejMsNUZMN5tl+sEmtutMHkGQsmjJxiJ3feF+Pys0I2+ojiiAfzqlMYar/5xOPl4Dj+HO+h91xVQ1/8nQRBc082fM7+ZJtDbRLtt4G8srlB5gew26jqfVASc/ui5gx4+BR9DG9VH8w+rJWBGfhOAaWqLFE2M3YuEWkjEmQMR1SQK1WFQ/oNiWJO2K5L3rk2LcAmyR6nQMtClVxYZ7CQOwa3uFL+JNXp9AhiiAtVaqhrEK81NJrJNh/+egTBl5STphxIShXd4KI9wyvkGlCIvNIMO94iXPVaWUXXbsGnU03+dsUkBzGf0eJ4DePInCk/RtunlSmOsjGld+rpS9g0VRxPrzbQRWuhpkgpV+CldyrI3C/rOxJRs2vSAKXocRsGwhqEKseAJzHXmiZ5ncsaGKoeB5lUkWLwcKjyok2tHVCDlzDUpE4aA/JHNEhT48det9RqtjC71yz8m0PeK2ySI/I+Qb7eBgevgduBmt+OUxgvfKi2UB6s=","AAAAB3NzaC1yc2EAAAADAQABAAACAQDIVXBwKGhi35gabwHNZi6Bxls1BGtDVVZFhwvhTpJKTKhV4T2HnDFG7+FBpYejc92wH026Wf+uJHIpnKkVQRnnOV98zKXW68Tz+OnwT8aBQdLI+QYDC7wLwGRf+cOiXEAkpMrp2OJme+GwQ97oBccEwdu2j9vcYAFQ0+eCPNfwPrcZhwVb00kt/moLVSxWRdsDMzQiNDZf2zel+FQIAl5cCfaLSAQa1TIXy8SM13B0brnlpdyIqukQS0zUv/PL/6AsfhgLXeQBgjs1XIf6qL+ZdtQss5AKUDuJgrWDcS3nyNZQg/CAt8XdIsLntu3bCn+VGA1O/gUGLS1a9GoGd/lRArlmODNtbds74m7hxaAf/gzg0LFJx6HhwubmVCzTXEHl95KHYHKoDvCtUOgUm7zUugxWjhsLPfT6UfZCwvCY21SGVYsoEPiTT2DhuAFriM+PT83JresFHgZDosbqW0VCi2bzAKSBu/vphaqTbSdDo0xhkW9JCb3zUkW2ge/e/GrjxV4cNXRC9XQ/XYEIWmtF/gHSi0i9KweX4sN5TEkB/41vDvyDOdyPJ8Jta0I9vBolDwJ6qdMHOPlOW5oW83yCgbmUJNYkZ+MivABlc6iS/006qYiIwknHezbY5foYd8kDON7YAssOwCJcG5viII50Z1N9VsGkUv5sZMr2p9ry8Q==","06ff3cce443ed832927576d982b69d5a526d0e63334c72e87201deda61679406",,,,,,,65537,"yFVwcChoYt+YGm8BzWYugcZbNQRrQ1VWRYcL4U6SSkyoVeE9h5wxRu/hQaWHo3PdsB9Nuln/riRyKZypFUEZ5zlffMyl1uvE8/jp8E/GgUHSyPkGAwu8C8BkX/nDolxAJKTK6djiZnvhsEPe6AXHBMHbto/b3GABUNPngjzX8D63GYcFW9NJLf5qC1UsVkXbAzM0IjQ2X9s3pfhUCAJeXAn2i0gEGtUyF8vEjNdwdG655aXciKrpEEtM1L/zy/+gLH4YC13kAYI7NVyH+qi/mXbULLOQClA7iYK1g3Et58jWUIPwgLfF3SLC57bt2wp/lRgNTv4FBi0tWvRqBnf5UQK5ZjgzbW3bO+Ju4cWgH/4M4NCxSceh4cLm5lQs01xB5feSh2ByqA7wrVDoFJu81LoMVo4bCz30+lH2QsLwmNtUhlWLKBD4k09g4bgBa4jPj0/Nya3rBR4GQ6LG6ltFQotm8wCkgbv76YWqk20nQ6NMYZFvSQm981JFtoHv3vxq48VeHDV0QvV0P12BCFprRf4B0otIvSsHl+LDeUxJAf+Nbw78gzncjyfCbWtCPbwaJQ8CeqnTBzj5TluaFvN8goG5lCTWJGfjIrwAZXOokv9NOqmIiMJJx3s22OX6GHfJAzje2ALLDsAiXBub4iCOdGdTfVbBpFL+bGTK9qfa8vE=",4096,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"publickey, keyboard-interactive, password","Cisco",,,,"enterprise" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv deleted file mode 100644 index 0b125001be..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","port","hostname","tag","handshake","asn","geo","region","city","cipher_suite","ssl_poodle","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serial_number","naics","sic","freak_vulnerable","freak_cipher_suite","sector","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","http_response_type","http_code","http_reason","content_type","http_connection","www_authenticate","set_cookie","server_type","content_length","transfer_encoding","http_date","cert_valid","self_signed","cert_expired","browser_trusted","validation_level","browser_error","tlsv13_support","tlsv13_cipher","jarm" -"2022-01-10 00:01:42","96.60.0.0",10443,"96-60-0-0.example.com","ssl,vpn","TLSv1.2",4181,"US","WISCONSIN","MILWAUKEE","TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA","N",1024,"FGT60D4614030700","support","2014-06-23 09:56:32","2038-01-19 03:14:07","5A:3D:FF:06:F9:E9:25:37:57:F9:09:52:33:A4:85:15:24:2D:88:7F","168CAE",2,"sha1WithRSAEncryption","rsaEncryption","Fortinet","FortiGate","US","California","Sunnyvale",,,,,"support@fortinet.com",,,"Fortinet","Certificate Authority","US","California","Sunnyvale",,,,,"support@fortinet.com",,,517311,,"N",,,"35:AB:B6:76:2A:3D:17:B2:FB:40:45:1B:FC:0A:99:0A:6E:48:57:F7:30:0A:3B:B1:1A:E6:99:70:5B:7C:32:41","88:7B:16:DB:39:44:0C:47:0E:4A:8F:0B:C5:FB:4D:45:BC:93:5A:00:43:A1:D9:7F:05:1D:86:33:02:F8:FC:57:67:A6:1D:C0:FF:F7:D2:40:D8:9A:21:AE:4E:6D:DC:E7:FF:72:BF:13:CB:EE:A7:5F:CD:83:EA:8A:5E:FB:87:DD","99:45:1F:2E:AE:EB:88:91:27:43:33:79:FA:93:7D:CA","HTTP/1.1",200,"OK","text/html",,,,"xxxxxxxx-xxxxx",131,,"Mon, 10 Jan 2022 00:01:44 GMT","Y","N","N","N","unknown","x509: unknown error",,, -"2022-01-10 00:01:42","113.160.0.0",10443,"","ssl","TLSv1.2",45899,"VN","THAI BINH","THAI BINH","TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256","N",2048,"1078-btb-tbi-HungHa-61d39c6d5a7e2","1078-btb-tbi-HungHa-61d39c6d5a7e2","2022-01-04 01:01:34","2023-02-06 01:01:34","A9:00:BB:E1:54:4D:56:54:59:F1:B7:EA:F1:1A:D5:36:5C:63:90:8E","36974C4C6B1B3785",2,"sha256WithRSAEncryption","rsaEncryption","pfSense webConfigurator Self-Signed Certificate",,,,,,,,,,,,"pfSense webConfigurator Self-Signed Certificate",,,,,,,,,,,,517311,,"N",,,"38:85:F0:44:1E:AD:84:B8:2F:43:68:BA:AC:EE:17:13:A4:BF:86:1D:48:75:7E:22:FA:08:4C:28:5F:AC:3E:5F","AE:1B:4F:D1:E4:C0:35:9D:2A:4F:7A:37:B8:7B:11:9D:84:25:23:21:AB:EF:B2:0F:DC:C9:F2:A3:72:28:92:E1:74:72:FA:E1:09:6C:E1:F6:B6:E3:A7:61:1C:58:89:34:D7:06:5C:3D:0A:A7:F6:CC:8A:D6:24:D0:04:4C:03:02","16:93:9A:F4:35:7F:9A:85:45:71:91:C7:7C:80:88:00","HTTP/1.1",200,"OK","text/html; charset=UTF-8","keep-alive",,"PHPSESSID=e15bdfa5739c36877608eb4cf46cc388; path=/; secure; HttpO","nginx",,"chunked","Mon, 10 Jan 2022 00:01:44 GMT","N","Y","N","N","unknown","x509: unknown error",,, -"2022-01-10 00:01:42","34.224.0.0",10443,"","ssl,vpn","TLSv1.2",14618,"US","VIRGINIA","ASHBURN","TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA","N",2048,"","Entrust Certification Authority - L1K","2021-10-07 15:30:28","2022-11-06 15:30:28","AD:19:B2:1C:CB:88:70:9B:DB:8E:7E:F5:65:50:13:D6:43:6C:BE:6E","7B388364A24B88E77E5553B5C6748100",2,"sha256WithRSAEncryption","rsaEncryption","Ciena Corporation",,"US","Maryland","Hanover",,,,,,,,"Entrust, Inc.","(c) 2012 Entrust, Inc. - for authorized use only","US",,,,,,,,,,454110,,"N",,"Retail Trade","9A:64:73:0B:8A:FA:DE:22:D4:6D:5A:C6:C4:6F:D4:A4:2A:28:FA:41:1E:FF:81:DC:D4:D9:00:FD:78:DF:C4:DD","9A:B7:BD:68:7D:F3:E7:C1:B7:D3:F4:2F:01:B6:C4:77:90:A3:2B:1E:C0:89:F5:08:EC:43:87:35:60:36:D4:87:61:AA:B8:A8:B3:8A:E9:F1:04:AA:5B:67:12:FF:63:D5:14:80:77:6E:8F:7D:C3:E2:3A:F3:13:DF:08:43:6C:B0","E7:34:BC:92:84:FA:39:DE:E1:46:6C:27:DA:5A:01:F4","HTTP/1.1",200,"OK","text/html",,,,"xxxxxxxx-xxxxx",131,,"Mon, 10 Jan 2022 00:01:44 GMT","Y","N","N","Y","OV",,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv deleted file mode 100644 index ab28456b4e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv +++ /dev/null @@ -1,46 +0,0 @@ -"timestamp","ip","port","hostname","tag","handshake","asn","geo","region","city","cipher_suite","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serial_number","naics","sic","freak_vulnerable","freak_cipher_suite","sector","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","http_response_type","http_code","http_reason","content_type","http_connection","www_authenticate","set_cookie","server_type","content_length","transfer_encoding","http_date","cert_valid","self_signed","cert_expired","browser_trusted","validation_level","browser_error","device_model","device_sector","device_type","device_vendor","device_version","jarm","page_sha256fp","raw_cert","raw_cert_chain","tlsv13_cipher","tlsv13_support" -"2018-04-23 13:25:21","198.51.100.232","443",,"ssl-freak","TLSv1.0","8447","AT","WIEN","VIENNA","TLS_RSA_WITH_RC4_128_SHA","1024","usg50_B0B2DC2FA69D","usg50_B0B2DC2FA69D","2012-05-10 00:01:19","2032-05-05 00:01:19","14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2","4FAB054F","sha1WithRSAEncryption","rsaEncryption",,,,,,,,,,,,,,,,,,,,,,,,,"0","0","Y","TLS_RSA_EXPORT_WITH_RC4_40_MD5",,"57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1","E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87","1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE","HTTP/1.1","200","OK","text/html",,,,,,"chunked","Mon, 23 Apr 2018 13:25:26 GMT","Y","Y","N","N","unknown","x509: unknown error",,,,,,,,,,, -"2018-04-23 13:25:26","198.51.100.224","443","198-51-100-224.example.net","ssl-freak","TLSv1.0","12577","AT","NIEDEROSTERREICH","BADEN","TLS_RSA_WITH_RC4_128_SHA","1024","usg20w_C86C870287EC","usg20w_C86C870287EC","2010-01-01 00:00:53","2029-12-27 00:00:53","14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2","4B3D3B35","sha1WithRSAEncryption","rsaEncryption",,,,,,,,,,,,,,,,,,,,,,,,,"0","0","Y","TLS_RSA_EXPORT_WITH_RC4_40_MD5",,"57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1","E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87","1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE","HTTP/1.1","200","OK","text/html",,,,,,"chunked","Mon, 23 Apr 2018 13:25:29 GMT","Y","Y","N","N","unknown","x509: unknown error",,,,,,,,,,, -2018-04-23 13:25:21,198.51.100.232,443,,ssl-freak,TLSv1.0,8447,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg50_B0B2DC2FA69D,usg50_B0B2DC2FA69D,2012-05-10 00:01:19,2032-05-05 00:01:19,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4FAB054F,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:25:26 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:25:26,198.51.100.224,443,198-51-100-224.example.net,ssl-freak,TLSv1.0,12577,AT,NIEDEROSTERREICH,BADEN,TLS_RSA_WITH_RC4_128_SHA,1024,usg20w_C86C870287EC,usg20w_C86C870287EC,2010-01-01 00:00:53,2029-12-27 00:00:53,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4B3D3B35,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:25:29 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:25:33,198.51.100.67,443,,ssl-freak,TLSv1.0,8447,AT,NIEDEROSTERREICH,WAIDHOFEN AN DER THAYA,TLS_RSA_WITH_RC4_128_SHA,1024,Technicolor TG670,Technicolor TG670,2005-01-01 00:00:00,2024-12-31 00:00:00,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,-7A2C610E,sha1WithRSAEncryption,rsaEncryption,Technicolor,1112WT0YK,,,,,,,,,,,Technicolor,1112WT0YK,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Moved Temporarily,,keep-alive,,xAuth_SESSION_ID=bm90aGluZyBoZXJlCg==; path=/;,,0,,"Mon, 23 Apr 2018 14:25:37 GMT",N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:25:36,198.51.100.3,443,,ssl-freak,TLSv1.2,8445,AT,SALZBURG,HINTERGLEMM,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,2048,uag2100_04BF6D22A5A9,uag2100_04BF6D22A5A9,2016-03-08 20:27:08,2026-03-06 20:27:08,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,B0F07D300BDB4FC4,sha256WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_DES40_CBC_SHA,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:25:39 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:25:38,198.51.100.198,443,198-51-100-198.example.net,ssl-freak,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,2048,198-51-100-198.example.net,Go Daddy Secure Certificate Authority - G2,2016-12-29 08:51:00,2019-12-29 08:51:00,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,AEA6D3637023B56B,sha256WithRSAEncryption,rsaEncryption,,Domain Control Validated,,,,,,,,,,,198-51-100-198.example.net," Inc.""",http://certs.godaddy.com/repository/,SomeState,SomeCity,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,403,Forbidden ( The server,text/html,close,,,,2024,,,Y,N,N,Y,DV,,,,,,,,,,,, -2018-04-23 13:25:38,198.51.100.98,443,198-51-100-98.example.net,ssl-freak,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg50_FCF528743754,usg50_FCF528743754,2013-04-29 00:00:26,2033-04-24 00:00:26,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,517DB81A,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:25:40 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:25:41,198.51.100.156,443,198-51-100-156.example.net,ssl-freak,TLSv1.0,8339,AT,NIEDEROSTERREICH,SCHWECHAT,TLS_RSA_WITH_AES_128_CBC_SHA,1024,usg200_404A036775FC,usg200_404A036775FC,2010-05-01 00:04:04,2030-04-26 00:04:04,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4BDB6FF4,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_DES40_CBC_SHA,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:25:43 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:25:53,198.51.100.200,443,,ssl-freak,TLSv1.2,8447,AT,NIEDEROSTERREICH,KREMS AN DER DONAU,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_5CF4AB67FC6F,usg20_5CF4AB67FC6F,2015-12-02 00:00:47,2035-11-27 00:00:47,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,565E34AF,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:25:56 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:02,198.51.100.83,443,198-51-100-83.example.net,ssl-freak,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg20w_FCF5286F5972,usg20w_FCF5286F5972,2013-03-23 00:00:43,2033-03-18 00:00:43,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,514CF0AB,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:05 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:03,198.51.100.155,443,198-51-100-155.example.net,ssl-freak,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-155.example.net,198-51-100-155.example.net,2018-03-19 19:47:07,2023-03-19 19:47:07,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,2DF52AA905C7A2B44C2B9F0012FD5745,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,403,Forbidden,text/html,,,,Microsoft-IIS/6.0,1939,,"Mon, 23 Apr 2018 13:11:52 GMT",N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:03,198.51.100.129,443,198-51-100-129.example.net,ssl-freak,TLSv1.0,29654,AT,SALZBURG,SALZBURG,TLS_RSA_WITH_RC4_128_SHA,1024,localhost,localhost,2007-01-31 19:00:29,2008-01-31 19:00:29,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,2,sha1WithRSAEncryption,rsaEncryption,Apache HTTP Server,Test Certificate,,,,,,,,,,,Apache HTTP Server,For testing purposes only,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Apache,318,,"Mon, 23 Apr 2018 17:42:37 GMT",N,N,Y,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:26:12,198.51.100.7,443,198-51-100-7.example.net,ssl-freak,TLSv1.0,8445,AT,SALZBURG,ALTENMARKT IM PONGAU,TLS_RSA_WITH_RC4_128_SHA,2048,IMM2-5cf3fcaf3abd,IMM2-5cf3fcaf3abd,2013-03-22 14:32:06,2023-03-20 14:32:06,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,D8C631398B585F10,sha1WithRSAEncryption,rsaEncryption,System X,,US,SomeState,SomeCity,,,,,,,,System X,,US,SomeState,SomeCity,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,301,Moved Permanently,application/x-appweb-php,keep-alive,,,Mbedthis-Appweb/2.4.2,0,,"Mon, 23 Apr 2018 13:37:08 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:15,198.51.100.93,443,,ssl-freak,TLSv1.2,8447,AT,KARNTEN,SPITTAL AN DER DRAU,TLS_RSA_WITH_RC4_128_SHA,1024,usg50_B0B2DC3308EF,usg50_B0B2DC3308EF,2012-05-25 00:00:39,2032-05-20 00:00:39,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4FBECBA7,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:17 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:16,198.51.100.81,443,198-51-100-81.example.net,ssl-freak,TLSv1.0,5385,AT,VORARLBERG,FELDKIRCH,TLS_RSA_WITH_RC4_128_SHA,1024,usg100_5067F03642A5,usg100_5067F03642A5,2010-10-01 00:04:48,2030-09-26 00:04:48,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4CA525A0,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,518210,737415,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:19 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:17,198.51.100.162,443,198-51-100-162.example.net,ssl-freak,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,rc1,Peppercon CA,2003-05-08 16:30:05,2008-05-06 16:30:05,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,18,md5WithRSAEncryption,rsaEncryption,,R&D,DE,SomeState,,,,,,198-51-100-162.example.net,,,,Security Department,DE,SomeState,SomeCity,,,,,198-51-100-162.example.net,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Redirect,,,,,,,,,N,N,Y,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:26:22,198.51.100.57,443,,ssl-freak,TLSv1.0,8447,AT,STEIERMARK,GLEISDORF,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_5CF4AB661192,usg20_5CF4AB661192,2015-09-22 00:00:46,2035-09-17 00:00:46,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,56009A2E,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:25 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:28,198.51.100.146,443,198-51-100-146.example.net,ssl-freak,TLSv1.0,8447,AT,WIEN,VIENNA,TLS_ECDHE_RSA_WITH_RC4_128_SHA,1024,zywall_110_A0E4CB7CE5AF,zywall_110_A0E4CB7CE5AF,2015-01-26 17:19:56,2025-01-23 17:19:56,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,54C6773C,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:31 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:34,198.51.100.233,443,,ssl-freak,TLSv1.0,8447,AT,KARNTEN,KLAGENFURT AM WORTHERSEE,TLS_RSA_WITH_RC4_128_SHA,1024,198.51.100.174,198-51-100-174.example.net,2009-04-14 07:26:09,2025-04-15 07:26:09,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,A0571920C03C9EE0DA1168E586E0E8D440E42EA69D898AC829,sha1WithRSAEncryption,rsaEncryption,,General,DE,SomeState,SomeCity,,,,,198-51-100-174.example.net,,,,General,DE,SomeState,SomeCity,,,,,198-51-100-174.example.net,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,Close,,,LANCOM 1781A 8.50.0161 / 09.08.2011,,,,Y,N,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:35,198.51.100.106,443,198-51-100-106.example.net,ssl-freak,TLSv1.0,12793,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-106.example.net,SHT-Gruppe CA,2004-07-20 07:28:10,2006-07-20 07:38:10,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,190DBE75000000000007,sha1WithRSAEncryption,rsaEncryption,,,AT,SomeState,SomeCity,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Microsoft-IIS/6.0,1508,,"Mon, 23 Apr 2018 13:26:37 GMT",N,N,Y,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:26:37,198.51.100.191,443,,ssl-freak,TLSv1.0,8447,AT,STEIERMARK,LEBRING,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_5CF4AB669448,usg20_5CF4AB669448,2015-10-01 00:00:38,2035-09-26 00:00:38,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,560C77A6,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:40 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:42,198.51.100.235,443,198-51-100-235.example.net,ssl-freak,TLSv1.2,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg50_107BEF33651A,usg50_107BEF33651A,2014-04-24 00:00:27,2034-04-19 00:00:27,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,5358541B,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:45 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:43,198.51.100.167,443,198-51-100-167.example.net,ssl-freak,TLSv1.0,8412,AT,BURGENLAND,ELTENDORF,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-167.example.net,198-51-100-167.example.net,2008-08-19 06:57:11,2010-08-19 06:57:11,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,2,sha1WithRSAEncryption,rsaEncryption,SuSE Linux Web Server,web server,XY,unknown,unknown,,,,,198-51-100-167.example.net,,,SuSE Linux Web Server,CA,XY,SomeState,unknown,,,,,198-51-100-167.example.net,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Apache/2.2.3 (Linux/SUSE),80,,"Mon, 23 Apr 2018 13:26:45 GMT",N,N,Y,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:26:47,198.51.100.42,443,198-51-100-42.example.net,ssl-freak,TLSv1.0,8437,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-42.example.net,iLO Default Issuer (Do not trust),2013-11-05 00:00:00,2028-11-04 00:00:00,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,72FD09EF,sha1WithRSAEncryption,rsaEncryption,,,US,SomeState,SomeCity,,,,,,,,,,US,SomeState,Houston,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,,,,,,,,,N,N,N,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:26:48,198.51.100.177,443,198-51-100-177.example.net,ssl-freak,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_5CF4AB625772,usg20_5CF4AB625772,2015-03-04 00:00:39,2035-02-27 00:00:39,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,54F64B27,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:50 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:48,198.51.100.66,443,198-51-100-66.example.net,ssl-freak,TLSv1.0,5385,AT,VORARLBERG,DORNBIRN,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-66.example.net,198-51-100-66.example.net,2009-10-06 11:23:48,2015-03-29 11:23:48,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,98B18BCD61B0CD5D,sha1WithRSAEncryption,rsaEncryption,,??,??,??,??,,,,,??,,,,??,??,??,??,,,,,??,,,518210,737415,Y,TLS_RSA_EXPORT_WITH_DES40_CBC_SHA,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Found,text/html; charset=utf-8,close,,DSSignInURL=/; path=/; secure,,,,,Y,Y,Y,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:48,198.51.100.29,443,198-51-100-29.example.net,ssl-freak,TLSv1.0,6830,AT,NIEDEROSTERREICH,GUNTRAMSDORF,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_FCF52878354B,usg20_FCF52878354B,2013-05-20 00:00:39,2033-05-15 00:00:39,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,519967A7,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:50 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:49,198.51.100.235,443,,ssl-freak,TLSv1.0,8447,AT,TIROL,KITZBUHEL,TLS_RSA_WITH_RC4_128_SHA,1024,usg50_B0B2DC3AEFE7,usg50_B0B2DC3AEFE7,2012-10-30 00:02:36,2032-10-25 00:02:36,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,508F191C,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:52 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:50,198.51.100.159,443,,ssl-freak,TLSv1.0,8218,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-159.example.net,198-51-100-159.example.net,2002-01-09 20:22:25,2003-01-09 20:22:25,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,0,md5WithRSAEncryption,rsaEncryption,SomeOrganization,SomeOrganizationalUnit,--,SomeState,SomeCity,,,,,198-51-100-159.example.net,,,SomeOrganization,SomeOrganizationalUnit,--,SomeState,SomeCity,,,,,198-51-100-159.example.net,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,close,,,Apache/1.3.33 (Unix) (Red-Hat/Linux) FrontPage/4.,31,,"Mon, 23 Apr 2018 13:26:52 GMT",Y,Y,Y,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:51,198.51.100.138,443,198-51-100-138.example.net,ssl-freak,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_B0B2DC34A1F6,usg20_B0B2DC34A1F6,2012-06-16 00:00:58,2032-06-11 00:00:58,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4FDBCCBA,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:54 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:52,198.51.100.64,443,,ssl-freak,TLSv1.0,1853,AT,OBEROSTERREICH,WILHERING,TLS_RSA_WITH_RC4_128_SHA,1024,198.51.100.171,198.51.100.117,2017-08-10 10:48:40,2020-08-09 10:48:40,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,598C3A08,sha1WithRSAEncryption,rsaEncryption,,,,,SomeCity,,,,,,,,,,,,SomeCity,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,5597,,"Mon, 23 Apr 2018 13:26:54 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:55,198.51.100.189,443,198-51-100-62.example.net,ssl-freak,TLSv1.0,25255,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg20w_107BEF3A4C9E,usg20w_107BEF3A4C9E,2014-07-04 00:00:43,2034-06-29 00:00:43,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,53B5EEAB,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:57 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:56,198.51.100.17,443,198-51-100-17.example.net,ssl-freak,TLSv1.0,8447,AT,STEIERMARK,SOEDING,TLS_RSA_WITH_AES_256_CBC_SHA,1024,Vimar By-Web,Vimar By-Web,2011-10-27 09:19:55,2016-10-25 09:19:55,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,B82B13ED1FB0FD71,sha1WithRSAEncryption,rsaEncryption,,R&D,IT,SomeState,SomeCity,,,,,,,,,R&D,IT,SomeState,SomeCity,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_DES40_CBC_SHA,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Moved Temporarily,text/html,keep-alive,,,nginx/0.6.32,,chunked,"Mon, 23 Apr 2018 13:26:56 GMT",Y,Y,Y,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:56,198.51.100.111,443,,ssl-freak,TLSv1.0,8218,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-111.example.net,198-51-100-111.example.net,2002-01-09 20:22:25,2003-01-09 20:22:25,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,0,md5WithRSAEncryption,rsaEncryption,SomeOrganization,SomeOrganizationalUnit,--,SomeState,SomeCity,,,,,198-51-100-111.example.net,,,SomeOrganization,SomeOrganizationalUnit,--,SomeState,SomeCity,,,,,198-51-100-111.example.net,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,close,,,Apache/1.3.33 (Unix) (Red-Hat/Linux) FrontPage/4.,31,,"Mon, 23 Apr 2018 13:26:58 GMT",Y,Y,Y,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:56,198.51.100.179,443,198-51-100-179.example.net,ssl-freak,TLSv1.0,12605,AT,OBEROSTERREICH,LINZ,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_5CF4AB665FB9,usg20_5CF4AB665FB9,2015-09-25 00:00:42,2035-09-20 00:00:42,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,56048EAA,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:58 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:58,198.51.100.143,443,,ssl-freak,TLSv1.0,8447,AT,KARNTEN,GLAN,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_FCF5285DEDC4,usg20_FCF5285DEDC4,2012-11-09 00:00:44,2032-11-04 00:00:44,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,509C47AC,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:27:00 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:58,198.51.100.111,443,198-51-100-111.example.net,ssl-freak,TLSv1.0,1901,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,*.*,198-51-100-111.example.net,2009-01-16 12:51:43,2010-01-16 12:51:43,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,6,md5WithRSAEncryption,rsaEncryption,,,IL,SomeState,,,,,,,,,,Visonic CA,IL,SomeState,,,,,,198-51-100-111.example.net,,,518210,737415,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Found,text/html,close,,PowerLink=226002836046b4bddcd2d16b809f76d9; path=/,Apache/1.3.31 (Unix) PHP/4.3.9 mod_ssl/2.8.20 Open,,chunked,"Wed, 23 Jan 2002 10:17:09 GMT",N,N,Y,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:26:59,198.51.100.79,443,,ssl-freak,TLSv1.0,8447,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_5CF4AB65A17C,usg20_5CF4AB65A17C,2015-09-01 00:00:51,2035-08-27 00:00:51,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,55E4EAB3,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:27:02 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:59,198.51.100.90,443,,ssl-freak,TLSv1.0,8218,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-90.example.net,198-51-100-90.example.net,2002-01-09 20:22:25,2003-01-09 20:22:25,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,0,md5WithRSAEncryption,rsaEncryption,SomeOrganization,SomeOrganizationalUnit,--,SomeState,SomeCity,,,,,198-51-100-90.example.net,,,SomeOrganization,SomeOrganizationalUnit,--,SomeState,SomeCity,,,,,198-51-100-90.example.net,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,close,,,Apache/1.3.33 (Unix) (Red-Hat/Linux) FrontPage/4.,31,,"Mon, 23 Apr 2018 13:27:02 GMT",Y,Y,Y,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:27:03,198.51.100.186,443,198-51-100-186.example.net,ssl-freak,TLSv1.0,31125,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-186.example.net,198-51-100-186.example.net,2013-07-11 12:20:19,2021-07-09 12:20:19,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,D947ED19BEAB28E6,sha1WithRSAEncryption,rsaEncryption,,IT,PL,SomeState,SomeCity,,,,,198-51-100-186.example.net,,,,IT,PL,SomeState,SomeCity,,,,,198-51-100-186.example.net,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,401,Unauthorized,text/plain,close,"Basic realm=""example.com""",,Microsoft-IIS/7.5,0,,"Mon, 23 Apr 2018 14:03:40 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:27:03,198.51.100.150,443,198-51-100-150.example.net,ssl-freak,TLSv1.0,8559,AT,BURGENLAND,NEUSIEDL AM SEE,TLS_ECDHE_RSA_WITH_RC4_128_SHA,2048,198-51-100-150.example.net,COMODO RSA Domain Validation Secure Server CA,2017-02-08 00:00:00,2019-05-09 23:59:59,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,B6EF6CF436532F0252627393BD7311FD,sha256WithRSAEncryption,rsaEncryption,,Domain Control Validated,,,,,,,,,,,,,GB,SomeState,SomeCity,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:27:06 GMT",N,N,N,N,DV,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:27:03,198.51.100.141,443,198-51-100-141.example.net,ssl-freak,TLSv1.0,39372,AT,OBEROSTERREICH,HINTERSTODER,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-141.example.net,iLO Default Issuer (Do not trust),2014-01-14 00:00:00,2029-01-13 00:00:00,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,7852761B,sha1WithRSAEncryption,rsaEncryption,,,US,SomeState,SomeCity,,,,,,,,,,US,SomeState,SomeCity,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,,,,,,,,,N,N,N,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:27:04,198.51.100.194,443,198-51-100-194.example.net,ssl-freak,TLSv1.0,8447,AT,KARNTEN,GLAN,TLS_RSA_WITH_RC4_128_SHA,1024,iDRAC6 default certificate,iDRAC6 default certificate,2009-09-17 22:47:28,2019-09-15 22:47:28,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,1,sha1WithRSAEncryption,rsaEncryption,,Remote Access Group,US,SomeState,SomeCity,,,,,,,,,Remote Access Group,US,SomeState,SomeCity,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Moved Temporarily,,keep-alive,,,Mbedthis-Appweb/2.4.2,0,,"Mon, 23 Apr 2018 13:25:57 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -"2022-02-07 00:01:41","2.136.0.0",10443,"2-136-0-0.example.com","ssl,ssl-freak,ssl-poodle,vpn","TLSv1.0",12345,"ES","MADRID","MADRID","TLS_RSA_WITH_RC4_128_SHA",1024,"usg50_107BEF336340","usg50_107BEF336340","2014-04-24 00:00:32","2034-04-19 00:00:32","F5:04:98:CD:D4:67:13:E1:77:B7:38:D4:B9:43:C0:72:50:6C:0D:58",53585420,"sha1WithRSAEncryption","rsaEncryption",,,,,,,,,,,,,,,,,,,,,,,,,517311,,"Y","TLS_RSA_EXPORT_WITH_RC4_40_MD5","Communications, Service Provider, and Hosting Service","AF:3A:71:B7:1B:A2:62:4E:87:22:FF:19:3F:84:1F:7F:CC:DC:06:E0:AF:80:E2:5D:33:A5:68:9A:E3:81:25:45","14:92:CC:6B:C7:B3:09:31:50:8C:1C:8D:5B:FD:D1:BE:41:78:80:97:E0:10:11:48:1F:EE:D6:CB:4F:F0:13:D5:05:56:AC:BA:12:12:02:F7:0F:03:40:95:17:8A:5F:79:98:E1:44:EF:E6:5A:44:E3:AC:3A:F8:49:F7:AC:B6:52","E8:5F:96:16:3F:76:35:F0:07:4F:4C:2C:38:FC:27:6B","HTTP/1.1",200,"OK","text/html",,,,"",,"chunked","Mon, 07 Feb 2022 00:01:43 GMT","Y","Y","N","N","unknown","x509: unknown error",,,,,,"Zyxel","firewall","ZyWALL USG 50",,"enterprise", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv deleted file mode 100644 index 4bcc6758ac..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv +++ /dev/null @@ -1,32 +0,0 @@ -"timestamp","ip","port","hostname","tag","handshake","asn","geo","region","city","cipher_suite","ssl_poodle","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serial_number","naics","sic","sector","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","http_response_type","http_code","http_reason","content_type","http_connection","www_authenticate","set_cookie","server_type","content_length","transfer_encoding","http_date","cert_valid","self_signed","cert_expired","browser_trusted","validation_level","browser_error","tlsv13_support","tlsv13_cipher","device_model","device_sector","device_type","device_vendor","device_version","jarm","page_sha256fp","raw_cert","raw_cert_chain" -"2018-08-08 00:51:42","203.0.113.85",8443,"example.com","ssl-poodle","TLSv1.0",65540,"AT","WIEN","VIENNA","TLS_RSA_WITH_RC4_128_SHA","Y",1024,"usg20_107BEF394BA5","usg20_107BEF394BA5","2014-06-25 00:00:42","2034-06-20 00:00:42","04:FA:DE:1D:BD:4A:05:25:61:FB:F3:D6:64:74:66:44:01:22:D7:C3","53AA112A",2,"sha1WithRSAEncryption","rsaEncryption",,,,,,,,,,,,,,,,,,,,,,,,,0,0,,"16:25:9F:C7:A1:8D:64:1F:D9:25:42:BF:87:5C:4F:F3:63:14:97:21:EC:B6:67:10:F2:CA:52:37:C9:FE:49:2E","0B:2D:48:8C:4B:55:8B:F3:AB:F8:45:ED:E0:A0:63:F4:84:2F:4C:19:DC:A8:6F:7D:6A:AF:61:D7:98:AA:58:0F:CB:CA:87:D2:C3:0B:C5:DF:49:A7:84:7C:47:58:89:7D:92:B6:7B:98:7D:B1:64:4B:DC:DD:BE:9D:11:2A:D1:AE","33:E3:61:3F:5D:AA:96:99:38:A5:D6:F1:11:C7:ED:FC","HTTP/1.1",200,"OK","text/html",,,,,,"chunked","Wed, 08 Aug 2018 00:51:44 GMT","Y","Y","N","N","unknown","x509: unknown error",,,,,,,,,,, -2018-04-19 13:32:27,198.51.100.147,443,,ssl-poodle,TLSv1.0,8445,AT,SALZBURG,SALZBURG,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-147.example.net,some_issuer,2017-09-18 08:22:17,2019-09-18 08:22:17,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,746481F100000000000C,2,sha1WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,AT,Tirol,Ehrwald,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Microsoft-IIS/7.5,689,,"Thu, 19 Apr 2018 13:32:32 GMT",N,N,N,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-19 13:32:32,198.51.100.207,443,198-51-100-94.example.net,ssl-poodle,TLSv1.0,25255,AT,SALZBURG,SALZBURG,TLS_RSA_WITH_RC4_128_SHA,Y,1024,example,some_issuer,2004-06-03 11:11:43,2024-05-29 11:11:43,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,0,2,md5WithRSAEncryption,rsaEncryption,,,US,,,,,,,,,,,,US,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,close,,"rg_cookie_session_id=1654544029; path=/; expires=Fri, 01 Jan 2038",,,,"Thu, 19 Apr 2018 13:32:34 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:32,198.51.100.200,443,198-51-100-200.example.net,ssl-poodle,TLSv1.2,8445,AT,SALZBURG,,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-200.example.net,198-51-100-200.example.net,2016-10-01 14:09:12,2020-10-02 14:09:12,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,2E8C9E4A2C7D3EDC,2,sha256WithRSAEncryption,rsaEncryption,some_org_name,,AT,,,,,,,,,,some_org_name,,AT,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html; charset=utf-8,close,,,,,,,N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:33,198.51.100.239,443,198-51-100-239.example.net,ssl-poodle,TLSv1.0,8437,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,Y,1024,198-51-100-239.example.net,198-51-100-239.example.net,2011-07-27 13:30:18,2012-07-26 13:30:18,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,7C91,2,sha1WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,--,SomeState,SomeCity,,,,,198-51-100-239.example.net,,,SomeOrganization,SomeOrganizationalUnit,--,SomeState,SomeCity,,,,,198-51-100-239.example.net,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,403,Forbidden,text/html; charset=UTF-8,close,,,Apache/2.2.3 (CentOS),4958,,"Thu, 19 Apr 2018 13:32:35 GMT",Y,Y,Y,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:35,198.51.100.156,443,,ssl-poodle,TLSv1.0,8447,AT,NIEDEROSTERREICH,,TLS_RSA_WITH_RC4_128_SHA,Y,1024,example,some_issuer,2010-01-01 00:00:52,2029-12-27 00:00:52,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4B3D3B34,2,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Thu, 19 Apr 2018 13:32:37 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:36,198.51.100.122,443,198-51-100-122.example.net,ssl-poodle,TLSv1.2,36351,AT,AUSTRIA,?,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,Y,2048,198-51-100-122.example.net,COMODO RSA Organization Validation Secure Server CA,2017-04-06 00:00:00,2019-04-06 23:59:59,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,CAB81F32F3FF4766BC545A2C14DF34B5,2,sha256WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,AT,Wien,Wien,,1130,,,,,,COMODO CA Limited,,GB,Greater Manchester,Salford,,,,,,,,518210,737401,Information Technology,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Apache,0,,"Thu, 19 Apr 2018 13:32:20 GMT",Y,N,N,Y,OV,,,,,,,,,,,, -2018-04-19 13:32:37,198.51.100.58,443,198-51-100-58.example.net,ssl-poodle,TLSv1.2,12605,AT,OBEROSTERREICH,LINZ,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,example,some_issuer,2015-01-17 16:11:24,2020-01-17 16:11:24,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,6D9E2D4443F1D69E4A8865CC1C5B6963,2,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Microsoft-IIS/8.5,701,,"Thu, 19 Apr 2018 13:34:53 GMT",N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:38,198.51.100.18,443,198-51-100-18.example.net,ssl-poodle,TLSv1.2,6830,AT,OBEROSTERREICH,LINZ,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,Y,2048,198-51-100-18.example.net,TERENA SSL CA 3,2017-07-14 00:00:00,2020-07-22 12:00:00,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,0386AD387BEC13878473D23C8C786ECE,2,sha256WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,AT,,Linz,,,,,,,,TERENA,,NL,Noord-Holland,Amsterdam,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Found,,Close,,BNIS_ChallengeState=Bqyd+IQebjQwiiYNKBJkA5Ta0spL1gX5; Path=/; Exp,,61,,,Y,N,N,Y,OV,,,,,,,,,,,, -2018-04-19 13:32:38,198.51.100.246,443,,ssl-poodle,TLSv1.2,8447,AT,SALZBURG,SALZBURG,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,example,some_issuer,2014-09-01 16:18:46,2054-08-24 16:18:46,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,129FA64A4BE039B54E850F1AA65AD835,2,sha256WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Found,text/html; charset=utf-8,,,ASP.NET_SessionId=e3qfk1dfz2mtqwzoym3gul3r; path=/; HttpOnly,Microsoft-IIS/8.5,145,,"Thu, 19 Apr 2018 13:32:40 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:39,198.51.100.35,443,198-51-100-35.example.net,ssl-poodle,TLSv1.0,12605,AT,OBEROSTERREICH,LINZ,TLS_RSA_WITH_AES_128_CBC_SHA,Y,2048,198-51-100-35.example.net,Go Daddy Secure Certificate Authority - G2,2017-08-28 13:29:01,2018-09-10 06:28:49,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,90B22B4CEF57C0FC,2,sha256WithRSAEncryption,rsaEncryption,,some_org_name,,,,,,,,198-51-100-35.example.net,,,,,US,Arizona,Scottsdale,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Microsoft-IIS/7.5,266,,"Thu, 19 Apr 2018 13:35:03 GMT",Y,N,N,Y,DV,,,,,,,,,,,, -2018-04-19 13:32:39,198.51.100.142,443,,ssl-poodle,TLSv1.0,8447,AT,WIEN,VIENNA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,Y,2048,198.51.100.19,198-51-100-19.example.net,2014-12-11 09:57:33,2024-12-08 09:57:33,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,A0571DCBE5E1A2C062D8FB7001271581B5F69824157E385563FA23527E0B,2,sha256WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,DE,,,,,,,198-51-100-19.example.net,,,some_org_name,Engineering,DE,NRW,Wuerselen,,,,,198-51-100-19.example.net,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,Close,,,LANCOM,,,"Thur, 19 Apr 2018 13:32:41 GMT",Y,N,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:39,198.51.100.178,443,,ssl-poodle,TLSv1.0,8447,AT,NIEDEROSTERREICH,,TLS_RSA_WITH_RC4_128_SHA,Y,1024,example,some_issuer,2012-05-30 00:00:44,2032-05-25 00:00:44,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4FC5632C,2,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Thu, 19 Apr 2018 13:32:41 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:40,198.51.100.99,443,198-51-100-99.example.net,ssl-poodle,TLSv1.2,6830,AT,WIEN,VIENNA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-99.example.net,RapidSSL RSA CA 2018,2018-03-30 00:00:00,2019-04-29 12:00:00,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,0BDCB5D6D4C22BD2A1CF55584B6DE09C,2,sha256WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,DigiCert Inc,198-51-100-99.example.net,US,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,404,Not Found,text/html; charset=us-ascii,close,,,Microsoft-HTTPAPI/2.0,315,,"Thu, 19 Apr 2018 13:32:43 GMT",Y,N,N,Y,DV,,,,,,,,,,,, -2018-04-19 13:32:40,198.51.100.235,443,198-51-100-235.example.net,ssl-poodle,TLSv1.0,25255,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,Y,1024,Nextcloud,Nextcloud,2016-12-13 20:28:39,2017-01-12 20:28:39,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,CDE5769D28C80B6B,2,sha1WithRSAEncryption,rsaEncryption,some_org_name,,AU,Some-State,,,,,,,,,Internet Widgits Pty Ltd,,AU,Some-State,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,400,Bad Request,text/html; charset=UTF-8,close,,nc_sameSiteCookiestrict=true; path=/; httponly;secure; expires=Fr,Apache/2.4.10 (FreeBSD) OpenSSL/0.9.8zd-freebsd PH,6939,,"Thu, 19 Apr 2018 13:32:42 GMT",Y,Y,Y,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:41,198.51.100.187,443,198-51-100-187.example.net,ssl-poodle,TLSv1.2,28760,AT,OBEROSTERREICH,,TLS_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-187.example.net,Go Daddy Secure Certificate Authority - G2,2018-02-12 17:56:01,2020-02-12 17:56:01,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,03BA30FF4972177C,2,sha256WithRSAEncryption,rsaEncryption,,some_org_name,,,,,,,,198-51-100-187.example.net,,,,,US,Arizona,Scottsdale,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,400,No parameters passed t,text/html,,,,Microsoft-IIS/10.0,11,,"Thu, 19 Apr 2018 13:32:42 GMT",Y,N,N,Y,DV,,,,,,,,,,,, -2018-04-19 13:32:42,198.51.100.213,443,198-51-100-213.example.net,ssl-poodle,TLSv1.2,8447,AT,OBEROSTERREICH,,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-213.example.net,some_issuer,2016-09-22 08:12:17,2018-09-22 08:12:17,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,770000000EBB9429663601BAB700000000000E,2,sha1WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,AT,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,301,Moved Permanently,,close,,,Microsoft-IIS/8.5,0,,"Thu, 19 Apr 2018 13:32:44 GMT",N,N,N,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-19 13:32:42,198.51.100.74,443,198-51-100-74.example.net,ssl-poodle,TLSv1.0,62363,AT,STEIERMARK,,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,DC,DC,2016-12-30 17:15:38,2021-12-30 17:15:38,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,7753CCEB55990A834E15DAC5707D403A,2,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Microsoft-IIS/7.5,689,,"Thu, 19 Apr 2018 13:32:44 GMT",N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:43,198.51.100.145,443,198-51-100-145.example.net,ssl-poodle,TLSv1.0,8447,AT,KARNTEN,KLAGENFURT AM WORTHERSEE,TLS_RSA_WITH_RC4_128_SHA,Y,1024,localdomain,localdomain,2008-10-07 20:12:54,2018-10-07 20:12:54,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,91B04FFCF174CCFF,0,sha1WithRSAEncryption,rsaEncryption,some_org_name,,CA,,,,,,,198-51-100-145.example.net,,,some_org_name,,CA,Quebec,Gatineau,,,,,198-51-100-145.example.net,,,518210,737415,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.0,302,Found,text/html; charset=UTF-8,close,,"HOMEBASEID=658512b32961b9b6f8df7a3d4de7fa01; expires=Tue, 19-Jan-",Apache/2.2.3 (Red Hat),0,,"Thu, 19 Apr 2018 12:52:32 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:44,198.51.100.48,443,198-51-100-48.example.net,ssl-poodle,TLSv1.0,1901,AT,NIEDEROSTERREICH,,TLS_RSA_WITH_RC4_128_SHA,Y,1024,198-51-100-48.example.net,198-51-100-48.example.net,2013-06-15 20:10:49,2023-06-15 20:10:49,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,013F49762DAE,0,sha1WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,US,,,,,,,198-51-100-48.example.net,,,Western Digital,Branded Products,US,CS,Mountain View,,,,,,,,518210,737415,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Apache,225,,"Thu, 19 Apr 2018 03:08:06 GMT",N,N,N,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-19 13:32:45,198.51.100.94,443,198-51-100-94.example.net,ssl-poodle,TLSv1.2,6830,AT,WIEN,VIENNA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-94.example.net,RapidSSL CA,2013-04-03 17:02:33,2014-04-07 03:32:33,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,0B697D,2,sha1WithRSAEncryption,rsaEncryption,,some_org_name,,,,,,,,,,KtAjvog6HgAsml0cyxE4hpc9kv8dhgWZ,"GeoTrust, Inc.",,US,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Found,text/html; charset=utf-8,,,ASP.NET_SessionId=z5lph4ufefkvg1xzmd4q2m33; path=/; HttpOnly,Microsoft-IIS/8.0,144,,"Thu, 19 Apr 2018 13:32:48 GMT",Y,N,Y,N,unknown,x509: certificate has expired or is not yet valid,,,,,,,,,,, -2018-04-19 13:32:45,198.51.100.53,443,198-51-100-53.example.net,ssl-poodle,TLSv1.0,8447,AT,TIROL,,TLS_RSA_WITH_RC4_128_SHA,Y,1024,example,some_issuer,2008-11-13 13:47:18,2028-11-08 13:47:18,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,BE2B43544C0AFF2E,0,sha1WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,DE,,,,,,,198-51-100-53.example.net,,,some_org_name,some_org_name,DE,Niedersachsen,38162 Cremlingen (OT Schandelah),,,,,198-51-100-53.example.net,,,518210,737415,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html; charset=iso-8859-1;,,,,GoAhead-Webs,,,,Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:46,198.51.100.56,443,198-51-100-56.example.net,ssl-poodle,TLSv1.0,8445,AT,TIROL,,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-56.example.net,some_issuer,2016-11-28 08:05:12,2018-11-28 08:05:12,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,637D34F100010000000E,2,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Microsoft-IIS/7.5,689,,"Thu, 19 Apr 2018 13:32:49 GMT",N,N,N,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-19 13:32:46,198.51.100.82,443,198-51-100-82.example.net,ssl-poodle,TLSv1.0,6830,AT,OBEROSTERREICH,,TLS_RSA_WITH_AES_128_CBC_SHA,Y,1024,123AFG,7426AC8186F3,2011-01-01 00:00:06,2020-12-29 00:00:06,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,8186F3,2,sha1WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,US,,,,,,,,,,"Cisco Systems, Inc.",some_org_name,US,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.0,200,OK,text/html,close,,,Embedded HTTP Server.,107,,"Sat, 01 Jan 2011 00:00:45 GMT",N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:48,198.51.100.29,443,198-51-100-29.example.net,ssl-poodle,TLSv1.0,6830,AT,STEIERMARK,GRAZ,TLS_RSA_WITH_RC4_128_SHA,Y,1024,198.51.100.43,198.51.100.22,2018-04-18 13:32:09,2038-01-15 13:32:09,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,862D98F4B99D0042,2,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,518210,737415,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.0,200,OK,text/html; charset=utf-8,,,,,,,,Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:49,198.51.100.114,443,198-51-100-114.example.net,ssl-poodle,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_AES_256_CBC_SHA,Y,1024,198-51-100-114.example.net,198-51-100-114.example.net,2009-08-25 17:47:57,2019-05-25 17:47:57,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,FB09C7848A7F4D77,0,sha1WithRSAEncryption,rsaEncryption,some_org_name,,AT,Vienna,Vienna,,,,,198-51-100-114.example.net,,,Digispectrum,,AT,Vienna,Vienna,,,,,198-51-100-114.example.net,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html; charset=utf-8,close,,b69223925949d45306d32f1a3d23c011=6a01vehilfpml41pl3pq3oth52; path,Apache/2.2.3 (CentOS),,chunked,"Thu, 19 Apr 2018 13:32:52 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:49,198.51.100.11,443,198-51-100-11.example.net,ssl-poodle,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,Y,2048,FGT60C3G12019794,FGT60C3G12019794,2012-08-10 07:17:11,2022-08-11 07:17:11,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,-6CD83A89,2,sha1WithRSAEncryption,rsaEncryption,some_org_name,,,,,,,,,,,,Fortinet Ltd.,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,79,,"Thu, 19 Apr 2018 13:32:08 GMT",N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:49,198.51.100.49,443,198-51-100-49.example.net,ssl-poodle,TLSv1.2,8447,AT,NIEDEROSTERREICH,,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,Y,1024,localhost,localhost,2009-11-10 23:48:47,2019-11-08 23:48:47,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,B5C752C98781B503,0,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,518210,737415,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Apache/2.4.10 (Win32) OpenSSL/1.0.1i PHP/5.5.15,2190,,"Thu, 19 Apr 2018 13:32:55 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:50,198.51.100.236,443,,ssl-poodle,TLSv1.0,8447,AT,NIEDEROSTERREICH,,TLS_RSA_WITH_AES_128_CBC_SHA,Y,1024,example,some_issuer,2013-01-30 12:00:08,2023-01-28 12:00:08,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,-462A1420,2,sha1WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,US,,,,,,,,,,Netgear Inc.,Netgear Prosafe,US,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.0,200,OK,text/html,close,,,Embedded HTTP Server.,107,,"Sat, 01 Jan 2011 00:00:21 GMT",N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:50,198.51.100.224,443,198-51-100-224.example.net,ssl-poodle,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-224.example.net,some_issuer,2017-08-03 10:21:50,2019-08-03 10:21:50,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,6126D181000300000041,2,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,401,Unauthorized,text/html,,NTLM,,Microsoft-IIS/7.5,1344,,"Thu, 19 Apr 2018 13:32:52 GMT",N,N,N,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -"2022-02-07 00:01:41","206.162.0.0",10443,,"ssl,ssl-poodle,vpn","TLSv1.2",12345,"CA","BRITISH COLUMBIA","BURNABY","TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA","Y",1024,"FWF60D4615000455","support","2015-01-28 18:14:33","2038-01-19 03:14:07","C9:B0:4E:B7:79:94:B4:DD:A7:15:21:86:43:F9:6E:4B:C9:A2:87:D9","1CA40F",2,"sha1WithRSAEncryption","rsaEncryption","Fortinet","FortiGate","US","California","Sunnyvale",,,,,"support@fortinet.com",,,"Fortinet","Certificate Authority","US","California","Sunnyvale",,,,,"support@fortinet.com",,,517311,,"Communications, Service Provider, and Hosting Service","38:F7:E0:92:24:8C:CB:28:43:93:0B:91:17:30:B1:41:8F:4E:2D:E5:A8:93:AE:4D:FE:53:00:D3:0E:53:02:16","0C:F0:37:3F:A8:93:AE:4D:FE:53:00:D3:2A:E6:6D:0B:02:9D:B9:46:58:A6:9E:5A:35:40:FB:62:9C:81:47:0A:4F:15:5D:53:D9:2F:36:4A:0B:3B:10:61:A9:07:EE:94:EC:00:B8:9C:F7:E0:92:24:8C:CB:28:2C:DD:E7:07:C6","8A:B3:08:20:34:79:94:B4:DD:A7:36:D7:14:6E:33:50","HTTP/1.1",200,"OK","text/html",,,,,131,,"Mon, 07 Feb 2022 00:01:43 GMT","Y","N","N","N","unknown","x509: unknown error",,,,,,"Fortinet","firewall","FortiGate",,"enterprise", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv deleted file mode 100644 index fd671ec904..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","transaction_id","magic_cookie","message_length","message_type","mapped_family","mapped_address","mapped_port","xor_mapped_family","xor_mapped_address","xor_mapped_port","software","fingerprint","amplification","response_size" -"2010-02-10 00:00:00",192.168.0.1,udp,3478,node01.example.com,stun,64512,ZZ,Region,City,0,0,,000000000000000000000000,2112a442,88,0101,01,192.168.0.1,3243,01,192.168.0.1,3243,"Coturn-4.5.1.1 'dan Eider'",0xfaedd06e,5.40,108 -"2010-02-10 00:00:01",192.168.0.2,udp,3478,node02.example.com,stun,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",000000000000000000000000,2112a442,88,0101,01,51.77.39.195,45877,01,192.168.0.2,45877,"Coturn-4.5.1.1 'dan Eider'",0x21128641,5.40,108 -"2010-02-10 00:00:02",192.168.0.3,udp,3478,node03.example.com,stun,64512,ZZ,Region,City,0,0,,000000000000000000000000,2112a442,76,0101,01,192.168.0.3,16321,01,188.68.240.32,16321,"ApolloProxy-1.20.1.28 'sunflower'",,4.80,96 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv deleted file mode 100644 index 8f63554910..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sequence_number","ack_number","window_size","urgent_pointer","tcp_flags","raw_packet","sector" -"2022-01-10 09:18:23","66.9.0.0","tcp",80,,"synfulknock",18885,"US","NEW JERSEY","JERSEY CITY",,,0,791102,8192,0,4608,"3cfdfec601e4700f6a9a2000080045000034c3780000f706789442099555b869f7ee0050b20800000000000c123e8012200002aa0000020405b40101040201030305", -"2022-01-10 09:19:17","213.131.0.0","tcp",80,"host-213-131-55-210-customer.wanex.net","synfulknock",35805,"GE","TBILISI","TBILISI",,,0,791102,8192,0,4608,"90e2baaf0b84700f6a9a200008004500003434100000f2064382d58337d2b8698b720050916200000000000c123e8012200059d50000020405b40101040201030305", -"2022-01-10 09:27:39","213.178.0.0","tcp",80,,"synfulknock",29256,"SY","DIMASHQ","DAMASCUS",,,0,791102,8192,0,4608,"90e2bab9cfd4700f6a9a20000800450000340f1d0000ea068bdad5b2e6914a522f360050eb5200000000000c123e801220001b4a0000020405b40101040201030305" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv deleted file mode 100644 index 3309e9a3d8..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","banner" -"2019-09-04 12:27:34","198.123.245.145","tcp",5678,"example.local","telnet-alt",20255,"AA","LOCATION","LOCATION",0,0,"|MikroTik v6.5|Login:" -"2019-09-04 12:27:40","198.123.245.145","tcp",5678,"example.local","telnet-alt",20255,"AA","LOCATION","LOCATION",0,0,"|MikroTik v6.45.3 (stable)|Login:" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv deleted file mode 100644 index 3dde133d4e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","opcode","errorcode","error","errormessage","size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,35067,node01.example.com,tftp,64512,ZZ,Region,City,0,0,5,0,"Not defined","Get not supported",22,1.57 -"2010-02-10 00:00:01",192.168.0.2,udp,56709,node02.example.com,tftp,64512,ZZ,Region,City,0,0,5,1,"File not found","File not found",19,1.36 -"2010-02-10 00:00:02",192.168.0.3,udp,32785,node03.example.com,tftp,64512,ZZ,Region,City,0,0,5,2,"Access violation","Access violation",21,1.50 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv deleted file mode 100644 index efeab02c49..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","mac","radioname","essid","modelshort","modelfull","firmware","size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,10001,node01.example.com,"ubiquiti,iot",64512,ZZ,Region,City,0,0,00156db98c3a,kachine.meta.lidia.tereixa,Kachine-Meta-Lidia-Tereixa,NS5,,XS5.ar2313.v3.5.4494.091109.1459,148,37.00 -"2010-02-10 00:00:01",192.168.0.2,udp,10001,node02.example.com,"ubiquiti,iot",64512,ZZ,Region,City,0,0,00156d7c9188,adana.mason.lanikai.ozaner,Adana-Mason-Lanikai-Ozaner,LM5,"NanoStation Loco M5",XM.ar7240.v5.6.3.28591.151130.1749,156,39.00 -"2010-02-10 00:00:02",192.168.0.3,udp,10001,node03.example.com,"ubiquiti,iot",64512,ZZ,Region,City,0,0,0418d6000fd5,tailynn.kadija.noreen.dinkar,Tailynn-Kadija-Noreen-Dinkar,P2B-400,"PowerBeam M2 400",XW.ar934x.v5.6.5.29033.160515.2108,145,36.25 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv deleted file mode 100644 index 000f5ed42d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","port","hostname","asn","geo","region","city","naics","sic","product","banner","sector" -"2019-09-04 14:51:44","198.123.245.53",5678,,5678,"AA","LOCATION","LOCATION",0,0,"Apple remote desktop vnc","RFB 003.889", -"2019-09-04 14:51:44","198.123.245.112",5678,"localhost.localdomain",5678,"AA","LOCATION","LOCATION",517311,0,"RealVNC Enterprise v5.3 or later","RFB 005.000", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv deleted file mode 100644 index 7e279ca3e6..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","response_size","amplification","error","raw_response" -"2010-02-10 00:00:00",192.168.0.1,udp,3702,node01.example.com,ws-discovery,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",989,164.83,"Validation constraint violation: SOAP message expected",c2FtcGxlIHJlc3BvbnNlIGRhdGEK -"2010-02-10 00:00:01",192.168.0.2,udp,3702,node02.example.com,ws-discovery,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",918,183.60,"Validation constraint violation: missing root element",c2FtcGxlIHJlc3BvbnNlIGRhdGEK -"2010-02-10 00:00:02",192.168.0.3,udp,3702,node03.example.com,ws-discovery,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",989,197.80,"Validation constraint violation: SOAP message expected",c2FtcGxlIHJlc3BvbnNlIGRhdGEK diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv.license deleted file mode 100644 index 9f58c89ef0..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2023 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv deleted file mode 100644 index 7e83bbaf8f..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","opcode","reported_hostname","status","size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,177,node01.example.com,xdmcp,64512,ZZ,Region,City,0,0,Willing,node01.example.com,"Linux 3.0.101-100-default",44,6.29 -"2010-02-10 00:00:01",192.168.0.2,udp,47074,node02.example.com,xdmcp,64512,ZZ,Region,City,0,0,Willing,node02.example.com,"Linux 2.6.9-103.ELsmp",48,6.86 -"2010-02-10 00:00:02",192.168.0.3,udp,177,node03.example.com,xdmcp,64512,ZZ,Region,City,0,0,Willing,node03.example.com,"1 user, load: 6,5, 6,6, 6,6",46,6.57 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/sinkhole_http_drone.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/sinkhole_http_drone.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/sinkhole_http_drone.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/special.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/special.csv deleted file mode 100644 index 2e7b591582..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/special.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","port","protocol","asn","geo","region","city","hostname","naics","sector","tag","public_source","status","detail","method","device_vendor" -"2010-02-10 00:00:00",192.168.0.1,,,64512,ZZ,Region,City,node01.example.com,0,"Communications, Service Provider, and Hosting Service",cyclops-blink,,"likely compromised",,, -"2010-02-10 00:00:01",192.168.0.2,,,64512,ZZ,Region,City,node02.example.com,0,"Communications, Service Provider, and Hosting Service",cyclops-blink,,"likely compromised",,, -"2010-02-10 00:00:02",192.168.0.3,,,64512,ZZ,Region,City,node03.example.com,0,"Professional, Scientific, and Technical Services",cyclops-blink,,"likely compromised",,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/special.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/special.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/special.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later From b5416c7ea1690304afae0a630dca2424baac3949 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 11 Apr 2023 23:57:12 +0000 Subject: [PATCH 22/76] remove json parser - csv provides better performance --- .../shadowserver/collector_reports_api.py | 7 +- .../bots/parsers/shadowserver/parser_json.py | 171 ------------------ .../test_collector_reports_api.py | 7 +- 3 files changed, 7 insertions(+), 178 deletions(-) delete mode 100644 intelmq/bots/parsers/shadowserver/parser_json.py diff --git a/intelmq/bots/collectors/shadowserver/collector_reports_api.py b/intelmq/bots/collectors/shadowserver/collector_reports_api.py index 5e0b045c88..dc8bd6b420 100644 --- a/intelmq/bots/collectors/shadowserver/collector_reports_api.py +++ b/intelmq/bots/collectors/shadowserver/collector_reports_api.py @@ -34,7 +34,7 @@ class ShadowServerAPICollectorBot(CollectorBot, HttpMixin, CacheMixin): A list of strings or a comma-separated list of the mailing lists you want to process. types (list): A list of strings or a string of comma-separated values with the names of reporttypes you want to process. If you leave this empty, all the available reports will be downloaded and processed (i.e. 'scan', 'drones', 'intel', 'sandbox_connection', 'sinkhole_combined'). - file_format (str): File format to download ('csv' or 'json'). The default is 'json' for compatibility. Using 'csv' is recommended for best performance. + file_format (str): File format to download ('csv'). The 'json' option is not longer supported. """ country = None @@ -67,11 +67,10 @@ def init(self): self._report_list.append(self.country) if self.file_format is not None: - if not (self.file_format == 'csv' or self.file_format == 'json'): + if not (self.file_format == 'csv'): raise ValueError('Invalid file_format') else: - self.file_format = 'json' - self.logger.info("For best performance, set 'file_format' to 'csv' and use intelmq.bots.parsers.shadowserver.parser.") + self.file_format = 'csv' self.preamble = f'{{ "apikey": "{self.api_key}" ' diff --git a/intelmq/bots/parsers/shadowserver/parser_json.py b/intelmq/bots/parsers/shadowserver/parser_json.py deleted file mode 100644 index 893ad877b8..0000000000 --- a/intelmq/bots/parsers/shadowserver/parser_json.py +++ /dev/null @@ -1,171 +0,0 @@ -""" -Shadowserver JSON Parser - -SPDX-FileCopyrightText: 2020 Intelmq Team -SPDX-License-Identifier: AGPL-3.0-or-later -""" -import re -from typing import Any - -from intelmq.lib.bot import ParserBot -from intelmq.lib.exceptions import InvalidKey, InvalidValue -import intelmq.lib.message as libmessage -import intelmq.bots.parsers.shadowserver._config as config - - -class ShadowserverJSONParserBot(ParserBot): - """Parse all Shadowserver feeds in JSON format (data coming from the reports API) - Shadowserver JSON Parser - - Parameters: - feedname (str): The name of the feed - """ - __is_filename_regex = re.compile(r'^(?:\d{4}-\d{2}-\d{2}-)?(\w+)(-\w+)*\.json$') - feedname = None - _sparser_config = None - recover_line = ParserBot.recover_line_json - overwrite = True - - def init(self): - if self.feedname is not None: - feedname = self.feedname - self._sparser_config = config.get_feed_by_feedname(feedname) - if self._sparser_config: - self.logger.info('Using fixed feed name %r for parsing reports.', feedname) - else: - self.logger.info('Could not determine the feed by the feed name %r given by parameter. ' - 'Will determine the feed from the file names.', feedname) - - def parse(self, report): - report_name = report.get('extra.file_name') - if not report_name: - raise ValueError("No feedname given as parameter and the " - "processed report has no 'extra.file_name'. " - "Ensure that at least one is given. " - "Also have a look at the documentation of the bot.") - - filename_search = self.__is_filename_regex.search(report_name) - - if not filename_search: - raise ValueError(f"Report's 'extra.file_name' {report_name!r} is not valid.") - report_name = filename_search.group(1) - - self.logger.debug("Detected report's file name: %s.", report_name) - retval = config.get_feed_by_filename(report_name) - - if not retval: - raise ValueError('Could not get a config for {!r}, check the documentation.' - ''.format(report_name)) - self.feedname, self._sparser_config = retval - - return self.parse_json(report) - - def parse_line(self, line: Any, report: libmessage.Report): - conf = self._sparser_config - processedkeys = [] - - event = self.new_event(report) - event.add('feed.name', self.feedname, overwrite=self.overwrite) - - extra = {} - - for entry in conf.get('required_fields'): - intelmqkey, shadowserverkey = entry[0], entry[1] - value = self.get_value_from_config(line, entry) - - if value is not None: - event.add(intelmqkey, value) - processedkeys.append(shadowserverkey) - - # Now add optional fields. - # This action may fail, the value is added to - # extra if an add operation failed - for entry in conf.get('optional_fields'): - intelmqkey, shadowserverkey = entry[0], entry[1] - try: - value = self.get_value_from_config(line, entry) - except ValueError: - self.logger.warning('Optional key %s not found in feed %s. Possible change in data' - ' format or misconfiguration.', shadowserverkey, self.feedname) - continue - - intelmqkey, shadowserverkey = entry[0], entry[1] - if value is not None: - if intelmqkey == 'extra.': - extra[shadowserverkey] = value - processedkeys.append(shadowserverkey) - continue - elif intelmqkey and intelmqkey.startswith('extra.'): - extra[intelmqkey.replace('extra.', '', 1)] = value - processedkeys.append(shadowserverkey) - continue - elif intelmqkey is False: - # ignore it explicitly - processedkeys.append(shadowserverkey) - continue - try: - event.add(intelmqkey, value) - processedkeys.append(shadowserverkey) - except InvalidValue: - self.logger.debug('Could not add key %r in feed %r, adding it to extras.', - shadowserverkey, self.feedname) - except InvalidKey: - extra[intelmqkey] = value - processedkeys.append(shadowserverkey) - else: - processedkeys.append(shadowserverkey) - - # Now add additional constant fields. - event.update(conf.get('constant_fields', {})) - - event.add('raw', self.recover_line_json(line)) - - # Add everything which could not be resolved to extra. - for key in line: - if key not in processedkeys: - val = line[key] - if not val == "": - extra[key] = val - - if extra: - event.add('extra', extra) - - yield event - - def get_value_from_config(self, data, entry): - """ - Given a specific config, get the value for that data based on the entry - """ - conv_fun = None - - shadowserverkey = entry[1] - raw_value = data.get(shadowserverkey, None) - value = raw_value - - if raw_value is None: - raise ValueError('Key {!r} not found in feed {!r}. Possible change in data' - ' format or misconfiguration.'.format(shadowserverkey, self.feedname)) - if len(entry) > 2: - conv_fun = entry[2] - - if conv_fun is not None and raw_value is not None: - if len(entry) == 4 and entry[3]: - try: - value = conv_fun(raw_value, data) - except Exception: - self.logger.error('Could not convert shadowserverkey: %r in feed %r, ' - 'value: %r via conversion function %r.', - shadowserverkey, self.feedname, raw_value, conv_fun.__name__) - raise - else: - try: - value = conv_fun(raw_value) - except Exception: - self.logger.error('Could not convert shadowserverkey: %r in feed %r, ' - 'value: %r via conversion function %r.', - shadowserverkey, self.feedname, raw_value, conv_fun.__name__) - raise - return value - - -BOT = ShadowserverJSONParserBot diff --git a/intelmq/tests/bots/collectors/shadowserver/test_collector_reports_api.py b/intelmq/tests/bots/collectors/shadowserver/test_collector_reports_api.py index a625c9d34f..2bf6e61e9a 100644 --- a/intelmq/tests/bots/collectors/shadowserver/test_collector_reports_api.py +++ b/intelmq/tests/bots/collectors/shadowserver/test_collector_reports_api.py @@ -14,12 +14,13 @@ RANDSTR = secrets.token_urlsafe(50) ASSET_PATH = pathlib.Path(__file__).parent / 'reports-list.json' PARAMETERS = {'reports': 'anarres', 'api_key': RANDSTR, 'secret': RANDSTR, 'logging_level': 'DEBUG', 'types': ['scan_smb', 'cisco_smart_install', 'nonexistent'], 'name': 'shadowservercollector'} -REPORT = {'__type': 'Report', 'extra.file_name': '2020-08-02-scan_smb-anarres-geo.json', 'feed.accuracy': 100.0, 'feed.name': 'shadowservercollector', 'raw': 'e30='} +REPORT = {'__type': 'Report', 'extra.file_name': '2020-08-02-scan_smb-anarres-geo.csv', 'feed.accuracy': 100.0, 'feed.name': 'shadowservercollector', 'raw': 'e30='} def prepare_mocker(mocker): mocker.post('https://transform.shadowserver.org/api2/reports/list', content=ASSET_PATH.read_bytes()) - mocker.post('https://transform.shadowserver.org/api2/reports/download', text='{}') + mocker.get('https://dl.shadowserver.org/xNDSuwXrKnrLrDopU926rR75CAESMWesVCKsuyI8b8ncTv7GCX', text='{}') + mocker.get('https://dl.shadowserver.org/unnzVtn92tS9459rKIEz2J8qb7oJDv0Fa2feGUOiJLCDLqBXnN', text='{}') # Explicit skip_redis is required (although implicitly called by no_cache), otherwise fails in package build environments @@ -80,7 +81,7 @@ def test_report_sent(self, mocker): self.cache.flushdb() prepare_mocker(mocker) self.run_bot(iterations=1, parameters=PARAMETERS) - self.assertAnyLoglineEqual("Sent report: '2020-08-02-cisco_smart_install-anarres-geo.csv' (fixed: '2020-08-02-cisco_smart_install-anarres-geo.json', size: 0.00195 KiB).", 'DEBUG') + self.assertAnyLoglineEqual("Sent report: '2020-08-02-cisco_smart_install-anarres-geo.csv' (fixed: '2020-08-02-cisco_smart_install-anarres-geo.csv', size: 0.00195 KiB).", 'DEBUG') def test_report_content(self, mocker): self.cache.flushdb() From 876a41468db4d9d89a929b51abeb3df81644424d Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 11 Apr 2023 23:59:42 +0000 Subject: [PATCH 23/76] dynamic configuration model --- intelmq/bots/parsers/shadowserver/README.md | 7 + intelmq/bots/parsers/shadowserver/_config.py | 4202 +---------------- intelmq/bots/parsers/shadowserver/parser.py | 46 +- .../parsers/shadowserver/schema.json.test | 180 + .../parsers/shadowserver/update_schema.py | 12 + 5 files changed, 303 insertions(+), 4144 deletions(-) create mode 100644 intelmq/bots/parsers/shadowserver/schema.json.test create mode 100644 intelmq/bots/parsers/shadowserver/update_schema.py diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md index eb0ddfb4a7..297930861b 100644 --- a/intelmq/bots/parsers/shadowserver/README.md +++ b/intelmq/bots/parsers/shadowserver/README.md @@ -7,3 +7,10 @@ This module is maintained by [The Shadowserver Foundation](https://www.shadowser Please contact intelmq@shadowserver.org with any issues or concerns. +The report configuration is now stored in a _schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/_version_. + +For environments that have internet connectivity the `update_schema.py` script should be setup as a cron job to obtain the latest revision. + +For air-gapped systems automation will be required to download and copy the _schema.json_ file into this directory + +The parser will automatically reload the configuration when the file changes. diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index bea3d0c0b8..a7b80b7a6c 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -77,20 +77,34 @@ feed_idx is not complete. """ +import os import re import base64 import binascii +import json +import urllib.request +import tempfile from typing import Optional, Dict, Tuple, Any import intelmq.lib.harmonization as harmonization +class __Container: + pass + +__config = __Container() +__config.schema_file = os.path.join(os.path.dirname(__file__), 'schema.json') +__config.schema_mtime = 0.0 +__config.feedname_mapping = {} +__config.filename_mapping = {} def get_feed_by_feedname(given_feedname: str) -> Optional[Dict[str, Any]]: - return feedname_mapping.get(given_feedname, None) + reload() + return __config.feedname_mapping.get(given_feedname, None) def get_feed_by_filename(given_filename: str) -> Optional[Tuple[str, Dict[str, Any]]]: - return filename_mapping.get(given_filename, None) + reload() + return __config.filename_mapping.get(given_filename, None) def add_UTC_to_timestamp(value: str) -> str: @@ -165,11 +179,6 @@ def invalidate_zero(value: str) -> Optional[int]: return int(value) if value and int(value) != 0 else None -# TODO this function is a wild guess... -def set_tor_node(value: str) -> Optional[bool]: - return True if value else None - - def validate_ip(value: str) -> Optional[str]: """Remove "invalid" IP.""" # FIX: https://github.com/certtools/intelmq/issues/1720 # TODO: Find better fix @@ -240,4126 +249,63 @@ def scan_exchange_identifier(field): return 'exchange-server-webshell' return 'vulnerable-exchange-server' +functions = { + 'add_UTC_to_timestamp': add_UTC_to_timestamp, + 'convert_bool': convert_bool, + 'validate_to_none': validate_to_none, + 'convert_int': convert_int, + 'convert_float': convert_float, + 'convert_http_host_and_url': convert_http_host_and_url, + 'invalidate_zero': invalidate_zero, + 'validate_ip': validate_ip, + 'validate_network': validate_network, + 'validate_fqdn': validate_fqdn, + 'convert_date': convert_date, + 'convert_date_utc': convert_date_utc, + 'force_base64': force_base64, + 'scan_exchange_taxonomy': scan_exchange_taxonomy, + 'scan_exchange_type': scan_exchange_type, + 'scan_exchange_identifier': scan_exchange_identifier, + } + + +def reload (): + """ reload the configuration if it has changed """ + mtime = 0.0 + + if (os.path.isfile(__config.schema_file)): + mtime = os.path.getmtime(__config.schema_file) + if __config.schema_mtime == mtime: + return + schema_file = __config.schema_file + else: + # load a test schema if one has not been downloaded yet + schema_file = __config.schema_file + schema_file += '.test' + + __config.feedname_mapping.clear() + __config.filename_mapping.clear() + with open(schema_file) as fh: + schema = json.load(fh) + for report in schema: + __config.feedname_mapping[schema[report]['feed_name']] = (schema[report]['feed_name'], schema[report]) + __config.filename_mapping[schema[report]['file_name']] = (schema[report]['feed_name'], schema[report]) + __config.schema_mtime = mtime + +def update_schema (version): + """ download the latest configuration """ + (th, tmp) = tempfile.mkstemp() + url = 'https://interchange.shadowserver.org/intelmq/'+version + try: + urllib.request.urlretrieve(url, tmp) + except: + raise ValueError("Failed to download %r" % url) -# BEGIN CONFGEN - -# https://www.shadowserver.org/what-we-do/network-reporting/blocklist-report/ -blocklist = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ], - 'optional_fields': [ - ('source.network', 'ip', validate_network), - ('extra.', 'tag', validate_to_none), - ('source.reverse_dns', 'hostname'), - ('extra.', 'source', validate_to_none), - ('extra.', 'reason', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'blacklisted-ip', - 'classification.taxonomy': 'other', - 'classification.type': 'blacklist', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/compromised-website-report/ -compromised_website = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.application', 'application', validate_to_none), - ('source.url', 'url', convert_http_host_and_url, True), - ('source.fqdn', 'http_host', validate_fqdn), - ('source.reverse_dns', 'hostname'), - ('malware.name', 'tag'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('event_description.text', 'category', validate_to_none), - ('extra.', 'system', validate_to_none), - ('extra.', 'detected_since', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'redirect_target', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'cc_url', validate_to_none), - ('extra.', 'family', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'intrusions', - 'classification.type': 'system-compromise', - 'classification.identifier': 'compromised-website', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/device-identification-report/ -device_id = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'undetermined', - 'classification.identifier': 'device-id', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/ddos-participant-report/ -event_ddos_participant = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'duration', convert_int), - ('extra.', 'attack_src_port', convert_int), - ('extra.', 'http_usessl', convert_bool), - ('extra.', 'ip_header_seqnum', convert_int), - ('extra.', 'ip_header_ttl', convert_int), - ('extra.', 'number_of_connections', convert_int), - ('extra.', 'packet_length', convert_int), - ('extra.', 'packet_randomized', convert_bool), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'domain_source', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'dst_network', validate_to_none), - ('extra.', 'dst_netmask', validate_to_none), - ('extra.', 'attack', validate_to_none), - ('extra.', 'attack_src_ip', validate_to_none), - ('extra.', 'domain', validate_to_none), - ('extra.', 'domain_transaction_id', validate_to_none), - ('extra.', 'gcip', validate_to_none), - ('extra.', 'http_method', validate_to_none), - ('extra.', 'http_path', validate_to_none), - ('extra.', 'http_postdata', validate_to_none), - ('extra.', 'ip_header_ack', validate_to_none), - ('extra.', 'ip_header_acknum', validate_to_none), - ('extra.', 'ip_header_dont_fragment', validate_to_none), - ('extra.', 'ip_header_fin', validate_to_none), - ('extra.', 'ip_header_identity', validate_to_none), - ('extra.', 'ip_header_psh', validate_to_none), - ('extra.', 'ip_header_rst', validate_to_none), - ('extra.', 'ip_header_syn', validate_to_none), - ('extra.', 'ip_header_tos', validate_to_none), - ('extra.', 'ip_header_urg', validate_to_none), - ('extra.', 'http_agent', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'classification.identifier': 'ddos-participant', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-brute-force-events-report/ -event_honeypot_brute_force = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('classification.identifier', 'application'), - ('destination.account', 'username', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'service', validate_to_none), - ('extra.', 'start_time', convert_date_utc), - ('extra.', 'end_time', convert_date_utc), - ('extra.', 'client_version', validate_to_none), - ('extra.', 'password', validate_to_none), - ('extra.', 'payload_url', validate_to_none), - ('extra.', 'payload_md5', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'intrusion-attempts', - 'classification.type': 'brute-force', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-darknet-events-report/ -event_honeypot_darknet = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('classification.identifier', 'tag', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'count', convert_int), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-ddos-events/ -event_honeypot_ddos = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'duration', convert_int), - ('extra.', 'attack_src_port', convert_int), - ('extra.', 'http_usessl', convert_bool), - ('extra.', 'ip_header_seqnum', convert_int), - ('extra.', 'ip_header_ttl', convert_int), - ('extra.', 'number_of_connections', convert_int), - ('extra.', 'packet_length', convert_int), - ('extra.', 'packet_randomized', convert_bool), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'domain_source', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'dst_network', validate_to_none), - ('extra.', 'dst_netmask', validate_to_none), - ('extra.', 'attack', validate_to_none), - ('extra.', 'attack_src_ip', validate_to_none), - ('extra.', 'domain', validate_to_none), - ('extra.', 'domain_transaction_id', validate_to_none), - ('extra.', 'gcip', validate_to_none), - ('extra.', 'http_method', validate_to_none), - ('extra.', 'http_path', validate_to_none), - ('extra.', 'http_postdata', validate_to_none), - ('extra.', 'ip_header_ack', validate_to_none), - ('extra.', 'ip_header_acknum', validate_to_none), - ('extra.', 'ip_header_dont_fragment', validate_to_none), - ('extra.', 'ip_header_fin', validate_to_none), - ('extra.', 'ip_header_identity', validate_to_none), - ('extra.', 'ip_header_psh', validate_to_none), - ('extra.', 'ip_header_rst', validate_to_none), - ('extra.', 'ip_header_syn', validate_to_none), - ('extra.', 'ip_header_tos', validate_to_none), - ('extra.', 'ip_header_urg', validate_to_none), - ('extra.', 'http_agent', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'classification.identifier': 'honeypot-ddos', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-amplification-ddos-events-report/ -event_honeypot_ddos_amp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'avg_pps', convert_float), - ('extra.', 'max_pps', convert_float), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'request', validate_to_none), - ('extra.', 'count', convert_int), - ('extra.', 'bytes', convert_int), - ('extra.', 'end_time', convert_date_utc), - ('extra.', 'duration', convert_int), - ], - 'constant_fields': { - 'classification.identifier': 'amplification-ddos-victim', - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-ddos-target-events-report/ -event_honeypot_ddos_target = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'attack_src_port', convert_int), - ('extra.', 'http_usessl', convert_bool), - ('extra.', 'ip_header_seqnum', convert_int), - ('extra.', 'ip_header_ttl', convert_int), - ('extra.', 'number_of_connections', convert_int), - ('extra.', 'packet_length', convert_int), - ('extra.', 'packet_randomized', convert_bool), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'domain_source', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'dst_network', validate_to_none), - ('extra.', 'dst_netmask', validate_to_none), - ('extra.', 'attack', validate_to_none), - ('extra.', 'duration', convert_int), - ('extra.', 'attack_src_ip', validate_to_none), - ('extra.', 'domain', validate_to_none), - ('extra.', 'domain_transaction_id', validate_to_none), - ('extra.', 'gcip', validate_to_none), - ('extra.', 'http_method', validate_to_none), - ('extra.', 'http_path', validate_to_none), - ('extra.', 'http_postdata', validate_to_none), - ('extra.', 'ip_header_ack', validate_to_none), - ('extra.', 'ip_header_acknum', validate_to_none), - ('extra.', 'ip_header_dont_fragment', validate_to_none), - ('extra.', 'ip_header_fin', validate_to_none), - ('extra.', 'ip_header_identity', validate_to_none), - ('extra.', 'ip_header_psh', validate_to_none), - ('extra.', 'ip_header_rst', validate_to_none), - ('extra.', 'ip_header_syn', validate_to_none), - ('extra.', 'ip_header_tos', validate_to_none), - ('extra.', 'ip_header_urg', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'classification.identifier': 'honeypot-ddos-target', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-http-scanner-events/ -event_honeypot_http_scan = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('user_agent', 'http_agent', validate_to_none), - ('extra.method', 'http_request_method', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'pattern', validate_to_none), - ('destination.url', 'http_url', convert_http_host_and_url, True), - ('extra.', 'url_scheme', validate_to_none), - ('extra.', 'session_tags', validate_to_none), - ('extra.', 'vulnerability_enum', validate_to_none), - ('extra.', 'vulnerability_id', validate_to_none), - ('extra.', 'vulnerability_class', validate_to_none), - ('extra.', 'vulnerability_score', validate_to_none), - ('extra.', 'vulnerability_severity', validate_to_none), - ('extra.', 'vulnerability_version', validate_to_none), - ('extra.', 'threat_framework', validate_to_none), - ('extra.', 'threat_tactic_id', validate_to_none), - ('extra.', 'threat_technique_id', validate_to_none), - ('extra.', 'target_vendor', validate_to_none), - ('extra.', 'target_product', validate_to_none), - ('extra.', 'target_class', validate_to_none), - ('extra.', 'file_md5', validate_to_none), - ('extra.', 'file_sha256', validate_to_none), - ('extra.', 'request_raw', force_base64), - ('extra.', 'body_raw', force_base64), - ], - 'constant_fields': { - 'classification.taxonomy': 'information-gathering', - 'classification.type': 'scanner', - 'protocol.application': 'http', - 'classification.identifier': 'honeypot-http-scan', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-ics-scanner-events-report/ -event_honeypot_ics_scan = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'state', validate_to_none), - ('extra.', 'sensor_id', validate_to_none), - ('extra.', 'slave_id', validate_to_none), - ('extra.', 'function_code', validate_to_none), - ('extra.', 'request', validate_to_none), - ('extra.', 'response', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'ics', - 'classification.taxonomy': 'information-gathering', - 'classification.type': 'scanner', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/ip-spoofer-events-report/ -event_ip_spoofer = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'infection', validate_to_none), - ('source.network', 'network', validate_network), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'routedspoof', validate_to_none), - ('extra.', 'session', validate_to_none), - ('extra.', 'nat', convert_bool), - ], - 'constant_fields': { - 'classification.taxonomy': 'fraud', - 'classification.type': 'masquerade', - 'classification.identifier': 'ip-spoofer', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/sinkhole-events-report/ -event_sinkhole = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('classification.identifier', 'infection', validate_to_none), - ('malware.name', 'family', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('extra.', 'infection', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/sinkhole-dns-events-report/ -event_sinkhole_dns = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('extra.naics', 'src_naics', invalidate_zero), - ('extra.sector', 'src_sector', validate_to_none), - ('extra.dns_query_type', 'query_type'), - ('extra.dns_query', 'query'), - ('malware.name', 'family', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('extra.', 'infection', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'count', convert_int), - ], - 'constant_fields': { - 'classification.identifier': 'sinkholedns', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'dns', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/sinkhole-http-events-report/ -event_sinkhole_http = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('classification.identifier', 'tag'), - ('malware.name', 'family', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('extra.', 'infection', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('destination.url', 'http_url', convert_http_host_and_url, True), - ('destination.fqdn', 'http_host', validate_fqdn), - ('extra.', 'http_agent', validate_to_none), - ('extra.', 'forwarded_by', validate_to_none), - ('extra.', 'ssl_cipher', validate_to_none), - ('extra.', 'http_referer', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'protocol.application': 'http', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/sinkhole-http-referer-events-report/ -event_sinkhole_http_referer = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ], - 'optional_fields': [ - ('malware.name', 'family', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('extra.', 'infection', validate_to_none), - ('protocol.transport', 'protocol'), - ('extra.', 'http_referer_ip', validate_ip), - ('extra.', 'http_referer_port', convert_int), - ('extra.', 'http_referer_asn', invalidate_zero), - ('extra.', 'http_referer_geo', validate_to_none), - ('extra.', 'http_referer_region', validate_to_none), - ('extra.', 'http_referer_city', validate_to_none), - ('extra.', 'http_referer_hostname', validate_to_none), - ('extra.', 'http_referer_naics', invalidate_zero), - ('extra.', 'http_referer_sector', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('destination.url', 'http_url', convert_http_host_and_url, True), - ('destination.fqdn', 'http_host', validate_fqdn), - ('extra.', 'http_referer', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'classification.identifier': 'sinkhole-http-referer', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/malware-url-report/ -malware_url = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ], - 'optional_fields': [ - ('source.url', 'url', convert_http_host_and_url, True), - ('source.fqdn', 'host', validate_fqdn), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('malware.name', 'tag'), - ('extra.', 'source', validate_to_none), - ('malware.hash.sha256', 'sha256', validate_to_none), - ('extra.', 'application', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'malware-distribution', - 'classification.identifier': 'malware-url', - }, -} - -phish_url = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ], - 'optional_fields': [ - ('source.url', 'url', convert_http_host_and_url, True), - ('source.fqdn', 'host', validate_fqdn), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'source', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'fraud', - 'classification.type': 'phishing', - 'classification.identifier': 'phish-url', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-http-proxy-report/ -population_http_proxy = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('malware.name', 'tag'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'http', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'connection', validate_to_none), - ('extra.', 'proxy_authenticate', validate_to_none), - ('extra.', 'via', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-http-proxy', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'http', - }, -} - -# http://www.shadowserver.org/wiki/pmwiki.php/Services/Sandbox-Connection -sandbox_conn = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('destination.fqdn', 'host', validate_fqdn), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('malware.hash.md5', 'md5', validate_to_none), - ('protocol.transport', 'protocol'), - ('extra.', 'bytes_in', validate_to_none), - ('extra.', 'bytes_out', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'malware-distribution', - 'classification.identifier': 'sandbox-conn', - }, -} - -sandbox_dns = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ], - 'optional_fields': [ - ('extra.dns_query_type', 'type', validate_to_none), - ('malware.hash.md5', 'md5hash', validate_to_none), - ('extra.', 'request', validate_to_none), - ('extra.', 'response', validate_to_none), - ('extra.', 'family', validate_to_none), - ('malware.name', 'tag'), - ('extra.', 'source', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'dns', - 'classification.identifier': 'sandbox-dns', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/sandbox-url-report/ -sandbox_url = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ], - 'optional_fields': [ - ('destination.fqdn', 'host', validate_fqdn), - ('extra.http_request_method', 'method', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('malware.hash.md5', 'md5', validate_to_none), - ('destination.url', 'url', convert_http_host_and_url, True), - ('user_agent', 'user_agent', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'malware-distribution', - 'classification.identifier': 'sandbox-url', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-adb-report/ -scan_adb = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'name', validate_to_none), - ('extra.', 'model', validate_to_none), - ('extra.', 'device', validate_to_none), - ('extra.', 'features', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_sector', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-adb', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'adb', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-afp-report/ -scan_afp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'machine_type', validate_to_none), - ('extra.', 'afp_versions', validate_to_none), - ('extra.', 'uams', validate_to_none), - ('extra.', 'flags', validate_to_none), - ('extra.', 'server_name', validate_to_none), - ('extra.', 'signature', validate_to_none), - ('extra.', 'directory_service', validate_to_none), - ('extra.', 'utf8_servername', validate_to_none), - ('extra.', 'network_address', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-afp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'afp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-amqp-report/ -scan_amqp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'channel', validate_to_none), - ('extra.', 'message_length', convert_int), - ('extra.', 'class', validate_to_none), - ('extra.', 'method', validate_to_none), - ('extra.', 'version_major', validate_to_none), - ('extra.', 'version_minor', validate_to_none), - ('extra.', 'capabilities', validate_to_none), - ('extra.', 'cluster_name', validate_to_none), - ('extra.', 'platform', validate_to_none), - ('extra.', 'product', validate_to_none), - ('extra.', 'product_version', validate_to_none), - ('extra.', 'mechanisms', validate_to_none), - ('extra.', 'locales', validate_to_none), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-amqp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'amqp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-apple-remote-desktop-ard-report/ -scan_ard = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'machine_name', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-ard', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-chargen-report/ -scan_chargen = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.response_size', 'size', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'chargen', - 'classification.identifier': 'open-chargen', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-cisco-smart-install-report/ -scan_cisco_smart_install = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-cisco-smart-install', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'cisco-smart-install', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-coap-report/ -scan_coap = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'response', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-coap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'coap', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-couchdb-report/ -scan_couchdb = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'server_version', validate_to_none), - ('extra.', 'couchdb_message', validate_to_none), - ('extra.', 'couchdb_version', validate_to_none), - ('extra.', 'git_sha', validate_to_none), - ('extra.', 'features', validate_to_none), - ('extra.', 'vendor', validate_to_none), - ('extra.', 'visible_databases', validate_to_none), - ('extra.', 'error', validate_to_none), - ('extra.', 'error_reason', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'CouchDB', - 'classification.identifier': 'open-couchdb', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-cwmp-report/ -scan_cwmp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'http', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'connection', validate_to_none), - ('extra.', 'www_authenticate', validate_to_none), - ('extra.', 'set_cookie', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'date', validate_to_none), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'cwmp', - 'classification.identifier': 'open-cwmp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-db2-discovery-service-report/ -scan_db2 = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'size', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'db2_hostname', validate_to_none), - ('extra.', 'servername', validate_to_none), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'open-db2-discovery-service', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'db2', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/vulnerable-ddos-middlebox-report/ -scan_ddos_middlebox = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.application', 'tag'), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'source_port', validate_to_none), - ('extra.', 'bytes', convert_int), - ('extra.', 'amplification', convert_float), - ('extra.', 'method', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-ddos-middlebox', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/dns-open-resolvers-report/ -scan_dns = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'min_amplification', convert_float), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'dns_version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'dns-open-resolver', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'dns', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-docker-service-report/ -scan_docker = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'http', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'date', validate_to_none), - ('extra.', 'experimental', validate_to_none), - ('extra.', 'api_version', validate_to_none), - ('extra.', 'arch', validate_to_none), - ('extra.', 'go_version', validate_to_none), - ('extra.os.name', 'os', validate_to_none), - ('extra.', 'kernel_version', validate_to_none), - ('extra.', 'git_commit', validate_to_none), - ('extra.', 'min_api_version', validate_to_none), - ('extra.', 'build_time', validate_to_none), - ('extra.', 'pkg_version', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'docker', - 'classification.identifier': 'open-docker', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-dvr-dhcpdiscover-report/ -scan_dvr_dhcpdiscover = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.application', 'tag'), - ('extra.', 'video_input_channels', convert_int), - ('extra.', 'alarm_input_channels', convert_int), - ('extra.', 'video_output_channels', convert_int), - ('extra.', 'alarm_output_channels', convert_int), - ('extra.', 'remote_video_input_channels', convert_int), - ('extra.', 'ipv4_dhcp_enable', convert_bool), - ('extra.', 'ipv6_dhcp_enable', convert_bool), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_id', validate_to_none), - ('extra.', 'device_serial', validate_to_none), - ('extra.', 'machine_name', validate_to_none), - ('extra.', 'manufacturer', validate_to_none), - ('extra.', 'method', validate_to_none), - ('extra.', 'http_port', convert_int), - ('extra.', 'internal_port', convert_int), - ('extra.', 'mac_address', validate_to_none), - ('extra.', 'ipv4_address', validate_to_none), - ('extra.', 'ipv4_gateway', validate_to_none), - ('extra.', 'ipv4_subnet_mask', validate_to_none), - ('extra.', 'ipv6_address', validate_to_none), - ('extra.', 'ipv6_link_local', validate_to_none), - ('extra.', 'ipv6_gateway', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-dvr-dhcpdiscover', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-elasticsearch-report/ -scan_elasticsearch = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'build_snapshot', convert_bool), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'ok', convert_bool), - ('extra.', 'name', validate_to_none), - ('extra.', 'cluster_name', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'build_hash', validate_to_none), - ('extra.', 'build_timestamp', validate_to_none), - ('extra.', 'lucene_version', validate_to_none), - ('extra.', 'tagline', validate_to_none), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'elasticsearch', - 'classification.identifier': 'open-elasticsearch', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-erlang-port-mapper-report-daemon/ -scan_epmd = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'nodes', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'Erlang Port Mapper Daemon', - 'classification.identifier': 'open-epmd', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/vulnerable-exchange-server-report/ -scan_exchange = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('classification.taxonomy', 'tag', scan_exchange_taxonomy), - ('classification.type', 'tag', scan_exchange_type), - ('classification.identifier', 'tag', scan_exchange_identifier), - ('extra.', 'tag', validate_to_none), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'servername', validate_to_none), - ('destination.url', 'url', convert_http_host_and_url, True), - ], - 'constant_fields': { - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-ftp-report/ -scan_ftp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'banner', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'auth_tls_response', validate_to_none), - ('extra.', 'auth_ssl_response', validate_to_none), - ('extra.', 'tlsv13_support', validate_to_none), - ('extra.', 'tlsv13_cipher', validate_to_none), - ('extra.', 'jarm', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_sector', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-ftp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ftp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-hadoop-report/ -scan_hadoop = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'total_disk', convert_int), - ('extra.', 'used_disk', convert_int), - ('extra.', 'free_disk', convert_int), - ('source.reverse_dns', 'hostname'), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'server_type', validate_to_none), - ('extra.', 'clusterid', validate_to_none), - ('extra.', 'livenodes', validate_to_none), - ('extra.', 'namenodeaddress', validate_to_none), - ('extra.', 'volumeinfo', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-hadoop', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'hadoop', - 'protocol.transport': 'tcp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-http-report/ -scan_http = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'http', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'connection', validate_to_none), - ('extra.', 'www_authenticate', validate_to_none), - ('extra.', 'set_cookie', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-http', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'http', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-http-proxy-report/ -scan_http_proxy = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'http', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'connection', validate_to_none), - ('extra.', 'proxy_authenticate', validate_to_none), - ('extra.', 'via', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), - ], - 'constant_fields': { - 'classification.identifier': 'open-http-proxy', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'http', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/vulnerable-http-report/ -scan_http_vulnerable = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'http', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'connection', validate_to_none), - ('extra.', 'www_authenticate', validate_to_none), - ('extra.', 'set_cookie', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), - ('extra.', 'version', validate_to_none), - ('extra.', 'build_date', validate_to_none), - ('extra.', 'detail', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-http', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'http', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-ics-report/ -scan_ics = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.application', 'tag'), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_id', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'raw_response', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-ics', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-ipmi-report/ -scan_ipmi = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'none_auth', convert_bool), - ('extra.', 'md2_auth', convert_bool), - ('extra.', 'md5_auth', convert_bool), - ('extra.', 'passkey_auth', convert_bool), - ('extra.', 'oem_auth', convert_bool), - ('extra.', 'permessage_auth', convert_bool), - ('extra.', 'userlevel_auth', convert_bool), - ('extra.', 'usernames', convert_bool), - ('extra.', 'nulluser', convert_bool), - ('extra.', 'anon_login', convert_bool), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'ipmi_version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'defaultkg', validate_to_none), - ('extra.', 'error', validate_to_none), - ('extra.', 'deviceid', validate_to_none), - ('extra.', 'devicerev', validate_to_none), - ('extra.', 'firmwarerev', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'manufacturerid', validate_to_none), - ('extra.', 'manufacturername', validate_to_none), - ('extra.', 'productid', validate_to_none), - ('extra.', 'productname', validate_to_none), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ipmi', - 'protocol.transport': 'udp', - 'classification.identifier': 'open-ipmi', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-ipp-report/ -scan_ipp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'ipp_version', validate_to_none), - ('extra.', 'cups_version', validate_to_none), - ('extra.', 'printer_uris', validate_to_none), - ('extra.', 'printer_name', validate_to_none), - ('extra.', 'printer_info', validate_to_none), - ('extra.', 'printer_more_info', validate_to_none), - ('extra.', 'printer_make_and_model', validate_to_none), - ('extra.', 'printer_firmware_name', validate_to_none), - ('extra.', 'printer_firmware_string_version', validate_to_none), - ('extra.', 'printer_firmware_version', validate_to_none), - ('extra.', 'printer_organization', validate_to_none), - ('extra.', 'printer_organization_unit', validate_to_none), - ('extra.', 'printer_uuid', validate_to_none), - ('extra.', 'printer_wifi_ssid', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ipp', - 'classification.identifier': 'open-ipp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/vulnerable-isakmp-report/ -scan_isakmp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'spi_size', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'initiator_spi', validate_to_none), - ('extra.', 'responder_spi', validate_to_none), - ('extra.', 'next_payload', validate_to_none), - ('extra.', 'exchange_type', validate_to_none), - ('extra.', 'flags', validate_to_none), - ('extra.', 'message_id', validate_to_none), - ('extra.', 'next_payload2', validate_to_none), - ('extra.', 'domain_of_interpretation', validate_to_none), - ('extra.', 'protocol_id', validate_to_none), - ('extra.', 'notify_message_type', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'open-ike', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ipsec', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-kubernetes-api-server-report/ -scan_kubernetes = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'http', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'date', validate_to_none), - ('extra.', 'major', validate_to_none), - ('extra.', 'minor', validate_to_none), - ('extra.', 'git_version', validate_to_none), - ('extra.', 'git_commit', validate_to_none), - ('extra.', 'git_tree_state', validate_to_none), - ('extra.', 'build_date', validate_to_none), - ('extra.', 'go_version', validate_to_none), - ('extra.', 'compiler', validate_to_none), - ('extra.', 'platform', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'browser_trusted', convert_bool), - ('extra.', 'browser_error', validate_to_none), - ('extra.', 'raw_cert', validate_to_none), - ('extra.', 'raw_cert_chain', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'kubernetes', - 'classification.identifier': 'open-kubernetes', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-ldap-tcp-report/ -scan_ldap_tcp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.local_hostname', 'dns_host_name', validate_to_none), - ('extra.', 'domain_controller_functionality', convert_int), - ('extra.', 'domain_functionality', convert_int), - ('extra.', 'forest_functionality', convert_int), - ('extra.', 'highest_committed_usn', convert_int), - ('extra.', 'is_global_catalog_ready', convert_bool), - ('extra.', 'is_synchronized', convert_bool), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'size', convert_int), - ('extra.', 'configuration_naming_context', validate_to_none), - ('extra.', 'current_time', validate_to_none), - ('extra.', 'default_naming_context', validate_to_none), - ('extra.', 'ds_service_name', validate_to_none), - ('extra.', 'ldap_service_name', validate_to_none), - ('extra.', 'naming_contexts', validate_to_none), - ('extra.', 'root_domain_naming_context', validate_to_none), - ('extra.', 'schema_naming_context', validate_to_none), - ('extra.', 'server_name', validate_to_none), - ('extra.', 'subschema_subentry', validate_to_none), - ('extra.', 'supported_capabilities', validate_to_none), - ('extra.', 'supported_control', validate_to_none), - ('extra.', 'supported_ldap_policies', validate_to_none), - ('extra.', 'supported_ldap_version', validate_to_none), - ('extra.', 'supported_sasl_mechanisms', validate_to_none), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ldap', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-ldap-report/ -scan_ldap_udp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.local_hostname', 'dns_host_name', validate_to_none), - ('extra.', 'domain_controller_functionality', convert_int), - ('extra.', 'domain_functionality', convert_int), - ('extra.', 'forest_functionality', convert_int), - ('extra.', 'highest_committed_usn', convert_int), - ('extra.', 'is_global_catalog_ready', convert_bool), - ('extra.', 'is_synchronized', convert_bool), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'size', convert_int), - ('extra.', 'configuration_naming_context', validate_to_none), - ('extra.', 'current_time', validate_to_none), - ('extra.', 'default_naming_context', validate_to_none), - ('extra.', 'ds_service_name', validate_to_none), - ('extra.', 'ldap_service_name', validate_to_none), - ('extra.', 'naming_contexts', validate_to_none), - ('extra.', 'root_domain_naming_context', validate_to_none), - ('extra.', 'schema_naming_context', validate_to_none), - ('extra.', 'server_name', validate_to_none), - ('extra.', 'subschema_subentry', validate_to_none), - ('extra.', 'supported_capabilities', validate_to_none), - ('extra.', 'supported_control', validate_to_none), - ('extra.', 'supported_ldap_policies', validate_to_none), - ('extra.', 'supported_ldap_version', validate_to_none), - ('extra.', 'supported_sasl_mechanisms', validate_to_none), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ldap', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-mdns-report/ -scan_mdns = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'mdns_name', validate_to_none), - ('extra.', 'mdns_ipv4', validate_to_none), - ('extra.', 'mdns_ipv6', validate_to_none), - ('extra.', 'services', validate_to_none), - ('extra.', 'workstation_name', validate_to_none), - ('extra.', 'workstation_ipv4', validate_to_none), - ('extra.', 'workstation_ipv6', validate_to_none), - ('extra.', 'workstation_info', validate_to_none), - ('extra.', 'http_name', validate_to_none), - ('extra.', 'http_ipv4', validate_to_none), - ('extra.', 'http_ipv6', validate_to_none), - ('extra.', 'http_ptr', validate_to_none), - ('extra.', 'http_info', validate_to_none), - ('extra.', 'http_target', validate_to_none), - ('extra.', 'http_port', convert_int), - ('extra.', 'spotify_name', validate_to_none), - ('extra.', 'spotify_ipv4', validate_to_none), - ('extra.', 'spotify_ipv6', validate_to_none), - ('extra.', 'opc_ua_discovery', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'mdns', - 'classification.identifier': 'open-mdns', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-memcached-report/ -scan_memcached = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'pid', convert_int), - ('extra.', 'pointer_size', convert_int), - ('extra.', 'uptime', convert_int), - ('extra.', 'curr_connections', convert_int), - ('extra.', 'total_connections', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'time', validate_to_none), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'memcached', - 'classification.identifier': 'open-memcached', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-mongodb-report/ -scan_mongodb = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'gitversion', validate_to_none), - ('extra.', 'sysinfo', validate_to_none), - ('extra.', 'opensslversion', validate_to_none), - ('extra.', 'allocator', validate_to_none), - ('extra.', 'javascriptengine', validate_to_none), - ('extra.', 'bits', validate_to_none), - ('extra.', 'maxbsonobjectsize', validate_to_none), - ('extra.', 'ok', convert_bool), - ('extra.', 'visible_databases', validate_to_none), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'mongodb', - 'classification.identifier': 'open-mongodb', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-mqtt-report/ -scan_mqtt = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'anonymous_access', convert_bool), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'raw_response', validate_to_none), - ('extra.', 'hex_code', validate_to_none), - ('extra.', 'code', validate_to_none), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serialNumber', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'mqtt', - 'classification.identifier': 'open-mqtt', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-mqtt-report/ -scan_mqtt_anon = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'raw_response', validate_to_none), - ('extra.', 'hex_code', validate_to_none), - ('extra.', 'code', validate_to_none), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serialNumber', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'mqtt', - 'classification.identifier': 'open-mqtt-anon', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-MSSQL -scan_mssql = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.local_hostname', 'server_name', validate_to_none), - ('extra.', 'tcp_port', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'instance_name', validate_to_none), - ('extra.', 'named_pipe', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'mssql', - 'classification.identifier': 'open-mssql', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-mysql-server-report/ -scan_mysql = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'client_can_handle_expired_passwords', convert_bool), - ('extra.', 'client_compress', convert_bool), - ('extra.', 'client_connect_attrs', convert_bool), - ('extra.', 'client_connect_with_db', convert_bool), - ('extra.', 'client_deprecated_eof', convert_bool), - ('extra.', 'client_found_rows', convert_bool), - ('extra.', 'client_ignore_sigpipe', convert_bool), - ('extra.', 'client_ignore_space', convert_bool), - ('extra.', 'client_interactive', convert_bool), - ('extra.', 'client_local_files', convert_bool), - ('extra.', 'client_long_flag', convert_bool), - ('extra.', 'client_long_password', convert_bool), - ('extra.', 'client_multi_results', convert_bool), - ('extra.', 'client_multi_statements', convert_bool), - ('extra.', 'client_no_schema', convert_bool), - ('extra.', 'client_odbc', convert_bool), - ('extra.', 'client_plugin_auth', convert_bool), - ('extra.', 'client_plugin_auth_len_enc_client_data', convert_bool), - ('extra.', 'client_protocol_41', convert_bool), - ('extra.', 'client_ps_multi_results', convert_bool), - ('extra.', 'client_reserved', convert_bool), - ('extra.', 'client_secure_connection', convert_bool), - ('extra.', 'client_session_track', convert_bool), - ('extra.', 'client_ssl', convert_bool), - ('extra.', 'client_transactions', convert_bool), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'mysql_protocol_version', validate_to_none), - ('extra.', 'server_version', validate_to_none), - ('extra.', 'error_code', validate_to_none), - ('extra.', 'error_id', validate_to_none), - ('extra.', 'error_message', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'browser_trusted', convert_bool), - ('extra.', 'browser_error', validate_to_none), - ('extra.', 'raw_cert', validate_to_none), - ('extra.', 'raw_cert_chain', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'mysql', - 'classification.identifier': 'open-mysql', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-NATPMP -scan_nat_pmp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'opcode', validate_to_none), - ('extra.', 'uptime', convert_int), - ('extra.', 'external_ip', validate_to_none), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'open-natpmp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'natpmp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-netbios-report/ -scan_netbios = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.account', 'username'), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'mac_address', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'workgroup', validate_to_none), - ('extra.', 'machine_name', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'open-netbios-nameservice', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'netbios-nameservice', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/netcore-netis-router-vulnerability-scan-report/ -scan_netis_router = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'response', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'open-netis', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.transport': 'udp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/ntp-version-report/ -scan_ntp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'clk_wander', convert_float), - ('extra.', 'frequency', convert_float), - ('extra.', 'jitter', convert_float), - ('extra.', 'leap', convert_float), - ('extra.', 'offset', convert_float), - ('extra.', 'peer', convert_int), - ('extra.', 'poll', convert_int), - ('extra.', 'precision', convert_int), - ('extra.', 'rootdelay', convert_float), - ('extra.', 'rootdispersion', convert_float), - ('extra.', 'stratum', convert_int), - ('extra.', 'tc', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'version', validate_to_none), - ('extra.', 'clock', validate_to_none), - ('extra.', 'error', validate_to_none), - ('extra.', 'mintc', validate_to_none), - ('extra.', 'noise', validate_to_none), - ('extra.', 'phase', validate_to_none), - ('extra.', 'processor', validate_to_none), - ('extra.', 'refid', validate_to_none), - ('extra.', 'reftime', validate_to_none), - ('extra.', 'stability', validate_to_none), - ('extra.', 'state', validate_to_none), - ('extra.', 'system', validate_to_none), - ('extra.', 'tai', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'ntp-version', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ntp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/ntp-monitor-report/ -scan_ntpmonitor = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'packets', convert_int), - ('extra.', 'size', convert_int), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'ntp-monitor', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ntp', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-Portmapper -scan_portmapper = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'programs', validate_to_none), - ('extra.', 'mountd_port', validate_to_none), - ('extra.', 'exports', validate_to_none), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'portmapper', - 'classification.identifier': 'open-portmapper', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-postgresql-server-report/ -scan_postgres = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'startup_error_line', convert_int), - ('extra.', 'client_ssl', convert_bool), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'supported_protocols', validate_to_none), - ('extra.', 'protocol_error_code', validate_to_none), - ('extra.', 'protocol_error_file', validate_to_none), - ('extra.', 'protocol_error_line', validate_to_none), - ('extra.', 'protocol_error_message', validate_to_none), - ('extra.', 'protocol_error_routine', validate_to_none), - ('extra.', 'protocol_error_severity', validate_to_none), - ('extra.', 'protocol_error_severity_v', validate_to_none), - ('extra.', 'startup_error_code', validate_to_none), - ('extra.', 'startup_error_file', validate_to_none), - ('extra.', 'startup_error_message', validate_to_none), - ('extra.', 'startup_error_routine', validate_to_none), - ('extra.', 'startup_error_severity', validate_to_none), - ('extra.', 'startup_error_severity_v', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'browser_trusted', convert_bool), - ('extra.', 'browser_error', validate_to_none), - ('extra.', 'raw_cert', validate_to_none), - ('extra.', 'raw_cert_chain', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'postgres', - 'classification.identifier': 'open-postgres', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-QOTD -scan_qotd = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'quote', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'qotd', - 'classification.identifier': 'open-qotd', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-quic-report/ -scan_quic = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'version_field_1', validate_to_none), - ('extra.', 'version_field_2', validate_to_none), - ('extra.', 'version_field_3', validate_to_none), - ('extra.', 'version_field_4', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'classification.identifier': 'open-quic', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-radmin-report/ -scan_radmin = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-radmin', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-rdp-report/ -scan_rdp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'cve20190708_vulnerable', convert_bool), - ('extra.', 'bluekeep_vulnerable', convert_bool), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'rdp_protocol', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'tlsv13_support', validate_to_none), - ('extra.', 'tlsv13_cipher', validate_to_none), - ('extra.', 'jarm', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'rdp', - 'protocol.transport': 'tcp', - 'classification.identifier': 'open-rdp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-ms-rdpeudp/ -scan_rdpeudp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sessionid', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-msrdpeudp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-Redis -scan_redis = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'git_sha1', validate_to_none), - ('extra.', 'git_dirty_flag', validate_to_none), - ('extra.', 'build_id', validate_to_none), - ('extra.', 'mode', validate_to_none), - ('extra.os.name', 'os', validate_to_none), - ('extra.', 'architecture', validate_to_none), - ('extra.', 'multiplexing_api', validate_to_none), - ('extra.', 'gcc_version', validate_to_none), - ('extra.', 'process_id', validate_to_none), - ('extra.', 'run_id', validate_to_none), - ('extra.', 'uptime', convert_int), - ('extra.', 'connected_clients', validate_to_none), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'redis', - 'classification.identifier': 'open-redis', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-rsync-report/ -scan_rsync = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'module', validate_to_none), - ('extra.', 'motd', validate_to_none), - ('extra.', 'has_password', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-rsync', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'rsync', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-sip-report/ -scan_sip = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'sip', validate_to_none), - ('extra.', 'sip_code', validate_to_none), - ('extra.', 'sip_reason', validate_to_none), - ('user_agent', 'user_agent', validate_to_none), - ('extra.', 'sip_via', validate_to_none), - ('extra.', 'sip_to', validate_to_none), - ('extra.', 'sip_from', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'content_type', validate_to_none), - ('extra.sip_server', 'server', validate_to_none), - ('extra.sip_contact', 'contact', validate_to_none), - ('extra.sip_cseq', 'cseq', validate_to_none), - ('extra.sip_call_id', 'call_id', validate_to_none), - ('extra.sip_allow', 'allow', validate_to_none), - ('extra.', 'amplification', convert_float), - ('extra.', 'response_size', convert_int), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'sip', - 'classification.identifier': 'open-sip', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-slp-service-report/ -scan_slp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'function', validate_to_none), - ('extra.', 'function_text', validate_to_none), - ('extra.', 'flags', validate_to_none), - ('extra.', 'next_extension_offset', validate_to_none), - ('extra.', 'xid', validate_to_none), - ('extra.', 'language_tag_length', validate_to_none), - ('extra.', 'language_tag', validate_to_none), - ('extra.', 'error_code', validate_to_none), - ('extra.', 'error_code_text', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'raw_response', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'slp', - 'classification.identifier': 'open-slp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-smb-report/ -scan_smb = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'smb_implant', convert_bool), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'arch', validate_to_none), - ('extra.', 'key', validate_to_none), - ('extra.', 'smbv1_support', validate_to_none), - ('extra.', 'smb_major_number', validate_to_none), - ('extra.', 'smb_minor_number', validate_to_none), - ('extra.', 'smb_revision', validate_to_none), - ('extra.', 'smb_version_string', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'smb', - 'protocol.transport': 'tcp', - 'classification.identifier': 'open-smb', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-smtp-report/ -scan_smtp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'banner', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'smtp', - 'classification.identifier': 'open-smtp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/vulnerable-smtp-report/ -scan_smtp_vulnerable = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'banner', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'smtp', - 'classification.identifier': 'vulnerable-smtp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-snmp-report/ -scan_snmp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'sysdesc', validate_to_none), - ('extra.', 'sysname', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'version', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_sector', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('extra.', 'community', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'snmp', - 'classification.identifier': 'open-snmp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-socks4-5-proxy-report/ -scan_socks = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.application', 'tag'), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-socks', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-SSDP -scan_ssdp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'header', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'systime', validate_to_none), - ('extra.', 'cache_control', validate_to_none), - ('extra.', 'location', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'search_target', validate_to_none), - ('extra.', 'unique_service_name', validate_to_none), - ('extra.', 'host', validate_to_none), - ('extra.', 'nts', validate_to_none), - ('extra.', 'nt', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'server_port', validate_to_none), - ('extra.', 'instance', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'updated_at', validate_to_none), - ('extra.', 'resource_identifier', validate_to_none), - ('extra.', 'amplification', convert_float), - ('extra.', 'response_size', convert_int), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ssdp', - 'classification.identifier': 'open-ssdp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-ssh-report/ -scan_ssh = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'serverid_raw', validate_to_none), - ('extra.', 'serverid_version', validate_to_none), - ('extra.', 'serverid_software', validate_to_none), - ('extra.', 'serverid_comment', validate_to_none), - ('extra.', 'server_cookie', validate_to_none), - ('extra.', 'available_kex', validate_to_none), - ('extra.', 'available_ciphers', validate_to_none), - ('extra.', 'available_mac', validate_to_none), - ('extra.', 'available_compression', validate_to_none), - ('extra.', 'selected_kex', validate_to_none), - ('extra.', 'algorithm', validate_to_none), - ('extra.', 'selected_cipher', validate_to_none), - ('extra.', 'selected_mac', validate_to_none), - ('extra.', 'selected_compression', validate_to_none), - ('extra.', 'server_signature_value', validate_to_none), - ('extra.', 'server_signature_raw', validate_to_none), - ('extra.', 'server_host_key', validate_to_none), - ('extra.', 'server_host_key_sha256', validate_to_none), - ('extra.', 'rsa_prime', validate_to_none), - ('extra.', 'rsa_prime_length', validate_to_none), - ('extra.', 'rsa_generator', validate_to_none), - ('extra.', 'rsa_generator_length', validate_to_none), - ('extra.', 'rsa_public_key', validate_to_none), - ('extra.', 'rsa_public_key_length', validate_to_none), - ('extra.', 'rsa_exponent', validate_to_none), - ('extra.', 'rsa_modulus', validate_to_none), - ('extra.', 'rsa_length', validate_to_none), - ('extra.', 'dss_prime', validate_to_none), - ('extra.', 'dss_prime_length', validate_to_none), - ('extra.', 'dss_generator', validate_to_none), - ('extra.', 'dss_generator_length', validate_to_none), - ('extra.', 'dss_public_key', validate_to_none), - ('extra.', 'dss_public_key_length', validate_to_none), - ('extra.', 'dss_dsa_public_g', validate_to_none), - ('extra.', 'dss_dsa_public_p', validate_to_none), - ('extra.', 'dss_dsa_public_q', validate_to_none), - ('extra.', 'dss_dsa_public_y', validate_to_none), - ('extra.', 'ecdsa_curve25519', validate_to_none), - ('extra.', 'ecdsa_curve', validate_to_none), - ('extra.', 'ecdsa_public_key_length', validate_to_none), - ('extra.', 'ecdsa_public_key_b', validate_to_none), - ('extra.', 'ecdsa_public_key_gx', validate_to_none), - ('extra.', 'ecdsa_public_key_gy', validate_to_none), - ('extra.', 'ecdsa_public_key_n', validate_to_none), - ('extra.', 'ecdsa_public_key_p', validate_to_none), - ('extra.', 'ecdsa_public_key_x', validate_to_none), - ('extra.', 'ecdsa_public_key_y', validate_to_none), - ('extra.', 'ed25519_curve25519', validate_to_none), - ('extra.', 'ed25519_cert_public_key_nonce', validate_to_none), - ('extra.', 'ed25519_cert_public_key_bytes', validate_to_none), - ('extra.', 'ed25519_cert_public_key_raw', validate_to_none), - ('extra.', 'ed25519_cert_public_key_sha256', validate_to_none), - ('extra.', 'ed25519_cert_public_key_serial', validate_to_none), - ('extra.', 'ed25519_cert_public_key_type_id', validate_to_none), - ('extra.', 'ed25519_cert_public_key_type_name', validate_to_none), - ('extra.', 'ed25519_cert_public_key_keyid', validate_to_none), - ('extra.', 'ed25519_cert_public_key_principles', validate_to_none), - ('extra.', 'ed25519_cert_public_key_valid_after', validate_to_none), - ('extra.', 'ed25519_cert_public_key_valid_before', validate_to_none), - ('extra.', 'ed25519_cert_public_key_duration', validate_to_none), - ('extra.', 'ed25519_cert_public_key_sigkey_bytes', validate_to_none), - ('extra.', 'ed25519_cert_public_key_sigkey_raw', validate_to_none), - ('extra.', 'ed25519_cert_public_key_sigkey_sha256', validate_to_none), - ('extra.', 'ed25519_cert_public_key_sigkey_value', validate_to_none), - ('extra.', 'ed25519_cert_public_key_sig_raw', validate_to_none), - ('extra.', 'banner', validate_to_none), - ('extra.', 'userauth_methods', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'classification.identifier': 'open-ssh', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-ssl-report/ -scan_ssl = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'ssl_poodle', convert_bool), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'freak_vulnerable', convert_bool), - ('extra.', 'freak_cipher_suite', validate_to_none), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'http_response_type', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'http_connection', validate_to_none), - ('extra.', 'www_authenticate', validate_to_none), - ('extra.', 'set_cookie', validate_to_none), - ('extra.', 'server_type', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'browser_trusted', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'browser_error', validate_to_none), - ('extra.', 'tlsv13_support', validate_to_none), - ('extra.', 'tlsv13_cipher', validate_to_none), - ('extra.', 'jarm', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'https', - 'classification.identifier': 'open-ssl', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Ssl-Freak-Scan -scan_ssl_freak = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'freak_vulnerable', convert_bool), - ('extra.', 'freak_cipher_suite', validate_to_none), - ('extra.', 'sector', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'http_response_type', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'http_connection', validate_to_none), - ('extra.', 'www_authenticate', validate_to_none), - ('extra.', 'set_cookie', validate_to_none), - ('extra.', 'server_type', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'browser_trusted', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'browser_error', validate_to_none), - ('extra.', 'tlsv13_support', validate_to_none), - ('extra.', 'tlsv13_cipher', validate_to_none), - ('extra.', 'raw_cert', validate_to_none), - ('extra.', 'raw_cert_chain', validate_to_none), - ('extra.', 'jarm', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_sector', validate_to_none), - ('extra.', 'page_sha256fp', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'https', - 'classification.identifier': 'ssl-freak', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Ssl-Scan -scan_ssl_poodle = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'ssl_poodle', convert_bool), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'http_response_type', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'http_connection', validate_to_none), - ('extra.', 'www_authenticate', validate_to_none), - ('extra.', 'set_cookie', validate_to_none), - ('extra.', 'server_type', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'browser_trusted', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'browser_error', validate_to_none), - ('extra.', 'tlsv13_support', validate_to_none), - ('extra.', 'tlsv13_cipher', validate_to_none), - ('extra.', 'raw_cert', validate_to_none), - ('extra.', 'raw_cert_chain', validate_to_none), - ('extra.', 'jarm', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_sector', validate_to_none), - ('extra.', 'page_sha256fp', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'https', - 'classification.identifier': 'ssl-poodle', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-stun-service-report/ -scan_stun = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'mapped_port', convert_int), - ('extra.', 'xor_mapped_port', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'transaction_id', validate_to_none), - ('extra.', 'magic_cookie', validate_to_none), - ('extra.', 'message_length', convert_int), - ('extra.', 'message_type', validate_to_none), - ('extra.', 'mapped_family', validate_to_none), - ('extra.', 'mapped_address', validate_to_none), - ('extra.', 'xor_mapped_family', validate_to_none), - ('extra.', 'xor_mapped_address', validate_to_none), - ('extra.', 'software', validate_to_none), - ('extra.', 'fingerprint', validate_to_none), - ('extra.', 'amplification', convert_float), - ('extra.', 'response_size', convert_int), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'Session Traversal Utilities for NAT', - 'classification.identifier': 'open-stun', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/synful-scan-report/ -scan_synfulknock = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'ack_number', convert_int), - ('extra.', 'window_size', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'sequence_number', validate_to_none), - ('extra.', 'urgent_pointer', validate_to_none), - ('extra.', 'tcp_flags', validate_to_none), - ('extra.', 'raw_packet', validate_to_none), - ('extra.source.sector', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-synfulknock', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-telnet-report/ -scan_telnet = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'banner', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'telnet', - 'classification.identifier': 'open-telnet', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-TFTP -scan_tftp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'opcode', validate_to_none), - ('extra.', 'errorcode', validate_to_none), - ('extra.', 'error', validate_to_none), - ('extra.', 'errormessage', validate_to_none), - ('extra.', 'size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'tftp', - 'classification.identifier': 'open-tftp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-ubiquiti-report/ -scan_ubiquiti = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.mac_address', 'mac', validate_to_none), - ('extra.radio_name', 'radioname', validate_to_none), - ('extra.model', 'modelshort', validate_to_none), - ('extra.model_full', 'modelfull', validate_to_none), - ('extra.firmwarerev', 'firmware', validate_to_none), - ('extra.response_size', 'size', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'essid', validate_to_none), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-ubiquiti-discovery-service', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-vnc-report/ -scan_vnc = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.reverse_dns', 'hostname'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'product', validate_to_none), - ('extra.', 'banner', validate_to_none), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'vnc', - 'protocol.transport': 'tcp', - 'classification.identifier': 'open-vnc', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-ws-discovery-service-report/ -scan_ws_discovery = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ('extra.', 'error', validate_to_none), - ('extra.', 'raw_response', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ws-discovery', - 'classification.identifier': 'open-ws-discovery', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-xdmcp-service-report/ -scan_xdmcp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'opcode', validate_to_none), - ('extra.', 'reported_hostname', validate_to_none), - ('extra.', 'status', validate_to_none), - ('extra.', 'size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'xdmcp', - 'classification.identifier': 'open-xdmcp', - }, -} - -# http://www.shadowserver.org/wiki/pmwiki.php/Services/Spam-URL -spam_url = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ], - 'optional_fields': [ - ('source.url', 'url', convert_http_host_and_url, True), - ('source.fqdn', 'host', validate_fqdn), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'source', validate_to_none), - ('extra.', 'sender', validate_to_none), - ('extra.', 'subject', validate_to_none), - ('malware.hash.md5', 'md5', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'abusive-content', - 'classification.type': 'spam', - 'classification.identifier': 'spam-url', - }, -} - -special = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('source.reverse_dns', 'hostname'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('malware.name', 'tag'), - ('extra.', 'public_source', validate_to_none), - ('extra.', 'status', validate_to_none), - ('extra.', 'detail', validate_to_none), - ('extra.', 'method', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'special', - }, -} - -mapping = ( - # feed name, file name, function - ('Blocklist', 'blocklist', blocklist), - ('Compromised-Website', 'compromised_website', compromised_website), - ('Device-Identification IPv4', 'device_id', device_id), - ('Device-Identification IPv6', 'device_id6', device_id), - ('DDoS-Participant', 'event4_ddos_participant', event_ddos_participant), - ('Honeypot-Brute-Force-Events', 'event4_honeypot_brute_force', event_honeypot_brute_force), - ('Honeypot-Darknet', 'event4_honeypot_darknet', event_honeypot_darknet), - ('Honeypot-DDoS', 'event4_honeypot_ddos', event_honeypot_ddos), - ('Honeypot-Amplification-DDoS-Events', 'event4_honeypot_ddos_amp', event_honeypot_ddos_amp), - ('Honeypot-DDoS-Target', 'event4_honeypot_ddos_target', event_honeypot_ddos_target), - ('Honeypot-HTTP-Scan', 'event4_honeypot_http_scan', event_honeypot_http_scan), - ('Honeypot-ICS-Scanner', 'event4_honeypot_ics_scan', event_honeypot_ics_scan), - ('IP-Spoofer-Events', 'event4_ip_spoofer', event_ip_spoofer), - ('Microsoft-Sinkhole-Events IPv4', 'event4_microsoft_sinkhole', event_sinkhole), - ('Microsoft-Sinkhole-Events-HTTP IPv4', 'event4_microsoft_sinkhole_http', event_sinkhole_http), - ('Sinkhole-Events IPv4', 'event4_sinkhole', event_sinkhole), - ('Sinkhole-DNS', 'event4_sinkhole_dns', event_sinkhole_dns), - ('Sinkhole-Events-HTTP IPv4', 'event4_sinkhole_http', event_sinkhole_http), - ('Sinkhole-Events-HTTP-Referer IPv4', 'event4_sinkhole_http_referer', event_sinkhole_http_referer), - ('Sinkhole-Events IPv6', 'event6_sinkhole', event_sinkhole), - ('Sinkhole-Events-HTTP IPv6', 'event6_sinkhole_http', event_sinkhole_http), - ('Sinkhole-Events-HTTP-Referer IPv6', 'event6_sinkhole_http_referer', event_sinkhole_http_referer), - ('Malware-URL', 'malware_url', malware_url), - ('Phish-URL', 'phish_url', phish_url), - ('IPv6-Accessible-HTTP-Proxy', 'population6_http_proxy', population_http_proxy), - ('Accessible-HTTP-Proxy', 'population_http_proxy', population_http_proxy), - ('Sandbox-Connections', 'sandbox_conn', sandbox_conn), - ('Sandbox-DNS', 'sandbox_dns', sandbox_dns), - ('Sandbox-URL', 'sandbox_url', sandbox_url), - ('IPv6-Accessible-CWMP', 'scan6_cwmp', scan_cwmp), - ('IPv6-DNS-Open-Resolvers', 'scan6_dns', scan_dns), - ('IPv6-Vulnerable-Exchange', 'scan6_exchange', scan_exchange), - ('IPv6-Accessible-FTP', 'scan6_ftp', scan_ftp), - ('IPv6-Accessible-HTTP', 'scan6_http', scan_http), - ('IPv6-Open-HTTP-Proxy', 'scan6_http_proxy', scan_http_proxy), - ('IPv6-Vulnerable-HTTP', 'scan6_http_vulnerable', scan_http_vulnerable), - ('IPv6-Open-IPP', 'scan6_ipp', scan_ipp), - ('IPv6-Open-LDAP-TCP', 'scan6_ldap_tcp', scan_ldap_tcp), - ('IPv6-Open-MQTT', 'scan6_mqtt', scan_mqtt), - ('IPv6-Open-Anonymous-MQTT', 'scan6_mqtt_anon', scan_mqtt_anon), - ('IPv6-Accessible-MySQL', 'scan6_mysql', scan_mysql), - ('IPv6-NTP-Version', 'scan6_ntp', scan_ntp), - ('IPv6-NTP-Monitor', 'scan6_ntpmonitor', scan_ntpmonitor), - ('IPv6-Accessible-PostgreSQL', 'scan6_postgres', scan_postgres), - ('IPv6-Accessible-RDP', 'scan6_rdp', scan_rdp), - ('IPv6-Accessible-SLP', 'scan6_slp', scan_slp), - ('IPv6-Accessible-SMB', 'scan6_smb', scan_smb), - ('IPv6-Accessible-SMTP', 'scan6_smtp', scan_smtp), - ('IPv6-Vulnerable-SMTP', 'scan6_smtp_vulnerable', scan_smtp_vulnerable), - ('IPv6-Open-SNMP', 'scan6_snmp', scan_snmp), - ('IPv6-Accessible-SSH', 'scan6_ssh', scan_ssh), - ('IPv6-Accessible-SSL', 'scan6_ssl', scan_ssl), - ('SSL-FREAK-Vulnerable-Servers IPv6', 'scan6_ssl_freak', scan_ssl_freak), - ('SSL-POODLE-Vulnerable-Servers IPv6', 'scan6_ssl_poodle', scan_ssl_poodle), - ('IPv6-Accessible-Session-Traversal-Utilities-for-NAT', 'scan6_stun', scan_stun), - ('IPv6-Accessible-Telnet', 'scan6_telnet', scan_telnet), - ('IPv6-Accessible-VNC', 'scan6_vnc', scan_vnc), - ('Accessible-ADB', 'scan_adb', scan_adb), - ('Accessible-AFP', 'scan_afp', scan_afp), - ('Accessible-AMQP', 'scan_amqp', scan_amqp), - ('Accessible-ARD', 'scan_ard', scan_ard), - ('Open-Chargen', 'scan_chargen', scan_chargen), - ('Accessible-Cisco-Smart-Install', 'scan_cisco_smart_install', scan_cisco_smart_install), - ('Accessible-CoAP', 'scan_coap', scan_coap), - ('Accessible-CouchDB', 'scan_couchdb', scan_couchdb), - ('Accessible-CWMP', 'scan_cwmp', scan_cwmp), - ('Open-DB2-Discovery-Service', 'scan_db2', scan_db2), - ('Vulnerable-DDoS-Middlebox', 'scan_ddos_middlebox', scan_ddos_middlebox), - ('DNS-Open-Resolvers', 'scan_dns', scan_dns), - ('Accessible-Docker', 'scan_docker', scan_docker), - ('Accessible-DVR-DHCPDiscover', 'scan_dvr_dhcpdiscover', scan_dvr_dhcpdiscover), - ('Open-Elasticsearch', 'scan_elasticsearch', scan_elasticsearch), - ('Accessible-Erlang-Port-Mapper-Daemon', 'scan_epmd', scan_epmd), - ('Vulnerable-Exchange-Server', 'scan_exchange', scan_exchange), - ('Accessible-FTP', 'scan_ftp', scan_ftp), - ('Accessible-Hadoop', 'scan_hadoop', scan_hadoop), - ('Accessible-HTTP', 'scan_http', scan_http), - ('Open-HTTP-Proxy', 'scan_http_proxy', scan_http_proxy), - ('Vulnerable-HTTP', 'scan_http_vulnerable', scan_http_vulnerable), - ('Accessible-ICS', 'scan_ics', scan_ics), - ('Open-IPMI', 'scan_ipmi', scan_ipmi), - ('Open-IPP', 'scan_ipp', scan_ipp), - ('Vulnerable-ISAKMP', 'scan_isakmp', scan_isakmp), - ('Accessible-Kubernetes-API', 'scan_kubernetes', scan_kubernetes), - ('Open-LDAP-TCP', 'scan_ldap_tcp', scan_ldap_tcp), - ('Open-LDAP', 'scan_ldap_udp', scan_ldap_udp), - ('Open-mDNS', 'scan_mdns', scan_mdns), - ('Open-Memcached', 'scan_memcached', scan_memcached), - ('Open-MongoDB', 'scan_mongodb', scan_mongodb), - ('Open-MQTT', 'scan_mqtt', scan_mqtt), - ('Open-Anonymous-MQTT', 'scan_mqtt_anon', scan_mqtt_anon), - ('Open-MSSQL', 'scan_mssql', scan_mssql), - ('Accessible-MySQL', 'scan_mysql', scan_mysql), - ('Open-NATPMP', 'scan_nat_pmp', scan_nat_pmp), - ('Open-NetBIOS-Nameservice', 'scan_netbios', scan_netbios), - ('Open-Netis', 'scan_netis_router', scan_netis_router), - ('NTP-Version', 'scan_ntp', scan_ntp), - ('NTP-Monitor', 'scan_ntpmonitor', scan_ntpmonitor), - ('Open-Portmapper', 'scan_portmapper', scan_portmapper), - ('Accessible-PostgreSQL', 'scan_postgres', scan_postgres), - ('Open-QOTD', 'scan_qotd', scan_qotd), - ('Accessible-QUIC', 'scan_quic', scan_quic), - ('Accessible-Radmin', 'scan_radmin', scan_radmin), - ('Accessible-RDP', 'scan_rdp', scan_rdp), - ('Accessible-MS-RDPEUDP', 'scan_rdpeudp', scan_rdpeudp), - ('Open-Redis', 'scan_redis', scan_redis), - ('Accessible-Rsync', 'scan_rsync', scan_rsync), - ('Accessible-SIP', 'scan_sip', scan_sip), - ('Accessible-SLP', 'scan_slp', scan_slp), - ('Accessible-SMB', 'scan_smb', scan_smb), - ('Accessible-SMTP', 'scan_smtp', scan_smtp), - ('Vulnerable-SMTP', 'scan_smtp_vulnerable', scan_smtp_vulnerable), - ('Open-SNMP', 'scan_snmp', scan_snmp), - ('Accessible-SOCKS4/5-Proxy', 'scan_socks', scan_socks), - ('Open-SSDP', 'scan_ssdp', scan_ssdp), - ('Accessible-SSH', 'scan_ssh', scan_ssh), - ('Accessible-SSL', 'scan_ssl', scan_ssl), - ('SSL-FREAK-Vulnerable-Servers', 'scan_ssl_freak', scan_ssl_freak), - ('SSL-POODLE-Vulnerable-Servers IPv4', 'scan_ssl_poodle', scan_ssl_poodle), - ('Accessible-Session-Traversal-Utilities-for-NAT', 'scan_stun', scan_stun), - ('SYNful-Knock', 'scan_synfulknock', scan_synfulknock), - ('Accessible-Telnet', 'scan_telnet', scan_telnet), - ('Open-TFTP', 'scan_tftp', scan_tftp), - ('Accessible-Ubiquiti-Discovery-Service', 'scan_ubiquiti', scan_ubiquiti), - ('Accessible-VNC', 'scan_vnc', scan_vnc), - ('Accessible-WS-Discovery-Service', 'scan_ws_discovery', scan_ws_discovery), - ('Open-XDMCP', 'scan_xdmcp', scan_xdmcp), - ('Spam-URL', 'spam_url', spam_url), - ('Special', 'special', special), - ('Accessible-RDPEUDP', 'scan_rdpeudp', scan_rdpeudp), - ('Sinkhole-Events', 'event4_sinkhole', event_sinkhole), - ('Sinkhole-Events-HTTP', 'event4_sinkhole_http', event_sinkhole_http), - ('Sinkhole-Events-HTTP-Referer', 'event4_sinkhole_http_referer', event_sinkhole_http_referer), -) -# END CONFGEN + try: + with open(tmp) as fh: + schema = json.load(fh) + except: + # leave tempfile behind for diagnosis + raise ValueError("Failed to validate %r" % tmp) -feedname_mapping = {feedname: function for feedname, filename, function in mapping} -filename_mapping = {filename: (feedname, function) for feedname, filename, function in mapping} + os.replace(tmp, __config.schema_file) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index 70ba3b4bb6..f14549141a 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -22,6 +22,7 @@ """ import copy import re +import os from intelmq.lib.bot import ParserBot from intelmq.lib.exceptions import InvalidKey, InvalidValue @@ -29,7 +30,13 @@ class ShadowserverParserBot(ParserBot): - """Parse all ShadowServer feeds""" + """ + Parse all ShadowServer feeds + + Parameters: + schema_file (str): Path to the report schema file + + """ recover_line = ParserBot.recover_line_csv_dict _csv_params = {'dialect': 'unix'} @@ -124,10 +131,17 @@ def parse_line(self, row, report): value = raw_value if conv_func is not None and raw_value is not None: - if len(item) == 4 and item[3]: - value = conv_func(raw_value, row) - else: - value = conv_func(raw_value) + try: + if len(item) == 4 and item[3]: + value = config.functions[conv_func](raw_value, row) + else: + value = config.functions[conv_func](raw_value) + except Exception: + """ fail early and often in this case. We want to be able to convert everything """ + self.logger.error('Could not convert shadowkey: %r in feed %r, ' + 'value: %r via conversion function %r.', + shadowkey, self.feedname, raw_value, conv_func) + raise if value is not None: event.add(intelmqkey, value) @@ -153,17 +167,17 @@ def parse_line(self, row, report): value = raw_value if conv_func is not None and raw_value is not None: - if len(item) == 4 and item[3]: - value = conv_func(raw_value, row) - else: - try: - value = conv_func(raw_value) - except Exception: - """ fail early and often in this case. We want to be able to convert everything """ - self.logger.error('Could not convert shadowkey: %r in feed %r, ' - 'value: %r via conversion function %r.', - shadowkey, self.feedname, raw_value, conv_func.__name__) - raise + try: + if len(item) == 4 and item[3]: + value = config.functions[conv_func](raw_value, row) + else: + value = config.functions[conv_func](raw_value) + except Exception: + """ fail early and often in this case. We want to be able to convert everything """ + self.logger.error('Could not convert shadowkey: %r in feed %r, ' + 'value: %r via conversion function %r.', + shadowkey, self.feedname, raw_value, conv_func) + raise if value is not None: if intelmqkey == 'extra.': diff --git a/intelmq/bots/parsers/shadowserver/schema.json.test b/intelmq/bots/parsers/shadowserver/schema.json.test new file mode 100644 index 0000000000..2cfb8bb1d3 --- /dev/null +++ b/intelmq/bots/parsers/shadowserver/schema.json.test @@ -0,0 +1,180 @@ +{ + "test_smb" : { + "constant_fields" : { + "classification.identifier" : "test-smb", + "classification.taxonomy" : "vulnerable", + "classification.type" : "vulnerable-system", + "protocol.application" : "smb", + "protocol.transport" : "tcp" + }, + "feed_name" : "Test-Accessible-SMB", + "file_name" : "test_smb", + "optional_fields" : [ + [ + "extra.", + "smb_implant", + "convert_bool" + ], + [ + "source.reverse_dns", + "hostname" + ], + [ + "extra.", + "tag" + ], + [ + "source.asn", + "asn", + "invalidate_zero" + ], + [ + "source.geolocation.cc", + "geo" + ], + [ + "source.geolocation.region", + "region" + ], + [ + "source.geolocation.city", + "city" + ], + [ + "extra.source.naics", + "naics", + "invalidate_zero" + ], + [ + "extra.source.sic", + "sic", + "invalidate_zero" + ], + [ + "extra.", + "arch", + "validate_to_none" + ], + [ + "extra.", + "key", + "validate_to_none" + ], + [ + "extra.", + "smbv1_support", + "validate_to_none" + ], + [ + "extra.", + "smb_major_number", + "validate_to_none" + ], + [ + "extra.", + "smb_minor_number", + "validate_to_none" + ], + [ + "extra.", + "smb_revision", + "validate_to_none" + ], + [ + "extra.", + "smb_version_string", + "validate_to_none" + ] + ], + "required_fields" : [ + [ + "time.source", + "timestamp", + "add_UTC_to_timestamp" + ], + [ + "source.ip", + "ip", + "validate_ip" + ], + [ + "source.port", + "port", + "convert_int" + ] + ] + }, + "test_telnet" : { + "constant_fields" : { + "classification.identifier" : "test-telnet", + "classification.taxonomy" : "vulnerable", + "classification.type" : "vulnerable-system", + "protocol.application" : "telnet" + }, + "feed_name" : "Test-Accessible-Telnet", + "file_name" : "test_telnet", + "optional_fields" : [ + [ + "protocol.transport", + "protocol" + ], + [ + "source.reverse_dns", + "hostname" + ], + [ + "extra.", + "tag", + "validate_to_none" + ], + [ + "source.asn", + "asn", + "invalidate_zero" + ], + [ + "source.geolocation.cc", + "geo" + ], + [ + "source.geolocation.region", + "region" + ], + [ + "source.geolocation.city", + "city" + ], + [ + "extra.", + "naics", + "invalidate_zero" + ], + [ "extra.", + "sic", + "invalidate_zero" + ], + [ + "extra.", + "banner", + "validate_to_none" + ] + ], + "required_fields" : [ + [ + "time.source", + "timestamp", + "add_UTC_to_timestamp" + ], + [ + "source.ip", + "ip", + "validate_ip" + ], + [ + "source.port", + "port", + "convert_int" + ] + ] + } +} diff --git a/intelmq/bots/parsers/shadowserver/update_schema.py b/intelmq/bots/parsers/shadowserver/update_schema.py new file mode 100644 index 0000000000..040f672593 --- /dev/null +++ b/intelmq/bots/parsers/shadowserver/update_schema.py @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: 2023 The Shadowserver Foundation +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +# -*- coding: utf-8 -*- + +import os +import intelmq.bots.parsers.shadowserver._config as config + +if __name__ == '__main__': # pragma: no cover + exec(open(os.path.join(os.path.dirname(__file__), '../../../version.py')).read()) # defines __version__ + config.update_schema(__version__) From b917a9484776cba1cc472b598748067a4821f52d Mon Sep 17 00:00:00 2001 From: elsif2 Date: Wed, 12 Apr 2023 00:01:32 +0000 Subject: [PATCH 24/76] revised tests --- .../bots/parsers/shadowserver/test_broken.py | 12 +- .../bots/parsers/shadowserver/test_mapping.py | 8 +- .../parsers/shadowserver/test_parameters.py | 37 +++--- .../parsers/shadowserver/test_report_smb.py | 124 ++++++++++++++++++ .../shadowserver/test_report_switch.py | 16 +-- .../shadowserver/test_report_telnet.py | 87 ++++++++++++ .../shadowserver/testdata/test_smb.csv | 4 + .../testdata/test_smb.csv.license | 2 + .../shadowserver/testdata/test_telnet.csv | 3 + .../testdata/test_telnet.csv.license | 2 + 10 files changed, 260 insertions(+), 35 deletions(-) create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_report_smb.py create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv.license create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv.license diff --git a/intelmq/tests/bots/parsers/shadowserver/test_broken.py b/intelmq/tests/bots/parsers/shadowserver/test_broken.py index 472dd0b90c..2b803142eb 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_broken.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_broken.py @@ -13,12 +13,12 @@ REPORT1 = {"raw": utils.base64_encode('adasdasdasdasd\nadasdasdafgf'), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_http-test-test.csv", + "extra.file_name": "2019-01-01-test_smb-test-test.csv", } REPORT2 = {"raw": utils.base64_encode('timestamp,ip,port\n2018-08-01T00:00:00+00,127.0.0.1,80'), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ftp-test-test.csv", + "extra.file_name": "2019-01-01-test_telnet-test-test.csv", } REPORT3 = {"raw": utils.base64_encode('adasdasdasdasd\nadasdasdafgf'), "__type": "Report", @@ -48,10 +48,10 @@ def test_broken(self): """ self.input_message = REPORT1 self.run_bot(allowed_error_count=1) - self.assertLogMatches(pattern="Detected report's file name: 'scan_http'.", + self.assertLogMatches(pattern="Detected report's file name: 'test_smb'.", levelname="DEBUG") self.assertLogMatches(pattern="Failed to parse line.") - self.assertLogMatches(pattern="ValueError: Required column 'timestamp' not found in feed 'Accessible-HTTP'. Possible change in data format or misconfiguration.") + self.assertLogMatches(pattern="ValueError: Required column 'timestamp' not found in feed 'Test-Accessible-SMB'. Possible change in data format or misconfiguration.") self.assertLogMatches(pattern=r"Sent 0 events and found 1 problem\(s\)\.", levelname="INFO") @@ -61,9 +61,9 @@ def test_half_broken(self): """ self.input_message = REPORT2 self.run_bot(allowed_warning_count=63) - self.assertLogMatches(pattern="Detected report's file name: 'scan_ftp'.", + self.assertLogMatches(pattern="Detected report's file name: 'test_telnet'.", levelname="DEBUG") - self.assertLogMatches(pattern="Optional key 'jarm' not found in feed 'Accessible-FTP'.", + self.assertLogMatches(pattern="Optional key 'banner' not found in feed 'Test-Accessible-Telnet'.", levelname="WARNING") self.assertLogMatches(pattern=r"Sent 1 events and found 0 problem\(s\)\.", levelname="INFO") diff --git a/intelmq/tests/bots/parsers/shadowserver/test_mapping.py b/intelmq/tests/bots/parsers/shadowserver/test_mapping.py index f58aed66eb..6a2af94475 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_mapping.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_mapping.py @@ -11,22 +11,22 @@ with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_telnet.csv')) as handle: + 'testdata/test_telnet.csv')) as handle: TELNET_FILE = handle.read() EXAMPLE_TELNET = { "raw": utils.base64_encode(TELNET_FILE), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_telnet.csv", + "extra.file_name": "2019-01-01-test_telnet.csv", } with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_vnc.csv')) as handle: + 'testdata/test_smb.csv')) as handle: TELNET_FILE = handle.read() EXAMPLE_VNC = { "raw": utils.base64_encode(TELNET_FILE), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_vnc.csv", + "extra.file_name": "2019-01-01-test_smb.csv", } diff --git a/intelmq/tests/bots/parsers/shadowserver/test_parameters.py b/intelmq/tests/bots/parsers/shadowserver/test_parameters.py index a5ea81f199..677cd0319b 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_parameters.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_parameters.py @@ -12,38 +12,41 @@ from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_dns.csv')) as handle: + 'testdata/test_smb.csv')) as handle: EXAMPLE_FILE = handle.read() EXAMPLE_LINES = EXAMPLE_FILE.splitlines() EXAMPLE_REPORT = {"raw": utils.base64_encode(EXAMPLE_FILE), "__type": "Report", "time.observation": "2018-07-30T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_dns-test-test.csv", + "extra.file_name": "2019-01-01-test_smb-test-test.csv", 'feed.name': 'report feedname', } EVENTS = [{ '__type': 'Event', 'feed.name': 'report feedname', - "classification.identifier": "dns-open-resolver", + "classification.identifier": 'test-smb', "classification.taxonomy": "vulnerable", "classification.type": "vulnerable-system", - "extra.dns_version": "dnsmasq-2.66", - "extra.min_amplification": 4.619, - "extra.tag": "openresolver", - "protocol.application": "dns", - "protocol.transport": "udp", + "extra.smb_implant": False, + "extra.smb_major_number": '2', + "extra.smb_minor_number": '1', + "extra.smb_version_string": 'SMB 2.1', + "extra.smbv1_support": 'N', + "extra.tag": "smb", + "protocol.application": "smb", + "protocol.transport": "tcp", 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - "source.asn": 25255, - "source.geolocation.cc": "AT", - "source.geolocation.city": "VIENNA", - "source.geolocation.region": "WIEN", - "source.ip": "198.51.100.179", - "source.port": 53, - "source.reverse_dns": "198-51-100-189.example.net", + "source.asn": 64512, + "source.geolocation.cc": "ZZ", + "source.geolocation.city": "City", + "source.geolocation.region": "Region", + "source.ip": "192.168.0.1", + "source.port": 445, + "source.reverse_dns": "node01.example.com", "time.observation": "2018-07-30T00:00:00+00:00", - "time.source": "2018-04-14T00:14:34+00:00" + "time.source": "2010-02-10T00:00:00+00:00" }, ] @@ -70,7 +73,7 @@ def test_overwrite_feed_name(self): self.run_bot(prepare=False) for i, EVENT in enumerate(EVENTS): event = EVENT.copy() - event['feed.name'] = 'DNS-Open-Resolvers' + event['feed.name'] = 'Test-Accessible-SMB' self.assertMessageEqual(i, event) diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py b/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py new file mode 100644 index 0000000000..c7eefdf0a9 --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py @@ -0,0 +1,124 @@ +# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +# -*- coding: utf-8 -*- + +import os +import unittest + +import intelmq.lib.test as test +import intelmq.lib.utils as utils +from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot + +with open(os.path.join(os.path.dirname(__file__), + 'testdata/test_smb.csv')) as handle: + EXAMPLE_FILE = handle.read() +EXAMPLE_LINES = EXAMPLE_FILE.splitlines() + +EXAMPLE_REPORT = {'feed.name': 'Test-Accessible-SMB', + "raw": utils.base64_encode(EXAMPLE_FILE), + "__type": "Report", + "time.observation": "2015-01-01T00:00:00+00:00", + "extra.file_name": "2019-01-01-test_smb-test-geo.csv", + } +EVENTS = [ +{ + '__type' : 'Event', + 'classification.identifier' : 'test-smb', + 'classification.taxonomy' : 'vulnerable', + 'classification.type' : 'vulnerable-system', + 'extra.smb_implant' : False, + 'extra.smb_major_number' : '2', + 'extra.smb_minor_number' : '1', + 'extra.smb_version_string' : 'SMB 2.1', + 'extra.smbv1_support' : 'N', + 'extra.tag' : 'smb', + 'feed.name' : 'Test-Accessible-SMB', + 'protocol.application' : 'smb', + 'protocol.transport' : 'tcp', + 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), + 'source.asn' : 64512, + 'source.geolocation.cc' : 'ZZ', + 'source.geolocation.city' : 'City', + 'source.geolocation.region' : 'Region', + 'source.ip' : '192.168.0.1', + 'source.port' : 445, + 'source.reverse_dns' : 'node01.example.com', + 'time.observation' : '2015-01-01T00:00:00+00:00', + 'time.source' : '2010-02-10T00:00:00+00:00' +}, + +{ + '__type' : 'Event', + 'classification.identifier' : 'test-smb', + 'classification.taxonomy' : 'vulnerable', + 'classification.type' : 'vulnerable-system', + 'extra.smb_implant' : False, + 'extra.smb_major_number' : '2', + 'extra.smb_minor_number' : '1', + 'extra.smb_version_string' : 'SMB 2.1', + 'extra.smbv1_support' : 'N', + 'extra.tag' : 'smb', + 'feed.name' : 'Test-Accessible-SMB', + 'protocol.application' : 'smb', + 'protocol.transport' : 'tcp', + 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), + 'source.asn' : 64512, + 'source.geolocation.cc' : 'ZZ', + 'source.geolocation.city' : 'City', + 'source.geolocation.region' : 'Region', + 'source.ip' : '192.168.0.2', + 'source.port' : 445, + 'source.reverse_dns' : 'node02.example.com', + 'time.observation' : '2015-01-01T00:00:00+00:00', + 'time.source' : '2010-02-10T00:00:01+00:00' +}, + +{ + '__type' : 'Event', + 'classification.identifier' : 'test-smb', + 'classification.taxonomy' : 'vulnerable', + 'classification.type' : 'vulnerable-system', + 'extra.smb_implant' : False, + 'extra.smb_major_number' : '2', + 'extra.smb_minor_number' : '1', + 'extra.smb_version_string' : 'SMB 2.1', + 'extra.smbv1_support' : 'N', + 'extra.tag' : 'smb', + 'feed.name' : 'Test-Accessible-SMB', + 'protocol.application' : 'smb', + 'protocol.transport' : 'tcp', + 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), + 'source.asn' : 64512, + 'source.geolocation.cc' : 'ZZ', + 'source.geolocation.city' : 'City', + 'source.geolocation.region' : 'Region', + 'source.ip' : '192.168.0.3', + 'source.port' : 445, + 'source.reverse_dns' : 'node03.example.com', + 'time.observation' : '2015-01-01T00:00:00+00:00', + 'time.source' : '2010-02-10T00:00:02+00:00' +} + ] + + +class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): + """ + A TestCase for a ShadowserverParserBot. + """ + + @classmethod + def set_bot(cls): + cls.bot_reference = ShadowserverParserBot + cls.default_input_message = EXAMPLE_REPORT + + def test_event(self): + """ Test if correct Event has been produced. """ + self.run_bot() + for i, EVENT in enumerate(EVENTS): + self.assertMessageEqual(i, EVENT) + + +if __name__ == '__main__': # pragma: no cover + unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py b/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py index 0a34a69f0a..570d612fb4 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py @@ -12,24 +12,24 @@ from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ftp.csv')) as handle: + 'testdata/test_smb.csv')) as handle: EXAMPLE_LINES = handle.read().splitlines()[:2] -FIRST_REPORT = {'feed.name': 'Accessible FTP', +FIRST_REPORT = {'feed.name': 'Test-Accessible-SMB', "raw": utils.base64_encode('\n'.join(EXAMPLE_LINES)), "__type": "Report", "time.observation": "2019-03-25T00:00:00+00:00", - "extra.file_name": "2019-03-25-scan_ftp-test-test.csv", + "extra.file_name": "2019-03-25-test_smb-test-test.csv", } -with open(os.path.join(os.path.dirname(__file__), 'testdata/blocklist.csv')) as handle: +with open(os.path.join(os.path.dirname(__file__), 'testdata/test_telnet.csv')) as handle: EXAMPLE_LINES = handle.read().splitlines()[:2] SECOND_REPORT = { - 'feed.name': 'Blocklist', + 'feed.name': 'Test-Accessible-Telnet', "raw": utils.base64_encode('\n'.join(EXAMPLE_LINES)), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-blocklist-test-geo.csv", + "extra.file_name": "2019-01-01-test_telnet-test-geo.csv", } @@ -48,9 +48,9 @@ def test_event(self): """ Test if the parser correctly detects and handles different report types. """ self.input_message = [FIRST_REPORT, SECOND_REPORT] self.run_bot(iterations=2) - self.assertLogMatches("Detected report's file name: 'scan_ftp'", + self.assertLogMatches("Detected report's file name: 'test_smb'", levelname='DEBUG') - self.assertLogMatches("Detected report's file name: 'blocklist'", + self.assertLogMatches("Detected report's file name: 'test_telnet'", levelname='DEBUG') diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py b/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py new file mode 100644 index 0000000000..6d539ac4a7 --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py @@ -0,0 +1,87 @@ +# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +# -*- coding: utf-8 -*- + +import os +import unittest + +import intelmq.lib.test as test +import intelmq.lib.utils as utils +from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot + +with open(os.path.join(os.path.dirname(__file__), + 'testdata/test_telnet.csv')) as handle: + EXAMPLE_FILE = handle.read() +EXAMPLE_LINES = EXAMPLE_FILE.splitlines() + +EXAMPLE_REPORT = {'feed.name': 'Test-Accessible-Telnet', + "raw": utils.base64_encode(EXAMPLE_FILE), + "__type": "Report", + "time.observation": "2015-01-01T00:00:00+00:00", + "extra.file_name": "2019-01-01-test_telnet-test-geo.csv", + } +EVENTS = [{'__type': 'Event', + 'feed.name': 'Test-Accessible-Telnet', + "classification.identifier": "test-telnet", + "classification.taxonomy": "vulnerable", + "classification.type": "vulnerable-system", + "extra.banner": "|MikroTik v6.5|Login:", + "extra.tag": "telnet-alt", + "protocol.application": "telnet", + "protocol.transport": "tcp", + 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], + EXAMPLE_LINES[1]])), + "source.asn": 20255, + "source.geolocation.cc": "AA", + "source.geolocation.city": "LOCATION", + "source.geolocation.region": "LOCATION", + "source.ip": "198.123.245.145", + "source.port": 5678, + "source.reverse_dns": "example.local", + "time.observation": "2015-01-01T00:00:00+00:00", + "time.source": "2019-09-04T12:27:34+00:00" + }, + {'__type': 'Event', + 'feed.name': 'Test-Accessible-Telnet', + "classification.identifier": "test-telnet", + "classification.taxonomy": "vulnerable", + "classification.type": "vulnerable-system", + "extra.banner": "|MikroTik v6.45.3 (stable)|Login:", + "extra.tag": "telnet-alt", + "protocol.application": "telnet", + "protocol.transport": "tcp", + 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], + EXAMPLE_LINES[2]])), + "source.asn": 20255, + "source.geolocation.cc": "AA", + "source.geolocation.city": "LOCATION", + "source.geolocation.region": "LOCATION", + "source.ip": "198.123.245.145", + "source.port": 5678, + "source.reverse_dns": "example.local", + "time.observation": "2015-01-01T00:00:00+00:00", + "time.source": "2019-09-04T12:27:40+00:00" + }] + + +class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): + """ + A TestCase for a ShadowserverParserBot. + """ + + @classmethod + def set_bot(cls): + cls.bot_reference = ShadowserverParserBot + cls.default_input_message = EXAMPLE_REPORT + + def test_event(self): + """ Test if correct Event has been produced. """ + self.run_bot() + for i, EVENT in enumerate(EVENTS): + self.assertMessageEqual(i, EVENT) + + +if __name__ == '__main__': # pragma: no cover + unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv new file mode 100644 index 0000000000..fc7fe2fff6 --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv @@ -0,0 +1,4 @@ +"timestamp","ip","port","hostname","tag","asn","geo","region","city","naics","sic","smb_implant","arch","key","smbv1_support","smb_major_number","smb_minor_number","smb_revision","smb_version_string" +"2010-02-10 00:00:00",192.168.0.1,445,node01.example.com,smb,64512,ZZ,Region,City,0,0,N,,,N,2,1,0,"SMB 2.1" +"2010-02-10 00:00:01",192.168.0.2,445,node02.example.com,smb,64512,ZZ,Region,City,0,0,N,,,N,2,1,0,"SMB 2.1" +"2010-02-10 00:00:02",192.168.0.3,445,node03.example.com,smb,64512,ZZ,Region,City,0,0,N,,,N,2,1,0,"SMB 2.1" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv.license new file mode 100644 index 0000000000..f512a890e4 --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv.license @@ -0,0 +1,2 @@ +SPDX-FileCopyrightText: 2022 The Shadowserver Foundation +SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv new file mode 100644 index 0000000000..3309e9a3d8 --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv @@ -0,0 +1,3 @@ +"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","banner" +"2019-09-04 12:27:34","198.123.245.145","tcp",5678,"example.local","telnet-alt",20255,"AA","LOCATION","LOCATION",0,0,"|MikroTik v6.5|Login:" +"2019-09-04 12:27:40","198.123.245.145","tcp",5678,"example.local","telnet-alt",20255,"AA","LOCATION","LOCATION",0,0,"|MikroTik v6.45.3 (stable)|Login:" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv.license new file mode 100644 index 0000000000..942a94035d --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv.license @@ -0,0 +1,2 @@ +SPDX-FileCopyrightText: 2019 Guillermo Rodriguez +SPDX-License-Identifier: AGPL-3.0-or-later From eafa15bc8ea8ac214db9cf349d971dbd450aa149 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Mon, 8 May 2023 15:05:12 +0000 Subject: [PATCH 25/76] Updated to reset report type on reload #2361 --- intelmq/bots/parsers/shadowserver/README.md | 2 +- intelmq/bots/parsers/shadowserver/_config.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md index 297930861b..bb6216b9a7 100644 --- a/intelmq/bots/parsers/shadowserver/README.md +++ b/intelmq/bots/parsers/shadowserver/README.md @@ -11,6 +11,6 @@ The report configuration is now stored in a _schema.json_ file downloaded from h For environments that have internet connectivity the `update_schema.py` script should be setup as a cron job to obtain the latest revision. -For air-gapped systems automation will be required to download and copy the _schema.json_ file into this directory +For air-gapped systems automation will be required to download and copy the _schema.json_ file into this directory. The parser will automatically reload the configuration when the file changes. diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index a7b80b7a6c..29382d2782 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -272,15 +272,14 @@ def scan_exchange_identifier(field): def reload (): """ reload the configuration if it has changed """ mtime = 0.0 + schema_file = __config.schema_file - if (os.path.isfile(__config.schema_file)): + if os.path.isfile(__config.schema_file): mtime = os.path.getmtime(__config.schema_file) if __config.schema_mtime == mtime: return - schema_file = __config.schema_file else: # load a test schema if one has not been downloaded yet - schema_file = __config.schema_file schema_file += '.test' __config.feedname_mapping.clear() From b2753cb9fe6ae15eb569b6d718f54333e476c62d Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 23 May 2023 01:12:47 +0000 Subject: [PATCH 26/76] Added schema download on startup and additional logging --- intelmq/bots/parsers/shadowserver/_config.py | 33 +++++++++++++------ intelmq/bots/parsers/shadowserver/parser.py | 1 + .../parsers/shadowserver/update_schema.py | 3 +- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index 29382d2782..f766be3221 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -106,6 +106,8 @@ def get_feed_by_filename(given_filename: str) -> Optional[Tuple[str, Dict[str, A reload() return __config.filename_mapping.get(given_filename, None) +def set_logger(logger): + __config.logger = logger def add_UTC_to_timestamp(value: str) -> str: return value + ' UTC' @@ -272,29 +274,38 @@ def scan_exchange_identifier(field): def reload (): """ reload the configuration if it has changed """ mtime = 0.0 - schema_file = __config.schema_file if os.path.isfile(__config.schema_file): mtime = os.path.getmtime(__config.schema_file) if __config.schema_mtime == mtime: return else: - # load a test schema if one has not been downloaded yet - schema_file += '.test' + __config.logger.info("The schema file does not exist.") + + if __config.schema_mtime == 0.0 and mtime == 0.0 and not os.environ.get('INTELMQ_SKIP_INTERNET'): + __config.logger.info("Attempting to download schema.") + update_schema() __config.feedname_mapping.clear() __config.filename_mapping.clear() - with open(schema_file) as fh: - schema = json.load(fh) - for report in schema: - __config.feedname_mapping[schema[report]['feed_name']] = (schema[report]['feed_name'], schema[report]) - __config.filename_mapping[schema[report]['file_name']] = (schema[report]['feed_name'], schema[report]) + for schema_file in [ __config.schema_file, ".".join([__config.schema_file, 'test']) ]: + if os.path.isfile(schema_file): + with open(schema_file) as fh: + schema = json.load(fh) + for report in schema: + if report == "_meta": + __config.logger.info("Loading schema %s." % schema[report]['date_created']) + for msg in schema[report]['change_log']: + __config.logger.info(msg) + else: + __config.feedname_mapping[schema[report]['feed_name']] = (schema[report]['feed_name'], schema[report]) + __config.filename_mapping[schema[report]['file_name']] = (schema[report]['feed_name'], schema[report]) __config.schema_mtime = mtime -def update_schema (version): +def update_schema (): """ download the latest configuration """ (th, tmp) = tempfile.mkstemp() - url = 'https://interchange.shadowserver.org/intelmq/'+version + url = 'https://interchange.shadowserver.org/intelmq/v1' try: urllib.request.urlretrieve(url, tmp) except: @@ -307,4 +318,6 @@ def update_schema (version): # leave tempfile behind for diagnosis raise ValueError("Failed to validate %r" % tmp) + if os.path.exists(__config.schema_file): + os.replace(__config.schema_file, ".".join([__config.schema_file, 'bak'])) os.replace(tmp, __config.schema_file) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index f14549141a..2f20262bfa 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -47,6 +47,7 @@ class ShadowserverParserBot(ParserBot): overwrite = False def init(self): + config.set_logger(self.logger) if self.feedname is not None: self._sparser_config = config.get_feed_by_feedname(self.feedname) if self._sparser_config: diff --git a/intelmq/bots/parsers/shadowserver/update_schema.py b/intelmq/bots/parsers/shadowserver/update_schema.py index 040f672593..a7975147ed 100644 --- a/intelmq/bots/parsers/shadowserver/update_schema.py +++ b/intelmq/bots/parsers/shadowserver/update_schema.py @@ -8,5 +8,4 @@ import intelmq.bots.parsers.shadowserver._config as config if __name__ == '__main__': # pragma: no cover - exec(open(os.path.join(os.path.dirname(__file__), '../../../version.py')).read()) # defines __version__ - config.update_schema(__version__) + config.update_schema() From fd0a8fd44c39a5dba2684846b9c03262ccf9307a Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 23 May 2023 23:32:53 +0000 Subject: [PATCH 27/76] Added version support to the schema update function. --- intelmq/bots/parsers/shadowserver/README.md | 6 ++-- intelmq/bots/parsers/shadowserver/_config.py | 32 +++++++++++++++++--- intelmq/bots/parsers/shadowserver/parser.py | 4 +++ 3 files changed, 34 insertions(+), 8 deletions(-) diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md index bb6216b9a7..c757020e94 100644 --- a/intelmq/bots/parsers/shadowserver/README.md +++ b/intelmq/bots/parsers/shadowserver/README.md @@ -7,10 +7,10 @@ This module is maintained by [The Shadowserver Foundation](https://www.shadowser Please contact intelmq@shadowserver.org with any issues or concerns. -The report configuration is now stored in a _schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/_version_. +The report configuration is now stored in a _schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/v1. -For environments that have internet connectivity the `update_schema.py` script should be setup as a cron job to obtain the latest revision. +For environments that have internet connectivity the `update_schema.py` script can be called from a cron job to obtain the latest revision. -For air-gapped systems automation will be required to download and copy the _schema.json_ file into this directory. +For air-gapped systems automation will be required to download and copy the _schema.json_ file into this directory. The parser will automatically reload the configuration when the file changes. diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index f766be3221..bb67db525a 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -97,6 +97,11 @@ class __Container: __config.feedname_mapping = {} __config.filename_mapping = {} +def set_logger(logger): + """ Sets the logger instance. """ + __config.logger = logger + + def get_feed_by_feedname(given_feedname: str) -> Optional[Dict[str, Any]]: reload() return __config.feedname_mapping.get(given_feedname, None) @@ -106,8 +111,6 @@ def get_feed_by_filename(given_filename: str) -> Optional[Tuple[str, Dict[str, A reload() return __config.filename_mapping.get(given_filename, None) -def set_logger(logger): - __config.logger = logger def add_UTC_to_timestamp(value: str) -> str: return value + ' UTC' @@ -304,20 +307,39 @@ def reload (): def update_schema (): """ download the latest configuration """ - (th, tmp) = tempfile.mkstemp() + if os.environ.get('INTELMQ_SKIP_INTERNET'): + return None + + (th, tmp) = tempfile.mkstemp(dir=os.path.dirname(__file__)) url = 'https://interchange.shadowserver.org/intelmq/v1' try: urllib.request.urlretrieve(url, tmp) except: raise ValueError("Failed to download %r" % url) + new_version = '' + old_version = '' + try: with open(tmp) as fh: schema = json.load(fh) + new_version = schema['_meta']['date_created'] except: # leave tempfile behind for diagnosis raise ValueError("Failed to validate %r" % tmp) if os.path.exists(__config.schema_file): - os.replace(__config.schema_file, ".".join([__config.schema_file, 'bak'])) - os.replace(tmp, __config.schema_file) + old_version = '' + try: + with open(__config.schema_file) as fh: + schema = json.load(fh) + old_version = schema['_meta']['date_created'] + if new_version != old_version: + os.replace(__config.schema_file, ".".join([__config.schema_file, 'bak'])) + except: + pass + + if new_version != old_version: + os.replace(tmp, __config.schema_file) + else: + os.unlink(tmp) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index 2f20262bfa..71489e2ec1 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -48,6 +48,10 @@ class ShadowserverParserBot(ParserBot): def init(self): config.set_logger(self.logger) + try: + config.update_schema() + except Exception as e: + logger.warning(f"Schema update failed: {e}.") if self.feedname is not None: self._sparser_config = config.get_feed_by_feedname(self.feedname) if self._sparser_config: From 357aad523c5a875121a38f26164cfff9fbacd24b Mon Sep 17 00:00:00 2001 From: elsif2 Date: Sun, 28 May 2023 23:13:54 +0000 Subject: [PATCH 28/76] Documentation and style updates. --- CHANGELOG.md | 6 + .../shadowserver/collector_reports_api.py | 2 +- intelmq/bots/parsers/shadowserver/README.md | 39 ++++- intelmq/bots/parsers/shadowserver/_config.py | 52 +++--- intelmq/bots/parsers/shadowserver/parser.py | 2 +- .../bots/parsers/shadowserver/test_broken.py | 4 +- .../bots/parsers/shadowserver/test_mapping.py | 1 - .../parsers/shadowserver/test_report_smb.py | 151 +++++++++--------- .../shadowserver/test_report_switch.py | 10 +- .../shadowserver/test_report_telnet.py | 4 +- 10 files changed, 154 insertions(+), 117 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 72d9501937..ea36275bc0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -118,15 +118,21 @@ CHANGELOG ### Bots #### Collectors +<<<<<<< HEAD - `intelmq.bots.collector.rt`: - restrict `python-rt` to be below version 3.0 due to introduced breaking changes, - added support for `Subject NOT LIKE` queries, - added support for multiple values in ticket subject queries. - `intelmq.bots.collectors.rsync`: Support for optional private key, relative time parsing for the source path, extra rsync parameters and strict host key checking (PR#2241 by Mateo Durante). +======= +- `intelmq.bots.collectors.shadowserver.collector_reports_api`: + - The 'json' option is no longer supported as the 'csv' option provides better performance. Please see intelmq/bots/parsers/shadowserver/README.md for a sample configuration. (PR#2372) +>>>>>>> Documentation and style updates. #### Parsers - `intelmq.bots.parsers.shadowserver._config`: - Reset detected `feedname` at shutdown to re-detect the feedname on reloads (PR#2361 by @elsif2, fixes #2360). + - Switch to dynamic configuration to decouple report schema changes from IntelMQ releases. Please see intelmq/bots/parsers/shadowserver/README.md for details. (PR#2372) - `intelmq.bots.parsers.shadowserver._config`: - Added 'IPv6-Vulnerable-Exchange' alias and 'Accessible-WS-Discovery-Service' report. (PR#2338) - Removed unused `p0f_genre` and `p0f_detail` from the 'DNS-Open-Resolvers' report. (PR#2338) diff --git a/intelmq/bots/collectors/shadowserver/collector_reports_api.py b/intelmq/bots/collectors/shadowserver/collector_reports_api.py index dc8bd6b420..5e7117bd23 100644 --- a/intelmq/bots/collectors/shadowserver/collector_reports_api.py +++ b/intelmq/bots/collectors/shadowserver/collector_reports_api.py @@ -34,7 +34,7 @@ class ShadowServerAPICollectorBot(CollectorBot, HttpMixin, CacheMixin): A list of strings or a comma-separated list of the mailing lists you want to process. types (list): A list of strings or a string of comma-separated values with the names of reporttypes you want to process. If you leave this empty, all the available reports will be downloaded and processed (i.e. 'scan', 'drones', 'intel', 'sandbox_connection', 'sinkhole_combined'). - file_format (str): File format to download ('csv'). The 'json' option is not longer supported. + file_format (str): File format to download ('csv'). The 'json' option is no longer supported. """ country = None diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md index c757020e94..ae38dcb8cc 100644 --- a/intelmq/bots/parsers/shadowserver/README.md +++ b/intelmq/bots/parsers/shadowserver/README.md @@ -7,10 +7,45 @@ This module is maintained by [The Shadowserver Foundation](https://www.shadowser Please contact intelmq@shadowserver.org with any issues or concerns. -The report configuration is now stored in a _schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/v1. +The report configuration is now stored in a _schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/v1/schema. -For environments that have internet connectivity the `update_schema.py` script can be called from a cron job to obtain the latest revision. +For environments that have internet connectivity the `update_schema.py` script should be called from a cron job to obtain the latest revision. +The parser will attempt to download a schema update on startup unless INTELMQ_SKIP_INTERNET is set. For air-gapped systems automation will be required to download and copy the _schema.json_ file into this directory. The parser will automatically reload the configuration when the file changes. + + +## Sample configuration: + +``` +shadowserver-collector: + description: Our bot responsible for getting reports from Shadowserver + enabled: true + group: Collector + module: intelmq.bots.collectors.shadowserver.collector_reports_api + name: Shadowserver_Collector + parameters: + destination_queues: + _default: [shadowserver-parser-queue] + file_format: csv + api_key: "$API_KEY_received_from_the_shadowserver_foundation" + secret: "$SECRET_received_from_the_shadowserver_foundation" + run_mode: continuous +``` + +``` +shadowserver-parser: + bot_id: shadowserver-parser + name: Shadowserver Parser + enabled: true + group: Parser + groupname: parsers + module: intelmq.bots.parsers.shadowserver.parser + parameters: + destination_queues: + _default: [file-output-queue] + run_mode: continuous +``` + diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index bb67db525a..5219fdb344 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -88,15 +88,18 @@ import intelmq.lib.harmonization as harmonization + class __Container: pass + __config = __Container() __config.schema_file = os.path.join(os.path.dirname(__file__), 'schema.json') __config.schema_mtime = 0.0 __config.feedname_mapping = {} __config.filename_mapping = {} + def set_logger(logger): """ Sets the logger instance. """ __config.logger = logger @@ -254,27 +257,28 @@ def scan_exchange_identifier(field): return 'exchange-server-webshell' return 'vulnerable-exchange-server' + functions = { - 'add_UTC_to_timestamp': add_UTC_to_timestamp, - 'convert_bool': convert_bool, - 'validate_to_none': validate_to_none, - 'convert_int': convert_int, - 'convert_float': convert_float, - 'convert_http_host_and_url': convert_http_host_and_url, - 'invalidate_zero': invalidate_zero, - 'validate_ip': validate_ip, - 'validate_network': validate_network, - 'validate_fqdn': validate_fqdn, - 'convert_date': convert_date, - 'convert_date_utc': convert_date_utc, - 'force_base64': force_base64, - 'scan_exchange_taxonomy': scan_exchange_taxonomy, - 'scan_exchange_type': scan_exchange_type, - 'scan_exchange_identifier': scan_exchange_identifier, - } - - -def reload (): + 'add_UTC_to_timestamp': add_UTC_to_timestamp, + 'convert_bool': convert_bool, + 'validate_to_none': validate_to_none, + 'convert_int': convert_int, + 'convert_float': convert_float, + 'convert_http_host_and_url': convert_http_host_and_url, + 'invalidate_zero': invalidate_zero, + 'validate_ip': validate_ip, + 'validate_network': validate_network, + 'validate_fqdn': validate_fqdn, + 'convert_date': convert_date, + 'convert_date_utc': convert_date_utc, + 'force_base64': force_base64, + 'scan_exchange_taxonomy': scan_exchange_taxonomy, + 'scan_exchange_type': scan_exchange_type, + 'scan_exchange_identifier': scan_exchange_identifier, +} + + +def reload(): """ reload the configuration if it has changed """ mtime = 0.0 @@ -291,7 +295,7 @@ def reload (): __config.feedname_mapping.clear() __config.filename_mapping.clear() - for schema_file in [ __config.schema_file, ".".join([__config.schema_file, 'test']) ]: + for schema_file in [__config.schema_file, ".".join([__config.schema_file, 'test'])]: if os.path.isfile(schema_file): with open(schema_file) as fh: schema = json.load(fh) @@ -305,13 +309,14 @@ def reload (): __config.filename_mapping[schema[report]['file_name']] = (schema[report]['feed_name'], schema[report]) __config.schema_mtime = mtime -def update_schema (): + +def update_schema(): """ download the latest configuration """ if os.environ.get('INTELMQ_SKIP_INTERNET'): return None (th, tmp) = tempfile.mkstemp(dir=os.path.dirname(__file__)) - url = 'https://interchange.shadowserver.org/intelmq/v1' + url = 'https://interchange.shadowserver.org/intelmq/v1/schema' try: urllib.request.urlretrieve(url, tmp) except: @@ -329,7 +334,6 @@ def update_schema (): raise ValueError("Failed to validate %r" % tmp) if os.path.exists(__config.schema_file): - old_version = '' try: with open(__config.schema_file) as fh: schema = json.load(fh) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index 71489e2ec1..668a815341 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -51,7 +51,7 @@ def init(self): try: config.update_schema() except Exception as e: - logger.warning(f"Schema update failed: {e}.") + self.logger.warning("Schema update failed: %s." % e) if self.feedname is not None: self._sparser_config = config.get_feed_by_feedname(self.feedname) if self._sparser_config: diff --git a/intelmq/tests/bots/parsers/shadowserver/test_broken.py b/intelmq/tests/bots/parsers/shadowserver/test_broken.py index 2b803142eb..3797f03cd5 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_broken.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_broken.py @@ -24,12 +24,12 @@ "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", "extra.file_name": "2019-01-01-some_string-test-test.csv", -} + } REPORT4 = {"raw": utils.base64_encode('adasdasdasdasd\nadasdasdafgf'), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", "extra.file_name": "2020.wrong-filename.csv", -} + } class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): diff --git a/intelmq/tests/bots/parsers/shadowserver/test_mapping.py b/intelmq/tests/bots/parsers/shadowserver/test_mapping.py index 6a2af94475..d296dfdc26 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_mapping.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_mapping.py @@ -52,6 +52,5 @@ def test_changed_feed(self): self.run_bot(iterations=2) - if __name__ == '__main__': # pragma: no cover unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py b/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py index c7eefdf0a9..93d592d15c 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py @@ -22,85 +22,78 @@ "time.observation": "2015-01-01T00:00:00+00:00", "extra.file_name": "2019-01-01-test_smb-test-geo.csv", } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'test-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Test-Accessible-SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 445, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'test-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Test-Accessible-SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 445, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'test-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Test-Accessible-SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 445, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} - ] +EVENTS = [{'__type': 'Event', + 'classification.identifier': 'test-smb', + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + 'extra.smb_implant': False, + 'extra.smb_major_number': '2', + 'extra.smb_minor_number': '1', + 'extra.smb_version_string': 'SMB 2.1', + 'extra.smbv1_support': 'N', + 'extra.tag': 'smb', + 'feed.name': 'Test-Accessible-SMB', + 'protocol.application': 'smb', + 'protocol.transport': 'tcp', + 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), + 'source.asn': 64512, + 'source.geolocation.cc': 'ZZ', + 'source.geolocation.city': 'City', + 'source.geolocation.region': 'Region', + 'source.ip': '192.168.0.1', + 'source.port': 445, + 'source.reverse_dns': 'node01.example.com', + 'time.observation': '2015-01-01T00:00:00+00:00', + 'time.source': '2010-02-10T00:00:00+00:00' + }, + {'__type': 'Event', + 'classification.identifier': 'test-smb', + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + 'extra.smb_implant': False, + 'extra.smb_major_number': '2', + 'extra.smb_minor_number': '1', + 'extra.smb_version_string': 'SMB 2.1', + 'extra.smbv1_support': 'N', + 'extra.tag': 'smb', + 'feed.name': 'Test-Accessible-SMB', + 'protocol.application': 'smb', + 'protocol.transport': 'tcp', + 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), + 'source.asn': 64512, + 'source.geolocation.cc': 'ZZ', + 'source.geolocation.city': 'City', + 'source.geolocation.region': 'Region', + 'source.ip': '192.168.0.2', + 'source.port': 445, + 'source.reverse_dns': 'node02.example.com', + 'time.observation': '2015-01-01T00:00:00+00:00', + 'time.source': '2010-02-10T00:00:01+00:00' + }, + {'__type': 'Event', + 'classification.identifier': 'test-smb', + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + 'extra.smb_implant': False, + 'extra.smb_major_number': '2', + 'extra.smb_minor_number': '1', + 'extra.smb_version_string': 'SMB 2.1', + 'extra.smbv1_support': 'N', + 'extra.tag': 'smb', + 'feed.name': 'Test-Accessible-SMB', + 'protocol.application': 'smb', + 'protocol.transport': 'tcp', + 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), + 'source.asn': 64512, + 'source.geolocation.cc': 'ZZ', + 'source.geolocation.city': 'City', + 'source.geolocation.region': 'Region', + 'source.ip': '192.168.0.3', + 'source.port': 445, + 'source.reverse_dns': 'node03.example.com', + 'time.observation': '2015-01-01T00:00:00+00:00', + 'time.source': '2010-02-10T00:00:02+00:00' + }] class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py b/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py index 570d612fb4..a9be8a0a13 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py @@ -16,11 +16,11 @@ EXAMPLE_LINES = handle.read().splitlines()[:2] FIRST_REPORT = {'feed.name': 'Test-Accessible-SMB', - "raw": utils.base64_encode('\n'.join(EXAMPLE_LINES)), - "__type": "Report", - "time.observation": "2019-03-25T00:00:00+00:00", - "extra.file_name": "2019-03-25-test_smb-test-test.csv", - } + "raw": utils.base64_encode('\n'.join(EXAMPLE_LINES)), + "__type": "Report", + "time.observation": "2019-03-25T00:00:00+00:00", + "extra.file_name": "2019-03-25-test_smb-test-test.csv", + } with open(os.path.join(os.path.dirname(__file__), 'testdata/test_telnet.csv')) as handle: EXAMPLE_LINES = handle.read().splitlines()[:2] diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py b/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py index 6d539ac4a7..df9cf25dca 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py @@ -42,7 +42,7 @@ "source.reverse_dns": "example.local", "time.observation": "2015-01-01T00:00:00+00:00", "time.source": "2019-09-04T12:27:34+00:00" - }, + }, {'__type': 'Event', 'feed.name': 'Test-Accessible-Telnet', "classification.identifier": "test-telnet", @@ -63,7 +63,7 @@ "source.reverse_dns": "example.local", "time.observation": "2015-01-01T00:00:00+00:00", "time.source": "2019-09-04T12:27:40+00:00" - }] + }] class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): From 37c67459f7ea791c31cd36b74456be27d079f9fe Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 30 May 2023 16:05:26 +0000 Subject: [PATCH 29/76] Added schema.json.test.license. --- intelmq/bots/parsers/shadowserver/schema.json.test.license | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 intelmq/bots/parsers/shadowserver/schema.json.test.license diff --git a/intelmq/bots/parsers/shadowserver/schema.json.test.license b/intelmq/bots/parsers/shadowserver/schema.json.test.license new file mode 100644 index 0000000000..9f58c89ef0 --- /dev/null +++ b/intelmq/bots/parsers/shadowserver/schema.json.test.license @@ -0,0 +1,2 @@ +SPDX-FileCopyrightText: 2023 The Shadowserver Foundation +SPDX-License-Identifier: AGPL-3.0-or-later From ee8ce873977d3de18ebddddaac2c38c3ed5ca257 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Thu, 27 Jul 2023 20:19:25 +0000 Subject: [PATCH 30/76] Updates in response to feedback. --- .../shadowserver/collector_reports_api.py | 9 +++- intelmq/bots/parsers/shadowserver/README.md | 21 ++++++-- intelmq/bots/parsers/shadowserver/_config.py | 53 +++++++++++++------ intelmq/bots/parsers/shadowserver/parser.py | 45 +++++++++++++--- .../parsers/shadowserver/update_schema.py | 11 ---- .../shadowserver/test_download_schema.py | 28 ++++++++++ 6 files changed, 130 insertions(+), 37 deletions(-) delete mode 100644 intelmq/bots/parsers/shadowserver/update_schema.py create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_download_schema.py diff --git a/intelmq/bots/collectors/shadowserver/collector_reports_api.py b/intelmq/bots/collectors/shadowserver/collector_reports_api.py index 5e7117bd23..05bffa898e 100644 --- a/intelmq/bots/collectors/shadowserver/collector_reports_api.py +++ b/intelmq/bots/collectors/shadowserver/collector_reports_api.py @@ -68,12 +68,19 @@ def init(self): if self.file_format is not None: if not (self.file_format == 'csv'): - raise ValueError('Invalid file_format') + raise ValueError("Invalid file_format '%s'. Must be 'csv'." % self.file_format) else: self.file_format = 'csv' self.preamble = f'{{ "apikey": "{self.api_key}" ' + def check(parameters: dict): + for key in parameters: + if key == 'file_format' and parameters[key] != 'csv': + return [["error", "Invalid file_format '%s'. Must be 'csv'." % parameters[key]]] + elif key == 'country': + return [["warning", "Deprecated parameter 'country' found. Please use 'reports' instead. The backwards-compatibility will be removed in IntelMQ version 4.0.0."]] + def _headers(self, data): return {'HMAC2': hmac.new(self.secret.encode(), data.encode('utf-8'), digestmod=hashlib.sha256).hexdigest()} diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md index ae38dcb8cc..cd750d00b3 100644 --- a/intelmq/bots/parsers/shadowserver/README.md +++ b/intelmq/bots/parsers/shadowserver/README.md @@ -7,16 +7,28 @@ This module is maintained by [The Shadowserver Foundation](https://www.shadowser Please contact intelmq@shadowserver.org with any issues or concerns. -The report configuration is now stored in a _schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/v1/schema. +The report configuration is now stored in a _shadowserver-schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/v1/schema. -For environments that have internet connectivity the `update_schema.py` script should be called from a cron job to obtain the latest revision. -The parser will attempt to download a schema update on startup unless INTELMQ_SKIP_INTERNET is set. +The parser will attempt to download a schema update on startup when the *auto_update* option is enabled. -For air-gapped systems automation will be required to download and copy the _schema.json_ file into this directory. +Schema downloads can also be scheduled as a cron job: + +``` +02 01 * * * intelmq.bots.parsers.shadowserver.parser --update-schema +``` + +For air-gapped systems automation will be required to download and copy the file to VAR_STATE_PATH/shadowserver-schema.json. The parser will automatically reload the configuration when the file changes. +## Schema contract + +Once set the `classification.identifier`, `classification.taxonomy`, and `classification.type` fields will remain static. + +Once set report fields will not be deleted. + + ## Sample configuration: ``` @@ -46,6 +58,7 @@ shadowserver-parser: parameters: destination_queues: _default: [file-output-queue] + auto_update: true run_mode: continuous ``` diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index 5219fdb344..afe3a6b11f 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -82,11 +82,12 @@ import base64 import binascii import json -import urllib.request import tempfile from typing import Optional, Dict, Tuple, Any import intelmq.lib.harmonization as harmonization +from intelmq.lib.utils import create_request_session +from intelmq import VAR_STATE_PATH class __Container: @@ -94,8 +95,10 @@ class __Container: __config = __Container() -__config.schema_file = os.path.join(os.path.dirname(__file__), 'schema.json') +__config.schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json') +__config.schema_base = os.path.join(os.path.dirname(__file__), 'schema.json.test') __config.schema_mtime = 0.0 +__config.auto_update = False __config.feedname_mapping = {} __config.filename_mapping = {} @@ -105,13 +108,16 @@ def set_logger(logger): __config.logger = logger +def enable_auto_update(enable): + """ Enable automatic schema update. """ + __config.auto_update = enable + + def get_feed_by_feedname(given_feedname: str) -> Optional[Dict[str, Any]]: - reload() return __config.feedname_mapping.get(given_feedname, None) def get_feed_by_filename(given_filename: str) -> Optional[Tuple[str, Dict[str, Any]]]: - reload() return __config.filename_mapping.get(given_filename, None) @@ -289,19 +295,18 @@ def reload(): else: __config.logger.info("The schema file does not exist.") - if __config.schema_mtime == 0.0 and mtime == 0.0 and not os.environ.get('INTELMQ_SKIP_INTERNET'): - __config.logger.info("Attempting to download schema.") + if __config.schema_mtime == 0.0 and mtime == 0.0 and __config.auto_update: update_schema() __config.feedname_mapping.clear() __config.filename_mapping.clear() - for schema_file in [__config.schema_file, ".".join([__config.schema_file, 'test'])]: + for schema_file in [__config.schema_file, __config.schema_base]: if os.path.isfile(schema_file): with open(schema_file) as fh: schema = json.load(fh) for report in schema: if report == "_meta": - __config.logger.info("Loading schema %s." % schema[report]['date_created']) + __config.logger.info("Loading schema %r." % schema[report]['date_created']) for msg in schema[report]['change_log']: __config.logger.info(msg) else: @@ -313,37 +318,55 @@ def reload(): def update_schema(): """ download the latest configuration """ if os.environ.get('INTELMQ_SKIP_INTERNET'): - return None + return False - (th, tmp) = tempfile.mkstemp(dir=os.path.dirname(__file__)) + # download the schema to a temp file + (th, tmp) = tempfile.mkstemp(dir=VAR_STATE_PATH) url = 'https://interchange.shadowserver.org/intelmq/v1/schema' + __config.logger.info("Attempting to download schema from %r" % url) + __config.logger.debug("Using temp file %r for the download." % tmp) try: - urllib.request.urlretrieve(url, tmp) + with create_request_session() as session: + with session.get(url, stream=True) as r: + r.raise_for_status() + with open(tmp, 'wb') as f: + for chunk in r.iter_content(chunk_size=8192): + f.write(chunk) except: - raise ValueError("Failed to download %r" % url) + __config.logger.error("Failed to download %r" % url) + return False + __config.logger.info("Download successful.") new_version = '' old_version = '' try: + # validate the downloaded file with open(tmp) as fh: schema = json.load(fh) new_version = schema['_meta']['date_created'] except: # leave tempfile behind for diagnosis - raise ValueError("Failed to validate %r" % tmp) + __config.logger.error("Failed to validate %r" % tmp) + return False if os.path.exists(__config.schema_file): + # compare the new version against the old; rename the existing file try: with open(__config.schema_file) as fh: schema = json.load(fh) old_version = schema['_meta']['date_created'] if new_version != old_version: os.replace(__config.schema_file, ".".join([__config.schema_file, 'bak'])) - except: - pass + except Exception as e: + __config.logger.error("Unable to replace schema file: %s" % str(e)) + return False if new_version != old_version: os.replace(tmp, __config.schema_file) + __config.logger.info("New schema version is %r." % new_version) + return True else: os.unlink(tmp) + + return False diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index 668a815341..2e383a004e 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -26,6 +26,8 @@ from intelmq.lib.bot import ParserBot from intelmq.lib.exceptions import InvalidKey, InvalidValue +from intelmq.bin.intelmqctl import IntelMQController +import intelmq.lib.utils as utils import intelmq.bots.parsers.shadowserver._config as config @@ -34,8 +36,7 @@ class ShadowserverParserBot(ParserBot): Parse all ShadowServer feeds Parameters: - schema_file (str): Path to the report schema file - + auto_update (boolean): Enable automatic schema download """ recover_line = ParserBot.recover_line_csv_dict @@ -45,13 +46,15 @@ class ShadowserverParserBot(ParserBot): feedname = None _mode = None overwrite = False + auto_update = False def init(self): config.set_logger(self.logger) - try: - config.update_schema() - except Exception as e: - self.logger.warning("Schema update failed: %s." % e) + if self.auto_update: + config.enable_auto_update(True) + self.logger.debug("Feature 'auto_update' is enabled.") + config.reload() + if self.feedname is not None: self._sparser_config = config.get_feed_by_feedname(self.feedname) if self._sparser_config: @@ -228,5 +231,35 @@ def parse_line(self, row, report): def shutdown(self): self.feedname = None + @classmethod + def _create_argparser(cls): + argparser = super()._create_argparser() + argparser.add_argument("--update-schema", action='store_true', help='downloads latest report schema') + argparser.add_argument("--verbose", action='store_true', help='be verbose') + return argparser + + @classmethod + def run(cls, parsed_args=None): + if not parsed_args: + parsed_args = cls._create_argparser().parse_args() + if parsed_args.update_schema: + logger = utils.log(__name__, log_path=None) + if parsed_args.verbose: + logger.setLevel('INFO') + else: + logger.setLevel('ERROR') + config.set_logger(logger) + if config.update_schema(): + runtime_conf = utils.get_bots_settings() + try: + ctl = IntelMQController() + for bot in runtime_conf: + if runtime_conf[bot]["module"] == __name__ and runtime_conf[bot]['parameters'].get('auto_update', True): + ctl.bot_reload(bot) + except Exception as e: + logger.error("Failed to signal bot: %r" % str(e)) + else: + super().run(parsed_args=parsed_args) + BOT = ShadowserverParserBot diff --git a/intelmq/bots/parsers/shadowserver/update_schema.py b/intelmq/bots/parsers/shadowserver/update_schema.py deleted file mode 100644 index a7975147ed..0000000000 --- a/intelmq/bots/parsers/shadowserver/update_schema.py +++ /dev/null @@ -1,11 +0,0 @@ -# SPDX-FileCopyrightText: 2023 The Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import intelmq.bots.parsers.shadowserver._config as config - -if __name__ == '__main__': # pragma: no cover - config.update_schema() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py new file mode 100644 index 0000000000..e685876826 --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: 2023 The Shadowserver Foundation +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +# -*- coding: utf-8 -*- +""" +Created on Thu Jul 27 19:44:44 2023 + +""" + +import unittest +import os +import logging +from intelmq import VAR_STATE_PATH +import intelmq.bots.parsers.shadowserver._config as config +import intelmq.lib.utils as utils +import intelmq.lib.test as test + +@test.skip_internet() +class TestShadowserverSchemaDownload(unittest.TestCase): + + def test_download(self): + schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json') + config.set_logger(utils.log('test-bot', log_path=None)) + if os.path.exists(schema_file): + os.unlink(schema_file) + self.assertEqual(True, config.update_schema()) + self.assertEqual(True, os.path.exists(schema_file)) From 4a73f0b9af80d126b1e19de43097700e24ad7f63 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Fri, 28 Jul 2023 14:17:41 +0000 Subject: [PATCH 31/76] Removed file_format parameter --- .../shadowserver/collector_reports_api.py | 20 ++++--------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/intelmq/bots/collectors/shadowserver/collector_reports_api.py b/intelmq/bots/collectors/shadowserver/collector_reports_api.py index 05bffa898e..66169d96f1 100644 --- a/intelmq/bots/collectors/shadowserver/collector_reports_api.py +++ b/intelmq/bots/collectors/shadowserver/collector_reports_api.py @@ -34,7 +34,6 @@ class ShadowServerAPICollectorBot(CollectorBot, HttpMixin, CacheMixin): A list of strings or a comma-separated list of the mailing lists you want to process. types (list): A list of strings or a string of comma-separated values with the names of reporttypes you want to process. If you leave this empty, all the available reports will be downloaded and processed (i.e. 'scan', 'drones', 'intel', 'sandbox_connection', 'sinkhole_combined'). - file_format (str): File format to download ('csv'). The 'json' option is no longer supported. """ country = None @@ -42,7 +41,6 @@ class ShadowServerAPICollectorBot(CollectorBot, HttpMixin, CacheMixin): secret = None types = None reports = None - file_format = None rate_limit: int = 86400 redis_cache_db: int = 12 redis_cache_host: str = "127.0.0.1" # TODO: type could be ipadress @@ -66,18 +64,12 @@ def init(self): self.logger.warn("Deprecated parameter 'country' found. Please use 'reports' instead. The backwards-compatibility will be removed in IntelMQ version 4.0.0.") self._report_list.append(self.country) - if self.file_format is not None: - if not (self.file_format == 'csv'): - raise ValueError("Invalid file_format '%s'. Must be 'csv'." % self.file_format) - else: - self.file_format = 'csv' - self.preamble = f'{{ "apikey": "{self.api_key}" ' def check(parameters: dict): for key in parameters: - if key == 'file_format' and parameters[key] != 'csv': - return [["error", "Invalid file_format '%s'. Must be 'csv'." % parameters[key]]] + if key == 'file_format': + return [["error", "The file_format parameter is no longer supported. All reports are CSV."]] elif key == 'country': return [["warning", "Deprecated parameter 'country' found. Please use 'reports' instead. The backwards-compatibility will be removed in IntelMQ version 4.0.0."]] @@ -129,11 +121,7 @@ def _report_download(self, reportid: str): data = self.preamble data += f',"id": "{reportid}"}}' self.logger.debug('Downloading report with data: %s.', data) - - if (self.file_format == 'json'): - response = self.http_session().post(APIROOT + 'reports/download', data=data, headers=self._headers(data)) - else: - response = self.http_session().get(DLROOT + reportid) + response = self.http_session().get(DLROOT + reportid) response.raise_for_status() return response.text @@ -150,7 +138,7 @@ def process(self): for item in reportslist: filename = item['file'] - filename_fixed = FILENAME_PATTERN.sub('.' + self.file_format, filename, count=1) + filename_fixed = FILENAME_PATTERN.sub('.csv', filename, count=1) if self.cache_get(filename): self.logger.debug('Processed file %r (fixed: %r) already.', filename, filename_fixed) continue From e413fb50513f900a146dbd4c1c45667ae8e04541 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Thu, 24 Aug 2023 16:04:21 +0000 Subject: [PATCH 32/76] Minor changes based on feedback 2023-08-24 --- CHANGELOG.md | 2 - intelmq/bots/parsers/shadowserver/README.md | 2 + intelmq/bots/parsers/shadowserver/_config.py | 49 ++++++++++--------- intelmq/bots/parsers/shadowserver/parser.py | 6 ++- .../bots/parsers/shadowserver/test_broken.py | 5 ++ .../bots/parsers/shadowserver/test_mapping.py | 1 + .../parsers/shadowserver/test_parameters.py | 3 +- .../parsers/shadowserver/test_report_smb.py | 1 + .../shadowserver/test_report_switch.py | 1 + .../shadowserver/test_report_telnet.py | 1 + 10 files changed, 45 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea36275bc0..8cee9e520c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -124,10 +124,8 @@ CHANGELOG - added support for `Subject NOT LIKE` queries, - added support for multiple values in ticket subject queries. - `intelmq.bots.collectors.rsync`: Support for optional private key, relative time parsing for the source path, extra rsync parameters and strict host key checking (PR#2241 by Mateo Durante). -======= - `intelmq.bots.collectors.shadowserver.collector_reports_api`: - The 'json' option is no longer supported as the 'csv' option provides better performance. Please see intelmq/bots/parsers/shadowserver/README.md for a sample configuration. (PR#2372) ->>>>>>> Documentation and style updates. #### Parsers - `intelmq.bots.parsers.shadowserver._config`: diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md index cd750d00b3..4969acb6d0 100644 --- a/intelmq/bots/parsers/shadowserver/README.md +++ b/intelmq/bots/parsers/shadowserver/README.md @@ -28,6 +28,8 @@ Once set the `classification.identifier`, `classification.taxonomy`, and `classi Once set report fields will not be deleted. +The schema revision history is maintained at https://github.com/The-Shadowserver-Foundation/report_schema/. + ## Sample configuration: diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index afe3a6b11f..4bfadb9d98 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -95,8 +95,10 @@ class __Container: __config = __Container() +__config.schema_url = 'https://interchange.shadowserver.org/intelmq/v1/schema' __config.schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json') __config.schema_base = os.path.join(os.path.dirname(__file__), 'schema.json.test') +__config.schema_active = __config.schema_file __config.schema_mtime = 0.0 __config.auto_update = False __config.feedname_mapping = {} @@ -108,6 +110,13 @@ def set_logger(logger): __config.logger = logger +def enable_test_mode(enable): + """ Set which schema to load. """ + if enable: + __config.schema_active = __config.schema_base + else: + __config.schema_active = __config.schema_file + def enable_auto_update(enable): """ Enable automatic schema update. """ __config.auto_update = enable @@ -300,40 +309,36 @@ def reload(): __config.feedname_mapping.clear() __config.filename_mapping.clear() - for schema_file in [__config.schema_file, __config.schema_base]: - if os.path.isfile(schema_file): - with open(schema_file) as fh: - schema = json.load(fh) - for report in schema: - if report == "_meta": - __config.logger.info("Loading schema %r." % schema[report]['date_created']) - for msg in schema[report]['change_log']: - __config.logger.info(msg) - else: - __config.feedname_mapping[schema[report]['feed_name']] = (schema[report]['feed_name'], schema[report]) - __config.filename_mapping[schema[report]['file_name']] = (schema[report]['feed_name'], schema[report]) + if os.path.isfile(__config.schema_active): + with open(__config.schema_active) as fh: + schema = json.load(fh) + for report in schema: + if report == "_meta": + __config.logger.info("Loading schema %r.", schema[report]['date_created']) + for msg in schema[report]['change_log']: + __config.logger.info(msg) + else: + __config.feedname_mapping[schema[report]['feed_name']] = (schema[report]['feed_name'], schema[report]) + __config.filename_mapping[schema[report]['file_name']] = (schema[report]['feed_name'], schema[report]) __config.schema_mtime = mtime def update_schema(): """ download the latest configuration """ - if os.environ.get('INTELMQ_SKIP_INTERNET'): - return False # download the schema to a temp file (th, tmp) = tempfile.mkstemp(dir=VAR_STATE_PATH) - url = 'https://interchange.shadowserver.org/intelmq/v1/schema' - __config.logger.info("Attempting to download schema from %r" % url) - __config.logger.debug("Using temp file %r for the download." % tmp) + __config.logger.info("Attempting to download schema from %r", __config.schema_url) + __config.logger.debug("Using temp file %r for the download.", tmp) try: with create_request_session() as session: - with session.get(url, stream=True) as r: + with session.get(__config.schema_url, stream=True) as r: r.raise_for_status() with open(tmp, 'wb') as f: for chunk in r.iter_content(chunk_size=8192): f.write(chunk) except: - __config.logger.error("Failed to download %r" % url) + __config.logger.error("Failed to download %r", __config.schema_url) return False __config.logger.info("Download successful.") @@ -347,7 +352,7 @@ def update_schema(): new_version = schema['_meta']['date_created'] except: # leave tempfile behind for diagnosis - __config.logger.error("Failed to validate %r" % tmp) + __config.logger.error("Failed to validate %r", tmp) return False if os.path.exists(__config.schema_file): @@ -359,12 +364,12 @@ def update_schema(): if new_version != old_version: os.replace(__config.schema_file, ".".join([__config.schema_file, 'bak'])) except Exception as e: - __config.logger.error("Unable to replace schema file: %s" % str(e)) + __config.logger.error("Unable to replace schema file: %s", str(e)) return False if new_version != old_version: os.replace(tmp, __config.schema_file) - __config.logger.info("New schema version is %r." % new_version) + __config.logger.info("New schema version is %r.", new_version) return True else: os.unlink(tmp) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index 2e383a004e..fd9fa6b2cf 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -37,6 +37,7 @@ class ShadowserverParserBot(ParserBot): Parameters: auto_update (boolean): Enable automatic schema download + test_mode (boolean): Use test schema """ recover_line = ParserBot.recover_line_csv_dict @@ -47,9 +48,12 @@ class ShadowserverParserBot(ParserBot): _mode = None overwrite = False auto_update = False + test_mode = False def init(self): config.set_logger(self.logger) + if self.test_mode: + config.enable_test_mode(True) if self.auto_update: config.enable_auto_update(True) self.logger.debug("Feature 'auto_update' is enabled.") @@ -254,7 +258,7 @@ def run(cls, parsed_args=None): try: ctl = IntelMQController() for bot in runtime_conf: - if runtime_conf[bot]["module"] == __name__ and runtime_conf[bot]['parameters'].get('auto_update', True): + if runtime_conf[bot]["module"] == __name__: ctl.bot_reload(bot) except Exception as e: logger.error("Failed to signal bot: %r" % str(e)) diff --git a/intelmq/tests/bots/parsers/shadowserver/test_broken.py b/intelmq/tests/bots/parsers/shadowserver/test_broken.py index 3797f03cd5..54a85e7802 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_broken.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_broken.py @@ -46,6 +46,7 @@ def test_broken(self): """ Test a report which does not have valid fields """ + self.prepare_bot(parameters={'test_mode': True}) self.input_message = REPORT1 self.run_bot(allowed_error_count=1) self.assertLogMatches(pattern="Detected report's file name: 'test_smb'.", @@ -59,6 +60,7 @@ def test_half_broken(self): """ Test a report which does not have an optional field. """ + self.prepare_bot(parameters={'test_mode': True}) self.input_message = REPORT2 self.run_bot(allowed_warning_count=63) self.assertLogMatches(pattern="Detected report's file name: 'test_telnet'.", @@ -72,6 +74,7 @@ def test_no_config(self): """ Test a report which does not have a valid extra.file_name """ + self.prepare_bot(parameters={'test_mode': True}) self.input_message = REPORT3 self.run_bot(allowed_error_count=1) self.assertLogMatches(pattern="ValueError: Could not get a config for 'some_string', check the documentation.") @@ -80,6 +83,7 @@ def test_invalid_filename(self): """ Test a report which does not have a valid extra.file_name """ + self.prepare_bot(parameters={'test_mode': True}) self.input_message = REPORT4 self.run_bot(allowed_error_count=1) self.assertLogMatches(pattern="ValueError: Report's 'extra.file_name' '2020.wrong-filename.csv' is not valid.") @@ -89,6 +93,7 @@ def test_no_report_name(self): Test a report without file_name and no given feedname as parameter. Error message should be verbose. """ + self.prepare_bot(parameters={'test_mode': True}) self.run_bot(allowed_error_count=1) self.assertLogMatches(pattern="ValueError: No feedname given as parameter and the " "processed report has no 'extra.file_name'. " diff --git a/intelmq/tests/bots/parsers/shadowserver/test_mapping.py b/intelmq/tests/bots/parsers/shadowserver/test_mapping.py index d296dfdc26..b764de8274 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_mapping.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_mapping.py @@ -48,6 +48,7 @@ def test_changed_feed(self): Tests if the parser correctly re-detects the feed for the second received report #1493 """ + self.prepare_bot(parameters={'test_mode': True}) self.input_message = (EXAMPLE_TELNET, EXAMPLE_VNC) self.run_bot(iterations=2) diff --git a/intelmq/tests/bots/parsers/shadowserver/test_parameters.py b/intelmq/tests/bots/parsers/shadowserver/test_parameters.py index 677cd0319b..45a4a87354 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_parameters.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_parameters.py @@ -63,13 +63,14 @@ def set_bot(cls): def test_default(self): """ Test if feed name is not overwritten has been produced. """ + self.prepare_bot(parameters={'test_mode': True}) self.run_bot() for i, EVENT in enumerate(EVENTS): self.assertMessageEqual(i, EVENT) def test_overwrite_feed_name(self): """ Test if feed name is overwritten if asked to do so. """ - self.prepare_bot(parameters={'overwrite': True}) + self.prepare_bot(parameters={'test_mode': True, 'overwrite': True}) self.run_bot(prepare=False) for i, EVENT in enumerate(EVENTS): event = EVENT.copy() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py b/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py index 93d592d15c..aa6940061b 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py @@ -108,6 +108,7 @@ def set_bot(cls): def test_event(self): """ Test if correct Event has been produced. """ + self.prepare_bot(parameters={'test_mode': True}) self.run_bot() for i, EVENT in enumerate(EVENTS): self.assertMessageEqual(i, EVENT) diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py b/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py index a9be8a0a13..488f5a51a1 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py @@ -46,6 +46,7 @@ def set_bot(cls): def test_event(self): """ Test if the parser correctly detects and handles different report types. """ + self.prepare_bot(parameters={'test_mode': True}) self.input_message = [FIRST_REPORT, SECOND_REPORT] self.run_bot(iterations=2) self.assertLogMatches("Detected report's file name: 'test_smb'", diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py b/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py index df9cf25dca..b2499c589d 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py @@ -78,6 +78,7 @@ def set_bot(cls): def test_event(self): """ Test if correct Event has been produced. """ + self.prepare_bot(parameters={'test_mode': True}) self.run_bot() for i, EVENT in enumerate(EVENTS): self.assertMessageEqual(i, EVENT) From df6e62235001d64d23fb8f667f14962b0beb14e9 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Thu, 24 Aug 2023 16:26:59 +0000 Subject: [PATCH 33/76] Added VAR_STATE_PATH check. --- intelmq/bots/parsers/shadowserver/_config.py | 1 + .../parsers/shadowserver/test_download_schema.py | 13 +++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index 4bfadb9d98..6ffffdae86 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -117,6 +117,7 @@ def enable_test_mode(enable): else: __config.schema_active = __config.schema_file + def enable_auto_update(enable): """ Enable automatic schema update. """ __config.auto_update = enable diff --git a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py index e685876826..f9512ca98c 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py @@ -20,9 +20,10 @@ class TestShadowserverSchemaDownload(unittest.TestCase): def test_download(self): - schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json') - config.set_logger(utils.log('test-bot', log_path=None)) - if os.path.exists(schema_file): - os.unlink(schema_file) - self.assertEqual(True, config.update_schema()) - self.assertEqual(True, os.path.exists(schema_file)) + if os.path.isdir(VAR_STATE_PATH): + schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json') + config.set_logger(utils.log('test-bot', log_path=None)) + if os.path.exists(schema_file): + os.unlink(schema_file) + self.assertEqual(True, config.update_schema()) + self.assertEqual(True, os.path.exists(schema_file)) From 9195213959d2e0e2e464cb1359a0b69bb9d14f94 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Fri, 25 Aug 2023 15:37:51 +0000 Subject: [PATCH 34/76] Changes based on feedback 2023-08-25. --- CHANGELOG.md | 6 +- docs/user/bots.rst | 171 ++++++------------ intelmq/bots/parsers/shadowserver/README.md | 57 ------ intelmq/bots/parsers/shadowserver/_config.py | 10 +- .../shadowserver/test_download_schema.py | 8 +- 5 files changed, 72 insertions(+), 180 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cee9e520c..9fdc102258 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -118,20 +118,18 @@ CHANGELOG ### Bots #### Collectors -<<<<<<< HEAD - `intelmq.bots.collector.rt`: - restrict `python-rt` to be below version 3.0 due to introduced breaking changes, - added support for `Subject NOT LIKE` queries, - added support for multiple values in ticket subject queries. - `intelmq.bots.collectors.rsync`: Support for optional private key, relative time parsing for the source path, extra rsync parameters and strict host key checking (PR#2241 by Mateo Durante). - `intelmq.bots.collectors.shadowserver.collector_reports_api`: - - The 'json' option is no longer supported as the 'csv' option provides better performance. Please see intelmq/bots/parsers/shadowserver/README.md for a sample configuration. (PR#2372) + - The 'json' option is no longer supported as the 'csv' option provides better performance. #### Parsers - `intelmq.bots.parsers.shadowserver._config`: - Reset detected `feedname` at shutdown to re-detect the feedname on reloads (PR#2361 by @elsif2, fixes #2360). - - Switch to dynamic configuration to decouple report schema changes from IntelMQ releases. Please see intelmq/bots/parsers/shadowserver/README.md for details. (PR#2372) -- `intelmq.bots.parsers.shadowserver._config`: + - Switch to dynamic configuration to decouple report schema changes from IntelMQ releases. - Added 'IPv6-Vulnerable-Exchange' alias and 'Accessible-WS-Discovery-Service' report. (PR#2338) - Removed unused `p0f_genre` and `p0f_detail` from the 'DNS-Open-Resolvers' report. (PR#2338) - Added 'Accessible-SIP' report. (PR#2348) diff --git a/docs/user/bots.rst b/docs/user/bots.rst index 2fbe27df8e..a758ff8ad8 100644 --- a/docs/user/bots.rst +++ b/docs/user/bots.rst @@ -673,6 +673,23 @@ The resulting reports contain the following special field: * `extra.file_name`: The name of the downloaded file, with fixed filename extension. The API returns file names with the extension `.csv`, although the files are JSON, not CSV. Therefore, for clarity and better error detection in the parser, the file name in `extra.file_name` uses `.json` as extension. +**Sample configuration** + +.. code-block:: yaml + + shadowserver-collector: + description: Our bot responsible for getting reports from Shadowserver + enabled: true + group: Collector + module: intelmq.bots.collectors.shadowserver.collector_reports_api + name: Shadowserver_Collector + parameters: + destination_queues: + _default: [shadowserver-parser-queue] + file_format: csv + api_key: "$API_KEY_received_from_the_shadowserver_foundation" + secret: "$SECRET_received_from_the_shadowserver_foundation" + run_mode: continuous .. _intelmq.bots.collectors.shodan.collector_stream: @@ -1557,17 +1574,15 @@ This does not affect URLs which already include the scheme. .. _intelmq.bots.parsers.shadowserver.parser: -.. _intelmq.bots.parsers.shadowserver.parser_json: Shadowserver ^^^^^^^^^^^^ -There are two Shadowserver parsers, one for data in ``CSV`` format (``intelmq.bots.parsers.shadowserver.parser``) and one for data in ``JSON`` format (``intelmq.bots.parsers.shadowserver.parser_json``). -The latter was added in IntelMQ 2.3 and is meant to be used together with the Shadowserver API collector. +The Shadowserver parser operates on ``CSV`` formatted data. **Information** -* `name:` `intelmq.bots.parsers.shadowserver.parser` (for CSV data) or `intelmq.bots.parsers.shadowserver.parser_json` (for JSON data) +* `name:` `intelmq.bots.parsers.shadowserver.parser` * `public:` yes * `description:` Parses different reports from Shadowserver. @@ -1603,107 +1618,45 @@ A list of possible feeds can be found in the table below in the column "feed nam **Supported reports** -These are the supported feed name and their corresponding file name for automatic detection: - - ======================================= ========================= - feed name file name - ======================================= ========================= - Accessible-ADB `scan_adb` - Accessible-AFP `scan_afp` - Accessible-AMQP `scan_amqp` - Accessible-ARD `scan_ard` - Accessible-Cisco-Smart-Install `cisco_smart_install` - Accessible-CoAP `scan_coap` - Accessible-CWMP `scan_cwmp` - Accessible-MS-RDPEUDP `scan_msrdpeudp` - Accessible-FTP `scan_ftp` - Accessible-Hadoop `scan_hadoop` - Accessible-HTTP `scan_http` - Accessible-Radmin `scan_radmin` - Accessible-RDP `scan_rdp` - Accessible-Rsync `scan_rsync` - Accessible-SMB `scan_smb` - Accessible-Telnet `scan_telnet` - Accessible-Ubiquiti-Discovery-Service `scan_ubiquiti` - Accessible-VNC `scan_vnc` - Blacklisted-IP (deprecated) `blacklist` - Blocklist `blocklist` - Compromised-Website `compromised_website` - Device-Identification IPv4 / IPv6 `device_id`/`device_id6` - DNS-Open-Resolvers `scan_dns` - Honeypot-Amplification-DDoS-Events `event4_honeypot_ddos_amp` - Honeypot-Brute-Force-Events `event4_honeypot_brute_force` - Honeypot-Darknet `event4_honeypot_darknet` - Honeypot-HTTP-Scan `event4_honeypot_http_scan` - HTTP-Scanners `hp_http_scan` - ICS-Scanners `hp_ics_scan` - IP-Spoofer-Events `event4_ip_spoofer` - Microsoft-Sinkhole-Events IPv4 `event4_microsoft_sinkhole` - Microsoft-Sinkhole-Events-HTTP IPv4 `event4_microsoft_sinkhole_http` - NTP-Monitor `scan_ntpmonitor` - NTP-Version `scan_ntp` - Open-Chargen `scan_chargen` - Open-DB2-Discovery-Service `scan_db2` - Open-Elasticsearch `scan_elasticsearch` - Open-IPMI `scan_ipmi` - Open-IPP `scan_ipp` - Open-LDAP `scan_ldap` - Open-LDAP-TCP `scan_ldap_tcp` - Open-mDNS `scan_mdns` - Open-Memcached `scan_memcached` - Open-MongoDB `scan_mongodb` - Open-MQTT `scan_mqtt` - Open-MSSQL `scan_mssql` - Open-NATPMP `scan_nat_pmp` - Open-NetBIOS-Nameservice `scan_netbios` - Open-Netis `netis_router` - Open-Portmapper `scan_portmapper` - Open-QOTD `scan_qotd` - Open-Redis `scan_redis` - Open-SNMP `scan_snmp` - Open-SSDP `scan_ssdp` - Open-TFTP `scan_tftp` - Open-XDMCP `scan_xdmcp` - Outdated-DNSSEC-Key `outdated_dnssec_key` - Outdated-DNSSEC-Key-IPv6 `outdated_dnssec_key_v6` - Sandbox-URL `cwsandbox_url` - Sinkhole-DNS `sinkhole_dns` - Sinkhole-Events `event4_sinkhole`/`event6_sinkhole` - Sinkhole-Events IPv4 `event4_sinkhole` - Sinkhole-Events IPv6 `event6_sinkhole` - Sinkhole-HTTP-Events `event4_sinkhole_http`/`event6_sinkhole_http` - Sinkhole-HTTP-Events IPv4 `event4_sinkhole_http` - Sinkhole-HTTP-Events IPv6 `event6_sinkhole_http` - Sinkhole-Events-HTTP-Referer `event4_sinkhole_http_referer`/`event6_sinkhole_http_referer` - Sinkhole-Events-HTTP-Referer IPv4 `event4_sinkhole_http_referer` - Sinkhole-Events-HTTP-Referer IPv6 `event6_sinkhole_http_referer` - Spam-URL `spam_url` - SSL-FREAK-Vulnerable-Servers `scan_ssl_freak` - SSL-POODLE-Vulnerable-Servers `scan_ssl_poodle`/`scan6_ssl_poodle` - Vulnerable-Exchange-Server `*` `scan_exchange` - Vulnerable-ISAKMP `scan_isakmp` - Vulnerable-HTTP `scan_http` - Vulnerable-SMTP `scan_smtp_vulnerable` - ======================================= ========================= - -`*` This report can also contain data on active webshells (column `tag` is `exchange;webshell`), and are therefore not only vulnerable but also actively infected. - -In addition, the following legacy reports are supported: - - =========================== =================================================== ======================== - feed name successor feed name file name - =========================== =================================================== ======================== - Amplification-DDoS-Victim Honeypot-Amplification-DDoS-Events ``ddos_amplification`` - CAIDA-IP-Spoofer IP-Spoofer-Events ``caida_ip_spoofer`` - Darknet Honeypot-Darknet ``darknet`` - Drone Sinkhole-Events ``botnet_drone`` - Drone-Brute-Force Honeypot-Brute-Force-Events, Sinkhole-HTTP-Events ``drone_brute_force`` - Microsoft-Sinkhole Sinkhole-HTTP-Events ``microsoft_sinkhole`` - Sinkhole-HTTP-Drone Sinkhole-HTTP-Events ``sinkhole_http_drone`` - IPv6-Sinkhole-HTTP-Drone Sinkhole-HTTP-Events ``sinkhole6_http`` - =========================== =================================================== ======================== - -More information on these legacy reports can be found in `Changes in Sinkhole and Honeypot Report Types and Formats `_. +The report configuration is stored in a `shadowserver-schema.json` file downloaded from https://interchange.shadowserver.org/intelmq/v1/schema. + +The parser will attempt to download a schema update on startup when the *auto_update* option is enabled. + +Schema downloads can also be scheduled as a cron job: + +.. code-block:: bash + + 02 01 * * * intelmq.bots.parsers.shadowserver.parser --update-schema + + +For air-gapped systems automation will be required to download and copy the file to VAR_STATE_PATH/shadowserver-schema.json. + +The parser will automatically reload the configuration when the file changes. + +**Schema contract** + +Once set in the schema, the `classification.identifier`, `classification.taxonomy`, and `classification.type` fields will remain static for a specific report. + +Report fields will not be removed from a report. + +The schema revision history is maintained at https://github.com/The-Shadowserver-Foundation/report_schema/. + +**Sample configuration** + +.. code-block:: yaml + + shadowserver-parser: + bot_id: shadowserver-parser + name: Shadowserver Parser + enabled: true + group: Parser + groupname: parsers + module: intelmq.bots.parsers.shadowserver.parser + parameters: + destination_queues: + _default: [file-output-queue] + auto_update: true + run_mode: continuous **Development** @@ -1715,14 +1668,6 @@ The parser consists of two files: Both files are required for the parser to work properly. -**Add new Feedformats** - -Add a new feed format and conversions if required to the file -``_config.py``. Don't forget to update the ``mapping`` dict. -It is required to look up the correct configuration. - -Look at the documentation in the bot's ``_config.py`` file for more information. - .. _intelmq.bots.parsers.shodan.parser: diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md index 4969acb6d0..eb0ddfb4a7 100644 --- a/intelmq/bots/parsers/shadowserver/README.md +++ b/intelmq/bots/parsers/shadowserver/README.md @@ -7,60 +7,3 @@ This module is maintained by [The Shadowserver Foundation](https://www.shadowser Please contact intelmq@shadowserver.org with any issues or concerns. -The report configuration is now stored in a _shadowserver-schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/v1/schema. - -The parser will attempt to download a schema update on startup when the *auto_update* option is enabled. - -Schema downloads can also be scheduled as a cron job: - -``` -02 01 * * * intelmq.bots.parsers.shadowserver.parser --update-schema -``` - -For air-gapped systems automation will be required to download and copy the file to VAR_STATE_PATH/shadowserver-schema.json. - -The parser will automatically reload the configuration when the file changes. - - -## Schema contract - -Once set the `classification.identifier`, `classification.taxonomy`, and `classification.type` fields will remain static. - -Once set report fields will not be deleted. - -The schema revision history is maintained at https://github.com/The-Shadowserver-Foundation/report_schema/. - - -## Sample configuration: - -``` -shadowserver-collector: - description: Our bot responsible for getting reports from Shadowserver - enabled: true - group: Collector - module: intelmq.bots.collectors.shadowserver.collector_reports_api - name: Shadowserver_Collector - parameters: - destination_queues: - _default: [shadowserver-parser-queue] - file_format: csv - api_key: "$API_KEY_received_from_the_shadowserver_foundation" - secret: "$SECRET_received_from_the_shadowserver_foundation" - run_mode: continuous -``` - -``` -shadowserver-parser: - bot_id: shadowserver-parser - name: Shadowserver Parser - enabled: true - group: Parser - groupname: parsers - module: intelmq.bots.parsers.shadowserver.parser - parameters: - destination_queues: - _default: [file-output-queue] - auto_update: true - run_mode: continuous -``` - diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index 6ffffdae86..279093dfe3 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -95,6 +95,7 @@ class __Container: __config = __Container() +__config.var_state_path = VAR_STATE_PATH __config.schema_url = 'https://interchange.shadowserver.org/intelmq/v1/schema' __config.schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json') __config.schema_base = os.path.join(os.path.dirname(__file__), 'schema.json.test') @@ -328,7 +329,7 @@ def update_schema(): """ download the latest configuration """ # download the schema to a temp file - (th, tmp) = tempfile.mkstemp(dir=VAR_STATE_PATH) + (th, tmp) = tempfile.mkstemp(dir=__config.var_state_path) __config.logger.info("Attempting to download schema from %r", __config.schema_url) __config.logger.debug("Using temp file %r for the download.", tmp) try: @@ -376,3 +377,10 @@ def update_schema(): os.unlink(tmp) return False + + +def prepare_update_schema_test(path): + """ Reconfigure internal settings to perform a schema update test. """ + __config.var_state_path = path + __config.schema_file = os.path.join(path, 'shadowserver-schema.json') + return __config.schema_file diff --git a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py index f9512ca98c..5246e6bb67 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py @@ -10,8 +10,8 @@ import unittest import os +import tempfile import logging -from intelmq import VAR_STATE_PATH import intelmq.bots.parsers.shadowserver._config as config import intelmq.lib.utils as utils import intelmq.lib.test as test @@ -20,10 +20,8 @@ class TestShadowserverSchemaDownload(unittest.TestCase): def test_download(self): - if os.path.isdir(VAR_STATE_PATH): - schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json') + with tempfile.TemporaryDirectory() as tmp_dir: + schema_file = config.prepare_update_schema_test(tmp_dir) config.set_logger(utils.log('test-bot', log_path=None)) - if os.path.exists(schema_file): - os.unlink(schema_file) self.assertEqual(True, config.update_schema()) self.assertEqual(True, os.path.exists(schema_file)) From cc48565bb325c26a7e92690185474525f48c04e5 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Fri, 25 Aug 2023 15:51:38 +0000 Subject: [PATCH 35/76] Added INTELMQ_SKIP_INTERNET check --- .../bots/parsers/shadowserver/test_download_schema.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py index 5246e6bb67..203a3c0b12 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py @@ -20,8 +20,9 @@ class TestShadowserverSchemaDownload(unittest.TestCase): def test_download(self): - with tempfile.TemporaryDirectory() as tmp_dir: - schema_file = config.prepare_update_schema_test(tmp_dir) - config.set_logger(utils.log('test-bot', log_path=None)) - self.assertEqual(True, config.update_schema()) - self.assertEqual(True, os.path.exists(schema_file)) + if not os.environ.get('INTELMQ_SKIP_INTERNET'): + with tempfile.TemporaryDirectory() as tmp_dir: + schema_file = config.prepare_update_schema_test(tmp_dir) + config.set_logger(utils.log('test-bot', log_path=None)) + self.assertEqual(True, config.update_schema()) + self.assertEqual(True, os.path.exists(schema_file)) From 16daee468f62209459647f242d938dac56fc40de Mon Sep 17 00:00:00 2001 From: elsif2 Date: Fri, 25 Aug 2023 16:11:21 +0000 Subject: [PATCH 36/76] Added debug logging for CI test. --- intelmq/bots/parsers/shadowserver/_config.py | 3 ++- .../tests/bots/parsers/shadowserver/test_download_schema.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index 279093dfe3..d573d12c61 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -339,8 +339,9 @@ def update_schema(): with open(tmp, 'wb') as f: for chunk in r.iter_content(chunk_size=8192): f.write(chunk) - except: + except Exception as e: __config.logger.error("Failed to download %r", __config.schema_url) + __config.logger.debug(str(e)) return False __config.logger.info("Download successful.") diff --git a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py index 203a3c0b12..abcd0ca2a4 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py @@ -23,6 +23,6 @@ def test_download(self): if not os.environ.get('INTELMQ_SKIP_INTERNET'): with tempfile.TemporaryDirectory() as tmp_dir: schema_file = config.prepare_update_schema_test(tmp_dir) - config.set_logger(utils.log('test-bot', log_path=None)) + config.set_logger(utils.log('test-bot', log_path=None, log_level=logging.DEBUG)) self.assertEqual(True, config.update_schema()) self.assertEqual(True, os.path.exists(schema_file)) From f102f2c0b7eef245db04c39aba28517090a93129 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Fri, 25 Aug 2023 18:47:54 +0000 Subject: [PATCH 37/76] Refactored test_download_schema to utilize mocking. --- intelmq/bots/parsers/shadowserver/parser.py | 6 ++++ .../shadowserver/test_download_schema.py | 30 ++++++++++++------- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index fd9fa6b2cf..48cbba901a 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -23,6 +23,7 @@ import copy import re import os +import tempfile from intelmq.lib.bot import ParserBot from intelmq.lib.exceptions import InvalidKey, InvalidValue @@ -265,5 +266,10 @@ def run(cls, parsed_args=None): else: super().run(parsed_args=parsed_args) + def test_update_schema(cls): + with tempfile.TemporaryDirectory() as tmp_dir: + schema_file = config.prepare_update_schema_test(tmp_dir) + return config.update_schema() + BOT = ShadowserverParserBot diff --git a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py index abcd0ca2a4..abf27a5bd4 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py @@ -8,21 +8,29 @@ """ -import unittest -import os -import tempfile import logging -import intelmq.bots.parsers.shadowserver._config as config +import unittest +import unittest.mock as mock +from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot import intelmq.lib.utils as utils import intelmq.lib.test as test + @test.skip_internet() -class TestShadowserverSchemaDownload(unittest.TestCase): +class TestShadowserverSchemaDownload(test.BotTestCase, unittest.TestCase): + + @classmethod + def set_bot(cls): + cls.bot_reference = ShadowserverParserBot + cls.sysconfig = {"logging_level": "DEBUG"} def test_download(self): - if not os.environ.get('INTELMQ_SKIP_INTERNET'): - with tempfile.TemporaryDirectory() as tmp_dir: - schema_file = config.prepare_update_schema_test(tmp_dir) - config.set_logger(utils.log('test-bot', log_path=None, log_level=logging.DEBUG)) - self.assertEqual(True, config.update_schema()) - self.assertEqual(True, os.path.exists(schema_file)) + self.prepare_bot(prepare_source_queue=False, parameters={'test_mode': True}) + result = False + with mock.patch('intelmq.lib.utils.load_configuration', new=self.mocked_config): + with mock.patch('intelmq.lib.utils.log', self.get_mocked_logger(self.logger)): + self.log_stream.truncate(0) + result = self.bot.test_update_schema() + self.bot.stop(exitcode=0) + print(self.log_stream.getvalue()) + self.assertEqual(True, result) From b103282cb083ba586b40559606d47e33ac8c5b86 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Mon, 28 Aug 2023 14:18:22 +0000 Subject: [PATCH 38/76] Added docstring for test_update_schema(). --- intelmq/bots/parsers/shadowserver/parser.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index 48cbba901a..4485a26020 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -267,6 +267,13 @@ def run(cls, parsed_args=None): super().run(parsed_args=parsed_args) def test_update_schema(cls): + """ + Test schema download to a temporary directory. + + This is necessary as the request session requires mocking in order to function. + + Returns True on success. + """ with tempfile.TemporaryDirectory() as tmp_dir: schema_file = config.prepare_update_schema_test(tmp_dir) return config.update_schema() From 356b956a3ce79eaa723c774bc54eafa149a5b528 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 29 Aug 2023 14:09:33 +0000 Subject: [PATCH 39/76] Removed logging output. --- intelmq/tests/bots/parsers/shadowserver/test_download_schema.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py index abf27a5bd4..84922bf176 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py @@ -29,8 +29,6 @@ def test_download(self): result = False with mock.patch('intelmq.lib.utils.load_configuration', new=self.mocked_config): with mock.patch('intelmq.lib.utils.log', self.get_mocked_logger(self.logger)): - self.log_stream.truncate(0) result = self.bot.test_update_schema() self.bot.stop(exitcode=0) - print(self.log_stream.getvalue()) self.assertEqual(True, result) From c72d553fdab8546dda0b669ab1557ace6745e644 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Thu, 31 Aug 2023 20:52:17 +0000 Subject: [PATCH 40/76] Removed the assertion regarding report fields. --- docs/user/bots.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/user/bots.rst b/docs/user/bots.rst index a758ff8ad8..ae17cbf556 100644 --- a/docs/user/bots.rst +++ b/docs/user/bots.rst @@ -1637,8 +1637,6 @@ The parser will automatically reload the configuration when the file changes. Once set in the schema, the `classification.identifier`, `classification.taxonomy`, and `classification.type` fields will remain static for a specific report. -Report fields will not be removed from a report. - The schema revision history is maintained at https://github.com/The-Shadowserver-Foundation/report_schema/. **Sample configuration** From 3b60c2f9699f576ecadb262fb2ad592112a9a69e Mon Sep 17 00:00:00 2001 From: elsif2 Date: Mon, 16 Oct 2023 17:57:46 +0000 Subject: [PATCH 41/76] Skip and log a warning message for fields not in the IDF. --- intelmq/bots/parsers/shadowserver/parser.py | 5 ++- .../parsers/shadowserver/schema.json.test | 37 +++++++++++++++++++ .../bots/parsers/shadowserver/test_broken.py | 15 ++++++++ 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index 4485a26020..cfa343138d 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -157,7 +157,10 @@ def parse_line(self, row, report): raise if value is not None: - event.add(intelmqkey, value) + try: + event.add(intelmqkey, value) + except InvalidKey: + self.logger.warning('Key not found in IDF %r.', intelmqkey) fields.remove(shadowkey) # Now add optional fields. diff --git a/intelmq/bots/parsers/shadowserver/schema.json.test b/intelmq/bots/parsers/shadowserver/schema.json.test index 2cfb8bb1d3..932b8df03b 100644 --- a/intelmq/bots/parsers/shadowserver/schema.json.test +++ b/intelmq/bots/parsers/shadowserver/schema.json.test @@ -176,5 +176,42 @@ "convert_int" ] ] + }, + "test_afs" : { + "constant_fields" : { + "classification.identifier" : "test-afs", + "classification.taxonomy" : "vulnerable", + "classification.type" : "vulnerable-system", + "protocol.application" : "afs" + }, + "feed_name" : "Test-Accessible-AFS", + "file_name" : "test_afs", + "required_fields" : [ + [ + "time.source", + "timestamp", + "add_UTC_to_timestamp" + ], + [ + "source.ip", + "ip", + "validate_ip" + ], + [ + "source.port", + "port", + "convert_int" + ], + [ + "not_in_idf", + "severity" + ] + ], + "optional_fields" : [ + [ + "protocol.transport", + "protocol" + ] + ] } } diff --git a/intelmq/tests/bots/parsers/shadowserver/test_broken.py b/intelmq/tests/bots/parsers/shadowserver/test_broken.py index 54a85e7802..f1af08e586 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_broken.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_broken.py @@ -30,6 +30,11 @@ "time.observation": "2015-01-01T00:00:00+00:00", "extra.file_name": "2020.wrong-filename.csv", } +REPORT5 = {"raw": utils.base64_encode('timestamp,ip,protocol,port,severity\n2018-08-01T00:00:00+00,127.0.0.1,tcp,7000,critical'), + "__type": "Report", + "time.observation": "2023-10-16T00:00:00+00:00", + "extra.file_name": "2023-10-16-test_afs-test-test.csv", + } class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): @@ -100,6 +105,16 @@ def test_no_report_name(self): "Ensure that at least one is given. " "Also have a look at the documentation of the bot.") + def test_field_not_in_idf(self): + """ + Test a report that contains a field mapping not in the IDF. + Error message should be verbose. + """ + self.prepare_bot(parameters={'test_mode': True}) + self.input_message = REPORT5 + self.run_bot(allowed_error_count=0, allowed_warning_count=1) + self.assertLogMatches(pattern="Key not found in IDF", levelname="WARNING") + if __name__ == '__main__': # pragma: no cover unittest.main() From 473f6a64c671d8910ec427daf5ba791eee82887a Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 11 Apr 2023 23:53:24 +0000 Subject: [PATCH 42/76] remove obsolete tests and data --- .../shadowserver/scan_rdpeudp.csv.license | 2 - .../parsers/shadowserver/test_blocklist.py | 103 ------- .../shadowserver/test_compromised_website.py | 88 ------ .../parsers/shadowserver/test_device_id.py | 116 -------- .../test_event4_ddos_participant.py | 131 --------- .../test_event4_honeypot_darknet.py | 106 ------- .../shadowserver/test_event4_honeypot_ddos.py | 148 ---------- .../test_event4_honeypot_ddos_target.py | 150 ---------- .../test_event4_honeypot_http_scan.py | 109 -------- .../shadowserver/test_event4_ip_spoofer.py | 182 ------------ .../test_event4_microsoft_sinkhole.py | 135 --------- .../test_event4_microsoft_sinkhole_http.py | 202 -------------- .../shadowserver/test_event4_sinkhole.py | 73 ----- .../shadowserver/test_event4_sinkhole_dns.py | 127 --------- .../shadowserver/test_event4_sinkhole_http.py | 189 ------------- .../test_event4_sinkhole_http_referer.py | 213 --------------- .../shadowserver/test_event6_sinkhole_http.py | 146 ---------- .../shadowserver/test_honeypot_brute_force.py | 72 ----- .../shadowserver/test_honeypot_ddos_amp.py | 91 ------ .../parsers/shadowserver/test_malware_url.py | 107 -------- .../parsers/shadowserver/test_phish_url.py | 106 ------- .../test_population_http_proxy.py | 130 --------- .../parsers/shadowserver/test_sandbox_conn.py | 99 ------- .../parsers/shadowserver/test_sandbox_dns.py | 95 ------- .../parsers/shadowserver/test_sandbox_url.py | 104 ------- .../parsers/shadowserver/test_scan_adb.py | 98 ------- .../parsers/shadowserver/test_scan_afp.py | 106 ------- .../parsers/shadowserver/test_scan_amqp.py | 144 ---------- .../parsers/shadowserver/test_scan_ard.py | 111 -------- .../parsers/shadowserver/test_scan_chargen.py | 110 -------- .../test_scan_cisco_smart_install.py | 82 ------ .../parsers/shadowserver/test_scan_coap.py | 121 -------- .../parsers/shadowserver/test_scan_couchdb.py | 128 --------- .../parsers/shadowserver/test_scan_cwmp.py | 103 ------- .../parsers/shadowserver/test_scan_db2.py | 91 ------ .../shadowserver/test_scan_ddos_middlebox.py | 119 -------- .../parsers/shadowserver/test_scan_dns.py | 91 ------ .../parsers/shadowserver/test_scan_docker.py | 159 ----------- .../test_scan_dvr_dhcpdiscover.py | 178 ------------ .../shadowserver/test_scan_elasticsearch.py | 126 --------- .../shadowserver/test_scan_exchange.py | 149 ---------- .../parsers/shadowserver/test_scan_ftp.py | 120 -------- .../parsers/shadowserver/test_scan_hadoop.py | 94 ------- .../parsers/shadowserver/test_scan_http.py | 100 ------- .../shadowserver/test_scan_http_proxy.py | 118 -------- .../shadowserver/test_scan_http_vulnerable.py | 125 --------- .../parsers/shadowserver/test_scan_ics.py | 125 --------- .../parsers/shadowserver/test_scan_ipmi.py | 106 ------- .../parsers/shadowserver/test_scan_ipp.py | 79 ------ .../parsers/shadowserver/test_scan_isakmp.py | 105 ------- .../shadowserver/test_scan_kubernetes.py | 214 --------------- .../shadowserver/test_scan_ldap_tcp.py | 154 ----------- .../shadowserver/test_scan_ldap_udp.py | 162 ----------- .../parsers/shadowserver/test_scan_mdns.py | 127 --------- .../shadowserver/test_scan_memcached.py | 130 --------- .../parsers/shadowserver/test_scan_mongodb.py | 103 ------- .../parsers/shadowserver/test_scan_mqtt.py | 89 ------ .../shadowserver/test_scan_mqtt_anon.py | 173 ------------ .../parsers/shadowserver/test_scan_mssql.py | 123 --------- .../parsers/shadowserver/test_scan_mysql.py | 258 ------------------ .../parsers/shadowserver/test_scan_nat_pmp.py | 116 -------- .../parsers/shadowserver/test_scan_netbios.py | 121 -------- .../shadowserver/test_scan_netis_router.py | 107 -------- .../parsers/shadowserver/test_scan_ntp.py | 161 ----------- .../shadowserver/test_scan_ntpmonitor.py | 108 -------- .../shadowserver/test_scan_portmapper.py | 120 -------- .../shadowserver/test_scan_postgres.py | 199 -------------- .../parsers/shadowserver/test_scan_qotd.py | 119 -------- .../parsers/shadowserver/test_scan_quic.py | 118 -------- .../parsers/shadowserver/test_scan_radmin.py | 236 ---------------- .../parsers/shadowserver/test_scan_rdp.py | 117 -------- .../parsers/shadowserver/test_scan_rdpeudp.py | 109 -------- .../parsers/shadowserver/test_scan_redis.py | 107 -------- .../parsers/shadowserver/test_scan_rsync.py | 116 -------- .../parsers/shadowserver/test_scan_sip.py | 124 --------- .../parsers/shadowserver/test_scan_slp.py | 137 ---------- .../parsers/shadowserver/test_scan_smb.py | 124 --------- .../shadowserver/test_scan_smb_json.py | 123 --------- .../shadowserver/test_scan_smtp_vulnerable.py | 92 ------- .../parsers/shadowserver/test_scan_snmp.py | 120 -------- .../parsers/shadowserver/test_scan_socks.py | 107 -------- .../parsers/shadowserver/test_scan_ssdp.py | 136 --------- .../parsers/shadowserver/test_scan_ssh.py | 182 ------------ .../parsers/shadowserver/test_scan_ssl.py | 218 --------------- .../shadowserver/test_scan_ssl_freak.py | 136 --------- .../shadowserver/test_scan_ssl_poodle.py | 91 ------ .../parsers/shadowserver/test_scan_stun.py | 146 ---------- .../shadowserver/test_scan_synfulknock.py | 117 -------- .../parsers/shadowserver/test_scan_telnet.py | 87 ------ .../parsers/shadowserver/test_scan_tftp.py | 121 -------- .../shadowserver/test_scan_ubiquiti.py | 124 --------- .../parsers/shadowserver/test_scan_vnc.py | 86 ------ .../shadowserver/test_scan_ws_discovery.py | 119 -------- .../parsers/shadowserver/test_scan_xdmcp.py | 117 -------- .../bots/parsers/shadowserver/test_special.py | 106 ------- .../parsers/shadowserver/test_testdata.py | 81 ------ .../shadowserver/testdata/blocklist.csv | 4 - .../testdata/blocklist.csv.license | 2 - .../testdata/botnet_drone.csv.license | 2 - .../testdata/caida_ip_spoofer.csv.license | 2 - .../testdata/compromised_website.csv | 4 - .../testdata/compromised_website.csv.license | 2 - .../shadowserver/testdata/darknet.csv.license | 2 - .../testdata/ddos_amplification.csv.license | 2 - .../shadowserver/testdata/device_id.csv | 4 - .../testdata/device_id.csv.license | 2 - .../testdata/drone_brute_force.csv.license | 2 - .../testdata/event4_ddos_participant.csv | 4 - .../event4_ddos_participant.csv.license | 2 - .../testdata/event4_honeypot_brute_force.csv | 7 - .../event4_honeypot_brute_force.csv.license | 2 - .../testdata/event4_honeypot_darknet.csv | 9 - .../event4_honeypot_darknet.csv.license | 2 - .../testdata/event4_honeypot_ddos.csv | 4 - .../testdata/event4_honeypot_ddos.csv.license | 2 - .../testdata/event4_honeypot_ddos_amp.csv | 6 - .../event4_honeypot_ddos_amp.csv.license | 2 - .../testdata/event4_honeypot_ddos_target.csv | 4 - .../event4_honeypot_ddos_target.csv.license | 2 - .../testdata/event4_honeypot_http_scan.csv | 3 - .../event4_honeypot_http_scan.csv.license | 2 - .../testdata/event4_ip_spoofer.csv | 7 - .../testdata/event4_ip_spoofer.csv.license | 2 - .../testdata/event4_microsoft_sinkhole.csv | 7 - .../event4_microsoft_sinkhole.csv.license | 2 - .../event4_microsoft_sinkhole_http.csv | 6 - ...event4_microsoft_sinkhole_http.csv.license | 2 - .../shadowserver/testdata/event4_sinkhole.csv | 4 - .../testdata/event4_sinkhole.csv.license | 2 - .../testdata/event4_sinkhole_dns.csv | 4 - .../testdata/event4_sinkhole_dns.csv.license | 2 - .../testdata/event4_sinkhole_http.csv | 6 - .../testdata/event4_sinkhole_http.csv.license | 2 - .../testdata/event4_sinkhole_http_referer.csv | 6 - .../event4_sinkhole_http_referer.csv.license | 2 - .../testdata/event6_sinkhole_http.csv | 4 - .../testdata/event6_sinkhole_http.csv.license | 2 - .../testdata/hp_http_scan.csv.license | 2 - .../testdata/hp_ics_scan.csv.license | 2 - .../shadowserver/testdata/malware_url.csv | 4 - .../testdata/malware_url.csv.license | 2 - .../testdata/outdated_dnssec_key.csv.license | 2 - .../shadowserver/testdata/phish_url.csv | 4 - .../testdata/phish_url.csv.license | 2 - .../testdata/population_http_proxy.csv | 4 - .../population_http_proxy.csv.license | 2 - .../shadowserver/testdata/sandbox_conn.csv | 4 - .../testdata/sandbox_conn.csv.license | 2 - .../shadowserver/testdata/sandbox_dns.csv | 4 - .../testdata/sandbox_dns.csv.license | 2 - .../shadowserver/testdata/sandbox_url.csv | 4 - .../testdata/sandbox_url.csv.license | 2 - .../shadowserver/testdata/scan_adb.csv | 3 - .../testdata/scan_adb.csv.license | 2 - .../shadowserver/testdata/scan_afp.csv | 3 - .../testdata/scan_afp.csv.license | 2 - .../shadowserver/testdata/scan_amqp.csv | 4 - .../testdata/scan_amqp.csv.license | 2 - .../shadowserver/testdata/scan_ard.csv | 4 - .../testdata/scan_ard.csv.license | 2 - .../shadowserver/testdata/scan_chargen.csv | 4 - .../testdata/scan_chargen.csv.license | 2 - .../testdata/scan_cisco_smart_install.csv | 3 - .../scan_cisco_smart_install.csv.license | 2 - .../shadowserver/testdata/scan_coap.csv | 4 - .../testdata/scan_coap.csv.license | 2 - .../shadowserver/testdata/scan_couchdb.csv | 4 - .../testdata/scan_couchdb.csv.license | 2 - .../shadowserver/testdata/scan_cwmp.csv | 3 - .../testdata/scan_cwmp.csv.license | 2 - .../shadowserver/testdata/scan_db2.csv | 3 - .../testdata/scan_db2.csv.license | 2 - .../testdata/scan_ddos_middlebox.csv | 4 - .../testdata/scan_ddos_middlebox.csv.license | 2 - .../shadowserver/testdata/scan_dns.csv | 101 ------- .../testdata/scan_dns.csv.license | 2 - .../shadowserver/testdata/scan_docker.csv | 4 - .../testdata/scan_docker.csv.license | 2 - .../testdata/scan_dvr_dhcpdiscover.csv | 4 - .../scan_dvr_dhcpdiscover.csv.license | 2 - .../testdata/scan_elasticsearch.csv | 4 - .../testdata/scan_elasticsearch.csv.license | 2 - .../shadowserver/testdata/scan_exchange.csv | 8 - .../testdata/scan_exchange.csv.license | 2 - .../shadowserver/testdata/scan_ftp.csv | 3 - .../testdata/scan_ftp.csv.license | 2 - .../shadowserver/testdata/scan_hadoop.csv | 3 - .../testdata/scan_hadoop.csv.license | 2 - .../shadowserver/testdata/scan_http.csv | 3 - .../testdata/scan_http.csv.license | 2 - .../shadowserver/testdata/scan_http_proxy.csv | 4 - .../testdata/scan_http_proxy.csv.license | 2 - .../testdata/scan_http_vulnerable.csv | 4 - .../testdata/scan_http_vulnerable.csv.license | 2 - .../shadowserver/testdata/scan_ics.csv | 4 - .../testdata/scan_ics.csv.license | 2 - .../shadowserver/testdata/scan_ipmi.csv | 96 ------- .../testdata/scan_ipmi.csv.license | 2 - .../shadowserver/testdata/scan_ipp.csv | 2 - .../testdata/scan_ipp.csv.license | 2 - .../shadowserver/testdata/scan_isakmp.csv | 3 - .../testdata/scan_isakmp.csv.license | 2 - .../shadowserver/testdata/scan_kubernetes.csv | 4 - .../testdata/scan_kubernetes.csv.license | 2 - .../shadowserver/testdata/scan_ldap_tcp.csv | 4 - .../testdata/scan_ldap_tcp.csv.license | 2 - .../shadowserver/testdata/scan_ldap_udp.csv | 4 - .../testdata/scan_ldap_udp.csv.license | 2 - .../shadowserver/testdata/scan_mdns.csv | 4 - .../testdata/scan_mdns.csv.license | 2 - .../shadowserver/testdata/scan_memcached.csv | 4 - .../testdata/scan_memcached.csv.license | 2 - .../shadowserver/testdata/scan_mongodb.csv | 11 - .../testdata/scan_mongodb.csv.license | 2 - .../shadowserver/testdata/scan_mqtt.csv | 2 - .../testdata/scan_mqtt.csv.license | 2 - .../shadowserver/testdata/scan_mqtt_anon.csv | 4 - .../testdata/scan_mqtt_anon.csv.license | 2 - .../shadowserver/testdata/scan_mssql.csv | 4 - .../testdata/scan_mssql.csv.license | 2 - .../shadowserver/testdata/scan_mysql.csv | 4 - .../testdata/scan_mysql.csv.license | 2 - .../shadowserver/testdata/scan_nat_pmp.csv | 4 - .../testdata/scan_nat_pmp.csv.license | 2 - .../shadowserver/testdata/scan_netbios.csv | 4 - .../testdata/scan_netbios.csv.license | 2 - .../testdata/scan_netis_router.csv | 4 - .../testdata/scan_netis_router.csv.license | 2 - .../shadowserver/testdata/scan_ntp.csv | 4 - .../testdata/scan_ntp.csv.license | 2 - .../shadowserver/testdata/scan_ntpmonitor.csv | 4 - .../testdata/scan_ntpmonitor.csv.license | 2 - .../shadowserver/testdata/scan_portmapper.csv | 4 - .../testdata/scan_portmapper.csv.license | 2 - .../shadowserver/testdata/scan_postgres.csv | 4 - .../testdata/scan_postgres.csv.license | 2 - .../shadowserver/testdata/scan_qotd.csv | 4 - .../testdata/scan_qotd.csv.license | 2 - .../shadowserver/testdata/scan_quic.csv | 4 - .../testdata/scan_quic.csv.license | 2 - .../shadowserver/testdata/scan_radmin.csv | 10 - .../testdata/scan_radmin.csv.license | 2 - .../shadowserver/testdata/scan_rdp.csv | 3 - .../testdata/scan_rdp.csv.license | 2 - .../shadowserver/testdata/scan_rdpeudp.csv | 4 - .../testdata/scan_rdpeudp.csv.license | 2 - .../shadowserver/testdata/scan_redis.csv | 94 ------- .../testdata/scan_redis.csv.license | 2 - .../shadowserver/testdata/scan_rsync.csv | 4 - .../testdata/scan_rsync.csv.license | 2 - .../shadowserver/testdata/scan_sip.csv | 4 - .../testdata/scan_sip.csv.license | 2 - .../shadowserver/testdata/scan_slp.csv | 4 - .../testdata/scan_slp.csv.license | 2 - .../shadowserver/testdata/scan_smb.csv | 4 - .../testdata/scan_smb.csv.license | 2 - .../testdata/scan_smtp_vulnerable.csv | 3 - .../testdata/scan_smtp_vulnerable.csv.license | 2 - .../shadowserver/testdata/scan_snmp.csv | 4 - .../testdata/scan_snmp.csv.license | 2 - .../shadowserver/testdata/scan_socks.csv | 4 - .../testdata/scan_socks.csv.license | 2 - .../shadowserver/testdata/scan_ssdp.csv | 4 - .../testdata/scan_ssdp.csv.license | 2 - .../shadowserver/testdata/scan_ssh.csv | 4 - .../testdata/scan_ssh.csv.license | 2 - .../shadowserver/testdata/scan_ssl.csv | 4 - .../testdata/scan_ssl.csv.license | 2 - .../shadowserver/testdata/scan_ssl_freak.csv | 46 ---- .../testdata/scan_ssl_freak.csv.license | 2 - .../shadowserver/testdata/scan_ssl_poodle.csv | 32 --- .../testdata/scan_ssl_poodle.csv.license | 2 - .../shadowserver/testdata/scan_stun.csv | 4 - .../testdata/scan_stun.csv.license | 2 - .../testdata/scan_synfulknock.csv | 4 - .../testdata/scan_synfulknock.csv.license | 2 - .../shadowserver/testdata/scan_telnet.csv | 3 - .../testdata/scan_telnet.csv.license | 2 - .../shadowserver/testdata/scan_tftp.csv | 4 - .../testdata/scan_tftp.csv.license | 2 - .../shadowserver/testdata/scan_ubiquiti.csv | 4 - .../testdata/scan_ubiquiti.csv.license | 2 - .../shadowserver/testdata/scan_vnc.csv | 3 - .../testdata/scan_vnc.csv.license | 2 - .../testdata/scan_ws_discovery.csv | 4 - .../testdata/scan_ws_discovery.csv.license | 2 - .../shadowserver/testdata/scan_xdmcp.csv | 4 - .../testdata/scan_xdmcp.csv.license | 2 - .../testdata/sinkhole_http_drone.csv.license | 2 - .../parsers/shadowserver/testdata/special.csv | 4 - .../shadowserver/testdata/special.csv.license | 2 - 291 files changed, 12939 deletions(-) delete mode 100644 intelmq/tests/bots/parsers/shadowserver/scan_rdpeudp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_blocklist.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_compromised_website.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_device_id.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_ddos_participant.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_darknet.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos_target.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_http_scan.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_ip_spoofer.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole_http.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_dns.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http_referer.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event6_sinkhole_http.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_honeypot_brute_force.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_honeypot_ddos_amp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_malware_url.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_phish_url.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_population_http_proxy.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_sandbox_conn.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_sandbox_dns.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_sandbox_url.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_adb.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_afp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_amqp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ard.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_chargen.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_cisco_smart_install.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_coap.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_couchdb.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_cwmp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_db2.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ddos_middlebox.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_dns.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_docker.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_dvr_dhcpdiscover.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_elasticsearch.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_exchange.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ftp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_hadoop.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_http.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_http_proxy.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_http_vulnerable.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ics.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ipmi.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ipp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_isakmp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_kubernetes.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_tcp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_udp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_mdns.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_memcached.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_mongodb.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt_anon.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_mssql.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_mysql.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_nat_pmp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_netbios.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_netis_router.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ntp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ntpmonitor.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_portmapper.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_postgres.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_qotd.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_quic.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_radmin.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_rdp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_rdpeudp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_redis.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_rsync.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_sip.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_slp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_smb.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_smb_json.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_smtp_vulnerable.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_snmp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_socks.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ssdp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ssh.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ssl.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_freak.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_poodle.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_stun.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_synfulknock.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_telnet.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_tftp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ubiquiti.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_vnc.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ws_discovery.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_xdmcp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_special.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_testdata.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/botnet_drone.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/caida_ip_spoofer.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/darknet.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/ddos_amplification.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/drone_brute_force.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/hp_http_scan.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/hp_ics_scan.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/outdated_dnssec_key.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sinkhole_http_drone.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/special.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/special.csv.license diff --git a/intelmq/tests/bots/parsers/shadowserver/scan_rdpeudp.csv.license b/intelmq/tests/bots/parsers/shadowserver/scan_rdpeudp.csv.license deleted file mode 100644 index 043ed079f1..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/scan_rdpeudp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Sebastian Waldbauer -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/test_blocklist.py b/intelmq/tests/bots/parsers/shadowserver/test_blocklist.py deleted file mode 100644 index 48509eea0e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_blocklist.py +++ /dev/null @@ -1,103 +0,0 @@ -# SPDX-FileCopyrightText: 2020 Thomas Hungenberg -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/blocklist.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = { - 'feed.name': 'Block Listed IP Addresses', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-blocklist-test-geo.csv", -} -EVENTS = [{ - '__type': 'Event', - 'feed.name': 'Block Listed IP Addresses', - "classification.identifier": "blacklisted-ip", - "classification.taxonomy": "other", - "classification.type": "blacklist", - "extra.naics": 517311, - "extra.reason": "Malicious Host AA", - "extra.source": "Alien Vault", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 5678, - "source.geolocation.cc": "XX", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.134", - "source.reverse_dns": "host.local", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T07:00:19+00:00" -}, -{ - '__type': 'Event', - 'feed.name': 'Block Listed IP Addresses', - "classification.identifier": "blacklisted-ip", - "classification.taxonomy": "other", - "classification.type": "blacklist", - "extra.naics": 517311, - "extra.reason": "Malicious Host AA", - "extra.source": "Alien Vault", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 5678, - "source.geolocation.cc": "XX", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.171", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T07:00:19+00:00" -}, -{ - '__type': 'Event', - 'feed.name': 'Block Listed IP Addresses', - "classification.identifier": "blacklisted-ip", - "classification.taxonomy": "other", - "classification.type": "blacklist", - "extra.naics": 517311, - "extra.reason": "Malicious Host AA", - "extra.source": "Alien Vault", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - "source.asn": 5678, - "source.geolocation.cc": "XX", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.network": "198.123.245.0/24", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T07:00:19+00:00" -},] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_compromised_website.py b/intelmq/tests/bots/parsers/shadowserver/test_compromised_website.py deleted file mode 100644 index 53c5b247b1..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_compromised_website.py +++ /dev/null @@ -1,88 +0,0 @@ -# SPDX-FileCopyrightText: 2017 Sebastian Wagner -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/compromised_website.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "ShadowServer Compromised Website", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-compromised_website-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'ShadowServer Compromised Website', - 'classification.taxonomy': 'intrusions', - 'classification.type': 'system-compromise', - 'classification.identifier': 'compromised-website', - 'extra.server': 'Microsoft-IIS/7.5', - 'extra.system': 'WINNT', - 'extra.detected_since': '2015-05-09 05:51:12', - 'protocol.application': 'http', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 64496, - 'source.geolocation.cc': 'AT', - 'source.geolocation.city': 'VIENNA', - 'source.geolocation.region': 'WIEN', - 'source.ip': '203.0.113.1', - 'source.port': 80, - 'source.url': 'http://example.com/header.php', - 'source.fqdn': 'example.com', - 'source.reverse_dns': 'example.com', - 'malware.name': 'hacked-webserver-stealrat-t1', - 'event_description.text': 'spam', - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2017-01-16T00:43:48+00:00'}, - {'__type': 'Event', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'classification.identifier': 'compromised-website', - 'classification.taxonomy': 'intrusions', - 'classification.type': 'system-compromise', - 'event_description.text': 'phishing', - 'feed.name': 'ShadowServer Compromised Website', - 'malware.name': 'phishing', - 'protocol.application': 'http', - 'source.asn': 64496, - 'source.fqdn': 'example.com', - 'source.geolocation.cc': 'AT', - 'source.geolocation.city': 'GRAZ', - 'source.geolocation.region': 'STEIERMARK', - 'source.ip': '203.0.113.1', - 'source.port': 80, - 'source.url': 'http://example.com/', - 'time.source': '2018-04-09T15:43:41+00:00'}, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_device_id.py b/intelmq/tests/bots/parsers/shadowserver/test_device_id.py deleted file mode 100644 index e8954e03c1..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_device_id.py +++ /dev/null @@ -1,116 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/device_id.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Device ID', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-device_id-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'device-id', - 'classification.taxonomy' : 'other', - 'classification.type' : 'undetermined', - 'extra.device_model' : 'FortiGate', - 'extra.device_type' : 'firewall', - 'extra.device_vendor' : 'Fortinet', - 'extra.naics' : 517311, - 'feed.name' : 'Device ID', - 'extra.tag' : 'ssl,vpn', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 2116, - 'source.geolocation.cc' : 'NO', - 'source.geolocation.city' : 'TROMVIK', - 'source.geolocation.region' : 'TROMS OG FINNMARK', - 'source.ip' : '88.84.0.0', - 'source.port' : 10443, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:42+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'device-id', - 'classification.taxonomy' : 'other', - 'classification.type' : 'undetermined', - 'extra.device_model' : 'FortiGate', - 'extra.device_type' : 'firewall', - 'extra.device_vendor' : 'Fortinet', - 'feed.name' : 'Device ID', - 'extra.tag' : 'ssl,vpn', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 27843, - 'source.geolocation.cc' : 'PE', - 'source.geolocation.city' : 'LIMA', - 'source.geolocation.region' : 'METROPOLITANA DE LIMA', - 'source.ip' : '170.231.0.0', - 'source.port' : 10443, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:42+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'device-id', - 'classification.taxonomy' : 'other', - 'classification.type' : 'undetermined', - 'extra.device_model' : 'FortiGate', - 'extra.device_type' : 'firewall', - 'extra.device_vendor' : 'Fortinet', - 'extra.naics' : 517311, - 'feed.name' : 'Device ID', - 'extra.tag' : 'ssl,vpn', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 4181, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'MILWAUKEE', - 'source.geolocation.region' : 'WISCONSIN', - 'source.ip' : '96.60.0.0', - 'source.port' : 10443, - 'source.reverse_dns' : '96-60-66-218.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:42+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_ddos_participant.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_ddos_participant.py deleted file mode 100644 index badc53a736..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_ddos_participant.py +++ /dev/null @@ -1,131 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_ddos_participant.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'DDoS Participant', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-event4_ddos_participant-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'ddos-participant', - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'destination.asn': 65534, - 'destination.geolocation.cc': 'ZZ', - 'destination.geolocation.city': 'City', - 'destination.geolocation.region': 'Region', - 'destination.ip': '172.16.0.1', - 'destination.port': 443, - 'destination.reverse_dns': 'node01.example.net', - 'extra.application': 'https', - 'extra.domain': 'www.example.com', - 'extra.http_method': 'GET', - 'extra.http_path': '/??=GovpfOoaWYlk', - 'feed.name': 'DDoS Participant', - 'malware.name': 'ddos-participant', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 38055, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'ddos-participant', - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'destination.asn': 65534, - 'destination.geolocation.cc': 'ZZ', - 'destination.geolocation.city': 'City', - 'destination.geolocation.region': 'Region', - 'destination.ip': '172.16.0.2', - 'destination.port': 53, - 'destination.reverse_dns': 'node02.example.net', - 'extra.application': 'dns', - 'feed.name': 'DDoS Participant', - 'malware.name': 'ddos-participant', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 53, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'ddos-participant', - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'destination.asn': 65534, - 'destination.geolocation.cc': 'ZZ', - 'destination.geolocation.city': 'City', - 'destination.geolocation.region': 'Region', - 'destination.ip': '172.16.0.3', - 'destination.port': 53, - 'destination.reverse_dns': 'node03.example.net', - 'extra.application': 'dns', - 'extra.device_model': 'Exchange', - 'extra.device_type': 'email', - 'extra.device_vendor': 'Microsoft', - 'feed.name': 'DDoS Participant', - 'malware.name': 'ddos-participant', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 53, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_darknet.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_darknet.py deleted file mode 100644 index 1d020f4737..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_darknet.py +++ /dev/null @@ -1,106 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_honeypot_darknet.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "ShadowServer Darknet", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_honeypot_darknet.csv", - } -EVENTS = [{'__type': 'Event', - 'classification.identifier': 'mirai', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'destination.port': 23, - 'extra.source.naics': 518210, - 'extra.tag': 'mirai', - 'protocol.transport': 'tcp', - 'feed.name': 'ShadowServer Darknet', - 'malware.name': 'mirai', - 'source.asn': 9829, - 'source.geolocation.cc': 'IN', - 'source.geolocation.city': 'CHENGANNUR', - 'source.geolocation.region': 'KERALA', - 'source.ip': '61.3.1.2', - 'source.port': 4717, - 'time.source': '2021-03-07T00:00:00+00:00', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - }, - {'__type': 'Event', - 'classification.identifier': 'mirai', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'destination.port': 23, - 'protocol.transport': 'tcp', - 'extra.source.naics': 517311, - 'extra.tag': 'mirai', - 'feed.name': 'ShadowServer Darknet', - 'malware.name': 'mirai', - 'source.asn': 4766, - 'source.geolocation.cc': 'KR', - 'source.geolocation.city': 'PYEONGCHANG-EUP', - 'source.geolocation.region': 'GANGWON-DO', - 'source.ip': '211.218.3.4', - 'source.port': 4405, - 'time.source': '2021-03-07T00:00:00+00:00', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - }, - {'__type': 'Event', - 'classification.identifier': 'mirai', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'extra.tag': 'mirai', - 'destination.port': 23, - 'protocol.transport': 'tcp', - 'feed.name': 'ShadowServer Darknet', - 'malware.name': 'mirai', - 'source.asn': 266915, - 'source.geolocation.cc': 'BR', - 'source.geolocation.city': 'VITORIA DA CONQUISTA', - 'source.geolocation.region': 'BAHIA', - 'source.ip': '45.225.5.6', - 'source.port': 59777, - 'source.reverse_dns': 'static-45-225-x-x.example.net', - 'time.source': '2021-03-07T00:00:00+00:00', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - }, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos.py deleted file mode 100644 index c62a610faf..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos.py +++ /dev/null @@ -1,148 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_honeypot_ddos.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Honeypot DDoS Events', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-event4_honeypot_ddos-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'honeypot-ddos', - 'classification.taxonomy' : 'availability', - 'classification.type' : 'ddos', - 'destination.asn' : 65534, - 'destination.geolocation.cc' : 'ZZ', - 'destination.geolocation.city' : 'City', - 'destination.geolocation.region' : 'Region', - 'destination.ip' : '172.16.0.1', - 'destination.port' : 88, - 'destination.reverse_dns' : 'node01.example.net', - 'extra.application' : 'mirai', - 'extra.attack' : 'atk10', - 'extra.dst_netmask' : '32', - 'extra.dst_network' : '121.12.110.28/32', - 'extra.duration' : 30, - 'extra.family' : 'mirai', - 'extra.packet_length' : 1440, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'mirai', - 'feed.name' : 'Honeypot DDoS Events', - 'malware.name' : 'ddos', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 61234, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'honeypot-ddos', - 'classification.taxonomy' : 'availability', - 'classification.type' : 'ddos', - 'destination.asn' : 65534, - 'destination.geolocation.cc' : 'ZZ', - 'destination.geolocation.city' : 'City', - 'destination.geolocation.region' : 'Region', - 'destination.ip' : '172.16.0.2', - 'destination.port' : 80, - 'destination.reverse_dns' : 'node02.example.net', - 'extra.application' : 'mirai', - 'extra.attack' : 'atk10', - 'extra.dst_netmask' : '32', - 'extra.dst_network' : '180.97.183.94/32', - 'extra.duration' : 30, - 'extra.family' : 'mirai', - 'extra.packet_length' : 1440, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'mirai', - 'feed.name' : 'Honeypot DDoS Events', - 'malware.name' : 'ddos', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 61234, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'honeypot-ddos', - 'classification.taxonomy' : 'availability', - 'classification.type' : 'ddos', - 'destination.asn' : 65534, - 'destination.geolocation.cc' : 'ZZ', - 'destination.geolocation.city' : 'City', - 'destination.geolocation.region' : 'Region', - 'destination.ip' : '172.16.0.3', - 'destination.reverse_dns' : 'node03.example.net', - 'extra.application' : 'mirai', - 'extra.attack' : 'atk7', - 'extra.destination.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.dst_netmask' : '32', - 'extra.dst_network' : '104.237.138.135/32', - 'extra.duration' : 10, - 'extra.family' : 'mirai', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'mirai', - 'feed.name' : 'Honeypot DDoS Events', - 'malware.name' : 'ddos', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 6379, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos_target.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos_target.py deleted file mode 100644 index f379d1c882..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos_target.py +++ /dev/null @@ -1,150 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_honeypot_ddos_target.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Honeypot DDoS Target Events', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-event4_honeypot_ddos_target-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'honeypot-ddos-target', - 'classification.taxonomy' : 'availability', - 'classification.type' : 'ddos', - 'destination.asn' : 65534, - 'destination.geolocation.cc' : 'ZZ', - 'destination.geolocation.city' : 'City', - 'destination.geolocation.region' : 'Region', - 'destination.ip' : '172.16.0.1', - 'destination.port' : 80, - 'destination.reverse_dns' : 'node01.example.net', - 'extra.application' : 'mirai', - 'extra.attack' : 'atk0', - 'extra.dst_netmask' : '32', - 'extra.dst_network' : '115.238.198.85/32', - 'extra.duration' : 30, - 'extra.family' : 'mirai', - 'extra.packet_length' : 1440, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'mirai', - 'feed.name' : 'Honeypot DDoS Target Events', - 'malware.name' : 'ddos', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 61234, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'honeypot-ddos-target', - 'classification.taxonomy' : 'availability', - 'classification.type' : 'ddos', - 'destination.asn' : 65534, - 'destination.geolocation.cc' : 'ZZ', - 'destination.geolocation.city' : 'City', - 'destination.geolocation.region' : 'Region', - 'destination.ip' : '172.16.0.2', - 'destination.port' : 43437, - 'destination.reverse_dns' : 'node02.example.net', - 'extra.application' : 'mirai', - 'extra.attack' : 'atk0', - 'extra.destination.sector' : 'Information', - 'extra.dst_netmask' : '32', - 'extra.dst_network' : '52.184.50.250/32', - 'extra.duration' : 30, - 'extra.family' : 'mirai', - 'extra.packet_length' : 1440, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'mirai', - 'feed.name' : 'Honeypot DDoS Target Events', - 'malware.name' : 'ddos', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 61234, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'honeypot-ddos-target', - 'classification.taxonomy' : 'availability', - 'classification.type' : 'ddos', - 'destination.asn' : 65534, - 'destination.geolocation.cc' : 'ZZ', - 'destination.geolocation.city' : 'City', - 'destination.geolocation.region' : 'Region', - 'destination.ip' : '172.16.0.3', - 'destination.port' : 80, - 'destination.reverse_dns' : 'node03.example.net', - 'extra.application' : 'mirai', - 'extra.attack' : 'atk10', - 'extra.dst_netmask' : '32', - 'extra.dst_network' : '211.99.102.216/32', - 'extra.duration' : 30, - 'extra.family' : 'mirai', - 'extra.packet_length' : 1440, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'mirai', - 'feed.name' : 'Honeypot DDoS Target Events', - 'malware.name' : 'ddos', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 61234, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_http_scan.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_http_scan.py deleted file mode 100644 index bcf268ba7d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_http_scan.py +++ /dev/null @@ -1,109 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Mikk Margus Möll -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_honeypot_http_scan.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Honeypot-HTTP-Scan', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2021-08-01T12:00:00+00:00", - "extra.file_name": "2021-08-01-event4_honeypot_http_scan.csv", - } - -EVENTS = [{'__type': 'Event', - 'feed.name': 'Honeypot-HTTP-Scan', - 'classification.identifier': 'honeypot-http-scan', - 'classification.taxonomy': 'information-gathering', - 'classification.type': 'scanner', - 'destination.asn': 5678, - 'destination.geolocation.cc': 'UK', - 'destination.geolocation.city': 'MAIDENHEAD', - 'destination.geolocation.region': 'WINDSOR AND MAIDENHEAD', - 'destination.ip': '109.87.65.43', - 'destination.port': 80, - 'extra.http_url': '/js/ueditor/wwwroot/way-board.cgi', - 'extra.destination.naics': 518210, - 'protocol.transport': 'tcp', - 'protocol.application': 'http', - 'extra.public_source': 'CAPRICA-EU', - 'extra.request_raw': 'R0VUIC9qcy91ZWRpdG9yL3d3d3Jvb3Qvd2F5LWJvYXJkLmNnaSBIVFRQLzEuMHJuQWNjZXB0OiB0ZXh0L2h0bWwsYXBwbGljYXRpb24veGh0bWwreG1sLGFwcGxpY2F0aW9uL3htbDtxPTAuOSwqLyo7cT0wLjhybkFjY2VwdC1FbmNvZGluZzogZ3ppcCwgZGVmbGF0ZXJuQWNjZXB0LUxhbmd1YWdlOiBlbi1VUyxlbjtxPTAuNXJuQ29ubmVjdGlvbjogY2xvc2VybkRudDogMXJuSG9zdDogMTA5Ljg3LjY1LjQzcm5PcmlnaW46IGh0dHA6Ly8xMDkuODcuNjUuNDNyblJlZmVyZXI6IGh0dHA6Ly8xMDkuODcuNjUuNDMvcm5Vc2VyLUFnZW50OiBNb3ppbGxhLzUuMCAoV2luZG93cyBOVCA2LjE7IFdPVzY0KSBBcHBsZVdlYktpdC81MzcuMzYgKEtIVE1MLCBsaWtlIEdlY2tvKSBDaHJvbWUvNTMuMC4yNzg1LjEwNCBTYWZhcmkvNTM3LjM2IENvcmUvMS41My4zMDg0LjQwMCBRUUJyb3dzZXIvOS42LjExMzQ2LjQwMA==', - 'extra.source.naics': 518210, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.version': '3.1.3-dev', - 'malware.name': 'http-scan', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 1234, - 'source.geolocation.cc': 'EE', - 'source.geolocation.city': 'TALLINN', - 'source.geolocation.region': 'HARJUMAA', - 'source.ip': '191.23.45.67', - 'source.port': 36455, - 'source.reverse_dns': '191-23-45-67-host.example.com', - 'time.observation': '2021-08-01T12:00:00+00:00', - 'time.source': '2021-08-01T00:24:08+00:00'}, - {'__type': 'Event', - 'feed.name': 'Honeypot-HTTP-Scan', - 'classification.identifier': 'honeypot-http-scan', - 'classification.taxonomy': 'information-gathering', - 'classification.type': 'scanner', - 'destination.asn': 23456, - 'destination.geolocation.cc': 'UA', - 'destination.geolocation.city': 'KHARKIV', - 'destination.geolocation.region': "KHARKIVS'KA OBLAST'", - 'destination.ip': '82.41.20.10', - 'destination.port': 8080, - 'extra.http_url': '/', - 'extra.method': 'GET', - 'protocol.transport': 'tcp', - 'protocol.application': 'http', - 'extra.public_source': 'CAPRICA-EU', - 'extra.request_raw': 'R0VUIC8gSFRUUC8xLjENCkhvc3Q6IDgyLjQxLjIwLjEwOjgwODANCkFjY2VwdDogdGV4dC9odG1sLGFwcGxpY2F0aW9uL3hodG1sK3htbCxhcHBsaWNhdGlvbi94bWw7cT0wLjksKi8qO3E9MC44DQpBY2NlcHQtRW5jb2Rpbmc6IGRlZmxhdGUsIGd6aXAsIGlkZW50aXR5DQpBY2NlcHQtTGFuZ3VhZ2U6IGVuLVVTO3E9MC42LGVuO3E9MC40DQpVc2VyLUFnZW50OiBNb3ppbGxhLzUuMCAoV2luZG93cyBOVCA1LjE7IHJ2OjkuMC4xKSBHZWNrby8yMDEwMDEwMSBGaXJlZm94LzkuMC4xDQoNCg==', - 'extra.url_scheme': 'http', - 'extra.user_agent': 'Mozilla/5.0 (Windows NT 5.1; rv:9.0.1) Gecko/20100101 Firefox/9.0.1', - 'malware.name': 'http-scan', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn': 12345, - 'source.geolocation.cc': 'EE', - 'source.geolocation.city': 'TALLINN', - 'source.geolocation.region': 'HARJUMAA', - 'source.ip': '45.67.89.123', - 'source.port': 58610, - 'time.observation': '2021-08-01T12:00:00+00:00', - 'time.source': '2021-08-01T05:21:59+00:00'}, -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_ip_spoofer.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_ip_spoofer.py deleted file mode 100644 index d21fb10c5b..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_ip_spoofer.py +++ /dev/null @@ -1,182 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), "testdata/event4_ip_spoofer.csv")) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = { - "feed.name": "CAIDA", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2020-08-19T00:00:00+00:00", - "extra.file_name": "2020-08-19-event4_ip_spoofer.csv", -} - -EVENTS = [ - { - "__type": "Event", - "feed.name": "CAIDA", - "time.source": "2021-03-28T00:42:59+00:00", - "source.ip": "98.191.250.0", - - "source.asn": 22898, - - "source.geolocation.cc": "US", - "source.geolocation.region": "OKLAHOMA", - "source.geolocation.city": "OKLAHOMA CITY", - "source.network": "98.191.250.0/24", - "source.reverse_dns": 'ip-98.191.250.0.atlinkservices.com', - "extra.routedspoof": "received", - "extra.session": '1112907', - "extra.nat": True, - "extra.public_source": "caida", - "extra.source.naics": 517311, - "extra.version": 'ipv4', - "protocol.transport": 'tcp', - "extra.infection": 'ip-spoofer', - - "classification.identifier": "ip-spoofer", - "classification.taxonomy": "fraud", - "classification.type": "masquerade", - - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[1]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "CAIDA", - "time.source": "2021-03-28T01:36:22+00:00", - "source.ip": "191.7.16.0", - - "source.asn": 262485, - - "source.geolocation.cc": "BR", - "source.geolocation.region": "RIO DE JANEIRO", - "source.geolocation.city": "NOVA IGUACU", - "source.network": "191.7.16.0/24", - "extra.routedspoof": "received", - "extra.session": '1112914', - "extra.nat": False, - "extra.public_source": "caida", - "extra.version": 'ipv4', - "protocol.transport": 'tcp', - "extra.infection": 'ip-spoofer', - - "classification.identifier": "ip-spoofer", - "classification.taxonomy": "fraud", - "classification.type": "masquerade", - - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[2]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "CAIDA", - "time.source": "2021-03-28T02:10:58+00:00", - "source.ip": "202.53.160.0", - - "source.asn": 23923, - - "source.geolocation.cc": "BD", - "source.geolocation.region": "DHAKA", - "source.geolocation.city": "DHAKA", - "source.network": "202.53.160.0/24", - "extra.routedspoof": "received", - "extra.session": '1112931', - "extra.nat": True, - "extra.public_source": "caida", - "extra.version": 'ipv4', - "protocol.transport": 'tcp', - "extra.infection": 'ip-spoofer', - - "classification.identifier": "ip-spoofer", - "classification.taxonomy": "fraud", - "classification.type": "masquerade", - - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[3]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "CAIDA", - "time.source": "2021-03-28T03:41:51+00:00", - "source.ip": "87.121.75.0", - - "source.asn": 134697, - - "source.geolocation.cc": "AU", - "source.geolocation.region": "QUEENSLAND", - "source.geolocation.city": "BRISBANE", - "source.network": "87.121.75.0/24", - "extra.routedspoof": "received", - "extra.session": '1112953', - "extra.nat": True, - "extra.public_source": "caida", - "extra.version": 'ipv4', - "protocol.transport": 'tcp', - "extra.infection": 'ip-spoofer', - - "classification.identifier": "ip-spoofer", - "classification.taxonomy": "fraud", - "classification.type": "masquerade", - - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[4]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "CAIDA", - "time.source": "2021-03-28T06:07:17+00:00", - "source.ip": "189.201.194.0", - - "source.asn": 262944, - - "source.network": "189.201.194.0/24", - "source.geolocation.cc": 'MX', - "source.geolocation.city": 'SALTILLO', - "source.geolocation.region": 'COAHUILA', - "source.reverse_dns": 'ip-189-201-194-0.slw.spectro.mx', - "extra.routedspoof": "received", - "extra.session": '1113015', - "extra.nat": True, - "extra.public_source": "caida", - "extra.version": 'ipv4', - "protocol.transport": 'tcp', - "extra.infection": 'ip-spoofer', - - "classification.identifier": "ip-spoofer", - "classification.taxonomy": "fraud", - "classification.type": "masquerade", - - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[5]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, -] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == "__main__": - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole.py deleted file mode 100644 index f008fd18e1..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole.py +++ /dev/null @@ -1,135 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_microsoft_sinkhole.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "ShadowServer Microsoft Sinkhole", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_microsoft_sinkhole.csv", - } -EVENTS = [{'__type': 'Event', - 'classification.identifier': 'b68-zeroaccess-2-32bit', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.geolocation.cc': 'HK', - 'destination.geolocation.city': 'HONG KONG', - 'destination.geolocation.region': 'HONG KONG', - 'destination.ip': '168.63.134.179', - 'destination.port': 16464, - 'extra.destination.naics': 334111, - 'extra.tag': 'b68-zeroaccess-2-32bit', - 'extra.infection': 'b68-zeroaccess-2-32bit', - 'extra.destination.sector': 'Information', - 'extra.public_source': 'MSDCU', - 'extra.source.naics': 517311, - 'extra.tag': 'b68-zeroaccess-2-32bit', - 'feed.name': 'ShadowServer Microsoft Sinkhole', - 'malware.name': 'zeroaccess', - 'protocol.transport': 'tcp', - 'source.asn': 7303, - 'source.geolocation.cc': 'AR', - 'source.geolocation.city': 'CASEROS', - 'source.geolocation.region': 'BUENOS AIRES', - 'source.ip': '190.229.1.2', - 'source.port': 52955, - 'time.source': '2021-06-07T00:00:00+00:00', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - }, - {'__type': 'Event', - 'classification.identifier': 'b68-zeroaccess-2-32bit', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.geolocation.cc': 'IE', - 'destination.geolocation.city': 'DUBLIN', - 'destination.geolocation.region': 'DUBLIN', - 'destination.ip': '52.169.3.4', - 'destination.port': 16464, - 'extra.destination.naics': 334111, - 'extra.tag': 'b68-zeroaccess-2-32bit', - 'extra.infection': 'b68-zeroaccess-2-32bit', - 'extra.destination.sector': 'Information', - 'extra.public_source': 'MSDCU', - 'extra.source.naics': 517311, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'feed.name': 'ShadowServer Microsoft Sinkhole', - 'malware.name': 'zeroaccess', - 'protocol.transport': 'tcp', - 'source.asn': 5769, - 'source.geolocation.cc': 'CA', - 'source.geolocation.city': 'LAVAL', - 'source.geolocation.region': 'QUEBEC', - 'source.ip': '96.20.3.4', - 'source.port': 16464, - 'time.source': '2021-06-07T00:00:00+00:00', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - }, - {'__type': 'Event', - 'classification.identifier': 'b68-zeroaccess-2-32bit', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.geolocation.cc': 'HK', - 'destination.geolocation.city': 'HONG KONG', - 'destination.geolocation.region': 'HONG KONG', - 'destination.ip': '168.63.134.179', - 'destination.port': 16464, - 'extra.tag': 'b68-zeroaccess-2-32bit', - 'extra.infection': 'b68-zeroaccess-2-32bit', - 'extra.destination.naics': 334111, - 'extra.destination.sector': 'Information', - 'extra.public_source': 'MSDCU', - 'extra.source.naics': 517311, - 'feed.name': 'ShadowServer Microsoft Sinkhole', - 'malware.name': 'zeroaccess', - 'protocol.transport': 'tcp', - 'source.asn': 8151, - 'source.geolocation.cc': 'MX', - 'source.geolocation.city': 'MEXICO CITY', - 'source.geolocation.region': "CIUDAD DE MEXICO", - 'source.ip': '187.222.5.6', - 'source.port': 55049, - 'time.source': '2021-06-07T00:00:00+00:00', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - }, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole_http.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole_http.py deleted file mode 100644 index 2f8c3d8e2e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole_http.py +++ /dev/null @@ -1,202 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_microsoft_sinkhole_http.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'HTTP Microsoft Sinkhole IPv4', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_microsoft_sinkhole_http.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'HTTP Microsoft Sinkhole IPv4', - 'classification.identifier': 'necurs', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'ASHBURN', - 'destination.geolocation.region': 'VIRGINIA', - 'destination.ip': '40.121.206.97', - 'destination.port': 80, - 'destination.url': 'http://40.121.206.97/locator.php', - 'extra.destination.naics': 334111, - 'extra.destination.sector': 'Information', - 'extra.public_source': 'MSDCU', - 'extra.infection': 'necurs', - 'extra.tag': 'necurs', - 'protocol.application': 'http', - 'malware.name': 'necurs', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 8386, - 'source.geolocation.cc': 'TR', - 'source.geolocation.city': 'KEPEZ', - 'source.geolocation.region': 'ANTALYA', - 'source.ip': '31.206.1.2', - 'source.port': 49245, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-06-07T00:00:00+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Microsoft Sinkhole IPv4', - 'classification.identifier': 'caphaw', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.fqdn': '3fo8jrthz3y.rgk.cc', - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'REDMOND', - 'destination.geolocation.region': 'WASHINGTON', - 'destination.ip': '204.95.99.204', - 'destination.port': 443, - 'destination.url': 'http://3fo8jrthz3y.rgk.cc/index.php', - 'protocol.application': 'http', - 'extra.infection': 'caphaw', - 'extra.tag': 'caphaw', - 'extra.destination.naics': 334111, - 'extra.destination.sector': 'Information', - 'extra.http_agent': 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.0.6103)', - 'extra.http_referer': 'null', - 'extra.public_source': 'MSDCU', - 'extra.source.naics': 517312, - 'malware.name': 'caphaw', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn': 28573, - 'source.geolocation.cc': 'BR', - 'source.geolocation.city': 'SAO PAULO', - 'source.geolocation.region': 'SAO PAULO', - 'source.ip': '177.140.3.4', - 'source.port': 35919, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-06-07T00:00:00+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Microsoft Sinkhole IPv4', - 'classification.identifier': 'necurs', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'ASHBURN', - 'destination.geolocation.region': 'VIRGINIA', - 'destination.ip': '40.121.206.97', - 'destination.port': 80, - 'destination.url': 'http://40.121.206.97/locator.php', - 'protocol.application': 'http', - 'extra.tag': 'necurs', - 'extra.infection': 'necurs', - 'extra.destination.naics': 334111, - 'extra.destination.sector': 'Information', - 'extra.public_source': 'MSDCU', - 'extra.source.naics': 517311, - 'malware.name': 'necurs', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn': 132199, - 'source.geolocation.cc': 'PH', - 'source.geolocation.city': 'MANDAUE', - 'source.geolocation.region': 'CEBU', - 'source.ip': '180.190.5.6', - 'source.port': 49264, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-06-07T00:00:01+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Microsoft Sinkhole IPv4', - 'classification.identifier': 'necurs', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.geolocation.cc': 'US', - 'destination.ip': '40.121.206.97', - 'destination.geolocation.city': 'ASHBURN', - 'destination.geolocation.region': 'VIRGINIA', - 'destination.port': 80, - 'destination.url': 'http://40.121.206.97/news/stream.php', - 'extra.destination.naics': 334111, - 'extra.destination.sector': 'Information', - 'extra.public_source': 'MSDCU', - 'malware.name': 'necurs', - 'extra.tag': 'necurs', - 'extra.infection': 'necurs', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[4]])), - 'source.asn': 37129, - 'source.geolocation.cc': 'KE', - 'source.geolocation.city': 'NAIROBI', - 'source.geolocation.region': 'NAIROBI CITY', - 'source.ip': '197.157.7.8', - 'source.port': 55307, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-06-07T00:00:01+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Microsoft Sinkhole IPv4', - 'classification.identifier': 'necurs', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'ASHBURN', - 'destination.geolocation.region': 'VIRGINIA', - 'destination.ip': '40.121.206.97', - 'destination.port': 80, - 'destination.url': 'http://40.121.206.97/locator.php', - 'extra.destination.naics': 334111, - 'extra.destination.sector': 'Information', - 'extra.public_source': 'MSDCU', - 'extra.source.naics': 517311, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'malware.name': 'necurs', - 'protocol.application': 'http', - 'extra.tag': 'necurs', - 'extra.infection': 'necurs', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[5]])), - 'source.asn': 812, - 'source.geolocation.cc': 'CA', - 'source.geolocation.city': 'OTTAWA', - 'source.geolocation.region': 'ONTARIO', - 'source.ip': '174.114.9.10', - 'source.port': 59000, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-06-07T00:00:01+00:00'}] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole.py deleted file mode 100644 index 2bb8aa6980..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole.py +++ /dev/null @@ -1,73 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_sinkhole.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "ShadowServer Sinkhole", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_sinkhole.csv", - } -EVENTS = [{'__type': 'Event', - 'classification.identifier': 'victorygate.b', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 28753, - 'destination.geolocation.cc': 'DE', - 'destination.geolocation.city': 'FRANKFURT AM MAIN', - 'destination.geolocation.region': 'HESSEN', - 'destination.ip': '178.162.1.2', - 'destination.port': 4455, - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.public_source': 'eset', - 'feed.name': 'ShadowServer Sinkhole', - 'malware.name': 'victorygate.b', - 'extra.infection': 'victorygate.b', - 'protocol.transport': 'tcp', - 'source.asn': 12252, - 'source.geolocation.cc': 'PE', - 'source.geolocation.city': 'LIMA', - 'source.geolocation.region': 'METROPOLITANA DE LIMA', - 'source.ip': '190.113.1.2', - 'source.port': 17409, - 'time.source': '2021-03-04T00:00:00+00:00', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - }, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_dns.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_dns.py deleted file mode 100644 index cf3bdb1623..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_dns.py +++ /dev/null @@ -1,127 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/event4_sinkhole_dns.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "Sinkhole DNS", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_sinkhole_dns-test-geo.csv", - } - -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'sinkholedns', - 'extra.tag' : 'msexchange', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.count' : 1, - 'extra.infection' : 'calypso', - 'extra.dns_query' : 'YolkIsh.COM', - 'extra.dns_query_type' : 'A', - 'extra.naics' : 518210, - 'extra.sector' : 'Communications, Service Provider, and Hosting Service', - 'feed.name' : 'Sinkhole DNS', - 'malware.name' : 'calypso', - 'protocol.application' : 'dns', - 'protocol.transport' : 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 8220, - 'source.geolocation.cc' : 'DE', - 'source.geolocation.city' : 'FRANKFURT AM MAIN', - 'source.geolocation.region' : 'HESSEN', - 'source.ip' : '217.110.0.0', - 'source.port' : 29614, - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2022-01-06T00:00:02+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'sinkholedns', - 'extra.tag' : 'rat', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.count' : 1, - 'extra.infection' : 'orcus', - 'extra.dns_query' : 'verble.rocks', - 'extra.dns_query_type' : 'A', - 'extra.naics' : 518210, - 'feed.name' : 'Sinkhole DNS', - 'malware.name' : 'orcus', - 'protocol.application' : 'dns', - 'protocol.transport' : 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 40934, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'ASHBURN', - 'source.geolocation.region' : 'VIRGINIA', - 'source.ip' : '209.66.0.0', - 'source.port' : 46189, - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2022-01-06T00:00:02+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'sinkholedns', - 'extra.tag' : 'msexchange', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.count' : 1, - 'extra.infection' : 'calypso', - 'extra.dns_query' : 'RAwFuNS.COM', - 'extra.dns_query_type' : 'A', - 'extra.naics' : 518210, - 'extra.sector' : 'Communications, Service Provider, and Hosting Service', - 'feed.name' : 'Sinkhole DNS', - 'malware.name' : 'calypso', - 'protocol.application' : 'dns', - 'protocol.transport' : 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 8220, - 'source.geolocation.cc' : 'DE', - 'source.geolocation.city' : 'FRANKFURT AM MAIN', - 'source.geolocation.region' : 'HESSEN', - 'source.ip' : '217.110.0.0', - 'source.port' : 3590, - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2022-01-06T00:00:02+00:00' -} -] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http.py deleted file mode 100644 index 60cd6b6efb..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http.py +++ /dev/null @@ -1,189 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_sinkhole_http.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'HTTP Sinkhole IPv4', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_sinkhole_http.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'HTTP Sinkhole IPv4', - 'classification.identifier': 'avalanche-andromeda', - 'extra.tag': 'avalanche-andromeda', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 6939, - 'destination.fqdn': 'differentia.ru', - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'FREMONT', - 'destination.geolocation.region': 'CALIFORNIA', - 'destination.ip': '184.105.1.2', - 'destination.port': 80, - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting Service', - 'malware.name': 'andromeda', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 134707, - 'source.geolocation.cc': 'PH', - 'source.geolocation.city': 'DEL PILAR', - 'source.geolocation.region': 'NUEVA ECIJA', - 'source.ip': '103.196.1.2', - 'source.port': 60902, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-03-04T00:00:00+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Sinkhole IPv4', - 'classification.identifier': 'avalanche-andromeda', - 'extra.tag': 'avalanche-andromeda', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 6939, - 'destination.fqdn': 'differentia.ru', - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'FREMONT', - 'destination.geolocation.region': 'CALIFORNIA', - 'destination.ip': '184.105.3.4', - 'destination.port': 80, - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting Service', - 'malware.name': 'andromeda', - 'extra.source.naics': 517311, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn': 8708, - 'source.geolocation.cc': 'RO', - 'source.geolocation.city': 'CONSTANTA', - 'source.geolocation.region': 'CONSTANTA', - 'source.ip': '5.14.3.4', - 'source.port': 55002, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-03-04T00:00:00+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Sinkhole IPv4', - 'classification.identifier': 'avalanche-andromeda', - 'extra.tag': 'avalanche-andromeda', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 6939, - 'destination.fqdn': 'disorderstatus.ru', - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'FREMONT', - 'destination.geolocation.region': 'CALIFORNIA', - 'destination.ip': '184.105.5.6', - 'destination.port': 80, - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.source.naics': 517311, - 'malware.name': 'andromeda', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn': 9299, - 'source.geolocation.cc': 'PH', - 'source.geolocation.city': 'CEBU', - 'source.geolocation.region': 'CEBU', - 'source.ip': '49.145.5.6', - 'source.port': 31350, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-03-04T00:00:00+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Sinkhole IPv4', - 'classification.identifier': 'avalanche-andromeda', - 'extra.tag': 'avalanche-andromeda', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 6939, - 'destination.fqdn': 'differentia.ru', - 'destination.geolocation.cc': 'US', - 'destination.ip': '184.105.7.8', - 'destination.geolocation.city': 'FREMONT', - 'destination.geolocation.region': 'CALIFORNIA', - 'destination.port': 80, - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.source.naics': 517311, - 'malware.name': 'andromeda', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[4]])), - 'source.asn': 8048, - 'source.geolocation.cc': 'VE', - 'source.geolocation.city': 'VALENCIA', - 'source.geolocation.region': 'CARABOBO', - 'source.ip': '200.44.7.8', - 'source.port': 28063, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-03-04T00:00:00+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Sinkhole IPv4', - 'classification.identifier': 'avalanche-andromeda', - 'extra.tag': 'avalanche-andromeda', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 6939, - 'destination.fqdn': 'differentia.ru', - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'FREMONT', - 'destination.geolocation.region': 'CALIFORNIA', - 'destination.ip': '184.105.9.10', - 'destination.port': 80, - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting Service', - 'malware.name': 'andromeda', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[5]])), - 'source.asn': 17072, - 'source.geolocation.cc': 'MX', - 'source.geolocation.city': 'JUAREZ', - 'source.geolocation.region': 'CHIHUAHUA', - 'source.ip': '187.189.9.10', - 'source.port': 45335, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-03-04T00:00:00+00:00'}] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http_referer.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http_referer.py deleted file mode 100644 index b1ccacd311..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http_referer.py +++ /dev/null @@ -1,213 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Mikk Margus Möll -# -# SPDX-License-Identifier: AGPL-3.0-or-later -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_sinkhole_http_referer.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Sinkhole-Events-HTTP-Referer IPv4', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2021-03-05T00:00:00+00:00", - "extra.file_name": "2021-03-04-event4_sinkhole_http_referer.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Sinkhole-Events-HTTP-Referer IPv4', - 'classification.identifier': 'sinkhole-http-referer', - 'extra.tag': 'kovter', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'destination.asn': 60781, - 'destination.fqdn': '12106.mobapptrack.com', - 'destination.geolocation.cc': 'NL', - 'destination.geolocation.city': 'AMSTERDAM', - 'destination.geolocation.region': 'NOORD-HOLLAND', - 'destination.ip': '85.17.31.82', - 'destination.port': 80, - 'destination.url': 'http://12106.mobapptrack.com/favicon.ico', - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.event_id': '1614816002', - 'malware.name': 'kovter', - 'extra.http_referer': 'http://12106.mobapptrack.com/click/redirect?feed_id=12106&sub_id=7&q=8A5491983C8FBE7743E2D2C36E45EBC4-18307118D2626C9BD756B3F09D14BB910E381EE4', - 'extra.http_referer_asn': 28753, - 'extra.http_referer_city': 'FRANKFURT AM MAIN', - 'extra.http_referer_geo': 'DE', - 'extra.http_referer_hostname': '12106.mobapptrack.com', - 'extra.http_referer_ip': '178.162.203.211', - 'extra.http_referer_naics': 518210, - 'extra.http_referer_port': 80, - 'extra.http_referer_region': 'HESSEN', - 'extra.http_referer_sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-03-04T00:00:02+00:00'}, - {'__type': 'Event', - 'classification.identifier': 'sinkhole-http-referer', - 'extra.tag': 'sunburst', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'destination.asn': 28753, - 'destination.fqdn': 'freescanonline.com', - 'destination.geolocation.cc': 'DE', - 'destination.geolocation.city': 'FRANKFURT AM MAIN', - 'destination.geolocation.region': 'HESSEN', - 'destination.port': 80, - 'destination.url': 'http://freescanonline.com/animalally.com', - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'destination.ip': '178.162.1.2', - 'extra.event_id': '1614816011', - 'malware.name': 'sunburst', - 'extra.http_referer': 'http://x.noizm.com/jump.php?u=http://freescanonline.com/animalally.com', - 'extra.http_referer_asn': 9370, - 'extra.http_referer_city': 'OSAKA', - 'extra.http_referer_geo': 'JP', - 'extra.http_referer_hostname': 'x.noizm.com', - 'extra.http_referer_naics': 518210, - 'extra.http_referer_port': 80, - 'extra.http_referer_ip': '59.106.1.2', - 'extra.http_referer_region': 'OSAKA', - 'extra.http_referer_sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'protocol.transport': 'tcp', - 'feed.name': 'Sinkhole-Events-HTTP-Referer IPv4', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'time.source': '2021-03-04T00:00:11+00:00'}, - {'__type': 'Event', - 'classification.identifier': 'sinkhole-http-referer', - 'extra.tag': 'kovter', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'destination.asn': 28753, - 'destination.fqdn': 'rxrtb.bid', - 'destination.geolocation.cc': 'DE', - 'destination.geolocation.city': 'FRANKFURT AM MAIN', - 'destination.geolocation.region': 'HESSEN', - 'destination.port': 80, - 'destination.url': 'http://rxrtb.bid/getjs?r=0.6393021999392658', - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'destination.ip': '178.162.1.2', - 'extra.event_id': '1614816012', - 'malware.name': 'kovter', - 'extra.http_referer': 'http://x.blogspot.com/', - 'extra.http_referer_ip': '142.250.3.4', - 'extra.http_referer_asn': 15169, - 'extra.http_referer_city': 'MOUNTAIN VIEW', - 'extra.http_referer_geo': 'US', - 'extra.http_referer_hostname': 'x.blogspot.com', - 'extra.http_referer_naics': 519130, - 'extra.http_referer_port': 80, - 'extra.http_referer_region': 'CALIFORNIA', - 'extra.http_referer_sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'protocol.transport': 'tcp', - 'feed.name': 'Sinkhole-Events-HTTP-Referer IPv4', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'time.source': '2021-03-04T00:00:12+00:00'}, - {'__type': 'Event', - 'classification.identifier': 'sinkhole-http-referer', - 'extra.tag': 'sunburst', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'destination.asn': 60781, - 'destination.fqdn': 'freescanonline.com', - 'destination.geolocation.cc': 'NL', - 'destination.geolocation.city': 'AMSTERDAM', - 'destination.geolocation.region': 'NOORD-HOLLAND', - 'destination.ip': '5.79.71.225', - 'destination.port': 80, - 'destination.url': 'http://freescanonline.com/personalationmall.com', - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'extra.event_id': '1614816013', - 'malware.name': 'sunburst', - 'extra.http_referer': 'http://www.example.com/teams/default.asp?u=EKL&t=c&s=lacrosse&p=remote&url=http://freescanonline.com/personalationmall.com', - 'extra.http_referer_asn': 14618, - 'extra.http_referer_city': 'ASHBURN', - 'extra.http_referer_geo': 'US', - 'extra.http_referer_hostname': 'www.example.com', - 'extra.http_referer_ip': '34.232.5.6', - 'extra.http_referer_naics': 454110, - 'extra.http_referer_port': 80, - 'extra.http_referer_region': 'VIRGINIA', - 'extra.http_referer_sector': 'Retail Trade', - 'protocol.transport': 'tcp', - 'feed.name': 'Sinkhole-Events-HTTP-Referer IPv4', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[4]])), - 'time.source': '2021-03-04T00:00:13+00:00'}, - {'__type': 'Event', - 'classification.identifier': 'sinkhole-http-referer', - 'extra.tag': 'sunburst', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'destination.asn': 60781, - 'destination.fqdn': 'freescanonline.com', - 'destination.geolocation.cc': 'NL', - 'destination.geolocation.city': 'AMSTERDAM', - 'destination.geolocation.region': 'NOORD-HOLLAND', - 'destination.port': 80, - 'destination.url': 'http://freescanonline.com/raftcomply.com', - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'destination.ip': '5.79.1.2', - 'extra.event_id': '1614816086', - 'malware.name': 'sunburst', - 'extra.http_referer': 'http://x.communes.jp/?url=http://freescanonline.com/raftcomply.com', - 'extra.http_referer_asn': 2516, - 'extra.http_referer_city': 'SAPPORO', - 'extra.http_referer_geo': 'JP', - 'extra.http_referer_hostname': 'x.communes.jp', - 'extra.http_referer_ip': '210.172.7.8', - 'extra.http_referer_naics': 517312, - 'extra.http_referer_port': 80, - 'extra.http_referer_region': 'HOKKAIDO', - 'extra.http_referer_sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'protocol.transport': 'tcp', - 'feed.name': 'Sinkhole-Events-HTTP-Referer IPv4', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[5]])), - 'time.source': '2021-03-04T00:01:26+00:00'}] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event6_sinkhole_http.py b/intelmq/tests/bots/parsers/shadowserver/test_event6_sinkhole_http.py deleted file mode 100644 index d6ff35dc11..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event6_sinkhole_http.py +++ /dev/null @@ -1,146 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/event6_sinkhole_http.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "Sinkhole-Events-HTTP IPv6", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-event6_sinkhole_http-test-geo.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : '3ve', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'infected-system', - 'destination.asn' : 6939, - 'destination.fqdn' : 'devps.net', - 'destination.geolocation.cc' : 'US', - 'destination.geolocation.city' : 'FREMONT', - 'destination.geolocation.region' : 'CALIFORNIA', - 'destination.ip' : '2001:470:1:332::fe', - 'destination.port' : 80, - 'destination.url' : 'http://devps.net/QKMSvF2hl11j%2fbMkyPbF5EpHYhd6VWTG4u19K3Rt7JGU3lMYRqpq8wPYEuOGKKeidKW3pefVfKSjBnL0cXizZbmuWWu8AQNRqw5g9Ny5vZtiv638XKoWwCLuUOTISTV%2fLcpcS1%2f22NjWqgXkHGISAuyVtafqyCC%2f5cA0eYg9Me8VzAIFDdTArogQOdYhElf2xluhEFPsstGQ%2bwrM4VmKHJpzyjD7Y%2fN%2bQV3wnZNdVkEVk1k2iKBJkotYv3ajgYWr56xxCbY5vE1IpZBRNhhaUDNZo0kJgi%2b6knXZ4m7JHt%2fGtJeP%2bNTxHSUL2ELlTIiT3ENlPYD6FdH6ZBxT1OneW%2f0ih%2fcN7vctb5B5Qwa1ez7ZjN2QxgBYkFDDHHTs42ej5eF2BysWAQDSUr%2fcySyGxcfPveIpfQEdrynGKR6z3OYqkFnP%2bYRDQp2rt1qt0FwCB4L9cg05TQlSSTJVGfPDrtcqjvKY4c9hWwSHtE8jMRpeCYO4Es%2bWgwr5DjzMicmuZo%2f4Ycr16jpN7xlDJdJ8iCFZxbSGgVC7ksVlGE8wlfWPI4KTuX5U5s61eNWPTlAC%2fOGb8grtw%2ffzizoIX9D6ZUMvslGLQIp%2fvNmNQkZy8HhNoV6Lns%2figITP%2fpN0H8h9HjUTl9qn65xFOEVpc0motSy8alcTPtTRKq5Jvc4Ao0x3N%2fvCB1v4Epx7XC0UpFbw8TrYEvAczEfGsGM', - 'extra.destination.naics' : 518210, - 'extra.destination.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.http_agent' : 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET4.0C; .NET4.0E)', - 'extra.infection' : 'boaxxe', - 'extra.tag' : '3ve', - 'feed.name' : 'Sinkhole-Events-HTTP IPv6', - 'malware.name' : 'boaxxe', - 'protocol.application' : 'http', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 7713, - 'source.geolocation.cc' : 'ID', - 'source.geolocation.city' : 'JAKARTA', - 'source.geolocation.region' : 'JAKARTA RAYA', - 'source.ip' : '2001:448a:1082:4d9b:7491:bf9e:3d5f:a634', - 'source.port' : 49431, - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2022-03-02T09:14:19+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : '3ve', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'infected-system', - 'destination.asn' : 6939, - 'destination.fqdn' : 'devps.net', - 'destination.geolocation.cc' : 'US', - 'destination.geolocation.city' : 'FREMONT', - 'destination.geolocation.region' : 'CALIFORNIA', - 'destination.ip' : '2001:470:1:332::ef', - 'destination.port' : 80, - 'destination.url' : 'http://devps.net/QKMSvF2hl11j%2fbMkyPbF5EpHYhd6VWTG4u19K3Rt7JGU3lMYRqpq8wPYEuOGKKeidKW3pefVfKSjBnL0cXizZbmuWWu8AQNRqw5g9Ny5vZtiv638XKoWwCLuUOTISTV%2fLcpcS1%2f22NjWqgXkHGISAuyVtafqyCC%2f5cA0eYg9Me8VzAIFDdTArogQOdYhElf2xluhEFPsstGQ%2bwrM4VmKHJpzyjD7Y%2fN%2bQV3wnZNdVkEVk1k2iKBJkotYv3ajgYWr56xxCbY5vE1IpZBRNhhaUDNZo0kJgi%2b6knXZ4m7JHt%2fGtJeP%2bNTxHSUL2ELlTIiT3ENlPYD6FdH6ZBxT1OneW%2f0ih%2fcN7vctb5B5Qwa1ez7ZjN2QxgBYkFDDHHTs42ej5eF2BysWAQDSUr%2fcySyGxcfPveIpfQEdrynGKR6z3OYqkFnP%2bYRDQp2rt1qt0FwCB4L9cg05TQlSSTJVGfPDrtcqjvKY4c9hWwSHtE8jMRpeCYO4Es%2bWgwr5DjzMicmuZo%2f4Ycr16jpN7xlDJdJ8iCFZxbSGgVC7ksVlGE8wlfWPI4KTuX5U5s61eNWPTlAC%2fOGb8grtw%2ffzizoIX9D6ZUMvslGLQIp%2fvNmNQkZy8HhNoV6Lns%2figITP%2fpN0H8h9HjUTl9qn65xFOEVpc0motSy8alcTPtTRKq5Jvc4Ao0x3N%2fvCB1v4Epx7XC0UpFbw8TrYEvAczEfGsGM', - 'extra.destination.naics' : 518210, - 'extra.destination.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.http_agent' : 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET4.0C; .NET4.0E)', - 'extra.infection' : 'boaxxe', - 'extra.tag' : '3ve', - 'feed.name' : 'Sinkhole-Events-HTTP IPv6', - 'malware.name' : 'boaxxe', - 'protocol.application' : 'http', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 7713, - 'source.geolocation.cc' : 'ID', - 'source.geolocation.city' : 'JAKARTA', - 'source.geolocation.region' : 'JAKARTA RAYA', - 'source.ip' : '2001:448a:1082:4d9b:7491:bf9e:3d5f:a634', - 'source.port' : 49460, - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2022-03-02T09:15:10+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : '3ve', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'infected-system', - 'destination.asn' : 6939, - 'destination.fqdn' : 'devps.net', - 'destination.geolocation.cc' : 'US', - 'destination.geolocation.city' : 'FREMONT', - 'destination.geolocation.region' : 'CALIFORNIA', - 'destination.ip' : '2001:470:1:332::fe', - 'destination.port' : 80, - 'destination.url' : 'http://devps.net/WMoUNCvuKGzdqSCeQcadP1%2f0B%2f3bzpOmyKBU85Z25HVOhvDQUPFl%2fk8uOcLewS%2b1BsuHXalRAOIgGOYYs2igj6UX8FkdCAmDewWPvfDhPD45nwd2tx1lLf2IoIfuOtIpGR6bN5Q6hGpSBgfERqCa0ImHcwfcZ2EdO%2fWvg7R8H6SLcTiuUC0I4pzvlWt1CRLgLdIEU1hZ0nnFHIHhchb6D7ITEgBQ2chQDxy5TJMrGjm4Dac6dKl%2ft5uYhRhSjAHkLLtgrJjsqVtVbelTAkt5kdcqLlO09m1SH%2fvtAb%2fOvR2DbhBss7%2f64DG7g6cAnghNA6JrFn1uW7sw%2bnKH8koKQwzUjdSsbrQAvmg4r0KDDW8Diq64gfDzxFWkzCLOYifc%2fwlinXPCl7aJiNCoieDC1U98RNQg%2f5td4SZmJnDQ2%2f96CPbFeSpCez5WD1rCjrxLj1h2cqzIgkydEWACceWP9ztxc4QaObzEcgOGxbRckWC7H2aaLeT8jaYEYdKi1pwEKChSL3YdEt4ZIb2IFrWwzNaXEpQzFXf07f902OEdI9vVA1ZdEOBPG6rAIkzMdebfprfVyhKEWtrCd3Skg3COUFtRQks5jzG1nv4sVGijTfSgyn6xE9Taka668Nycik6nmHy8Huj3oC01j3tee%2f1Z3eI6tV7lgM5d3uFJ84slRGHUCwMfVozOGmZRwNo%2fz%2bA', - 'extra.destination.naics' : 518210, - 'extra.destination.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.http_agent' : 'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko', - 'extra.infection' : 'boaxxe', - 'extra.source.naics' : 517311, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : '3ve', - 'feed.name' : 'Sinkhole-Events-HTTP IPv6', - 'malware.name' : 'boaxxe', - 'protocol.application' : 'http', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 11427, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'GARLAND', - 'source.geolocation.region' : 'TEXAS', - 'source.ip' : '2603:8080:b20a:dc00:f06e:8304:71f6:27e2', - 'source.port' : 62932, - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2022-03-02T14:15:10+00:00' -} - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_honeypot_brute_force.py b/intelmq/tests/bots/parsers/shadowserver/test_honeypot_brute_force.py deleted file mode 100644 index c376a73fbd..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_honeypot_brute_force.py +++ /dev/null @@ -1,72 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/event4_honeypot_brute_force.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Honeypot-Brute-Force-Events', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_honeypot_brute_force.csv" - } -EVENTS = [{'__type': 'Event', - 'classification.identifier': 'ssh', - 'classification.taxonomy': 'intrusion-attempts', - 'classification.type': 'brute-force', - 'extra.client_version': "b'SSH-2.0-Go'", - 'destination.asn': 26832, - 'destination.geolocation.cc': 'CA', - 'destination.geolocation.city': 'MONTREAL', - 'destination.geolocation.region': 'QUEBEC', - 'destination.ip': '162.250.1.2', - 'destination.port': 22, - 'extra.application': 'ssh', - 'extra.end_time': '2021-03-27T00:00:01.710968+00:00', - 'extra.public_source': 'CAPRICA-EU', - 'extra.start_time': '2021-03-27T00:00:00.521730+00:00', - 'malware.name': 'ssh-brute-force', - 'feed.name': 'Honeypot-Brute-Force-Events', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 209588, - 'source.geolocation.cc': 'NL', - 'source.geolocation.city': 'AMSTERDAM', - 'source.geolocation.region': 'NOORD-HOLLAND', - 'source.ip': '141.98.1.2', - 'source.port': 30123, - 'time.source': '2021-03-27T00:00:00+00:00'}, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_honeypot_ddos_amp.py b/intelmq/tests/bots/parsers/shadowserver/test_honeypot_ddos_amp.py deleted file mode 100644 index e95e59dcb3..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_honeypot_ddos_amp.py +++ /dev/null @@ -1,91 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/event4_honeypot_ddos_amp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Amplification DDoS Victim', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2019-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_honeypot_ddos_amp.csv" - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Amplification DDoS Victim', - 'classification.identifier': 'amplification-ddos-victim', - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'time.observation': '2019-01-01T00:00:00+00:00', - 'time.source': '2021-03-28T00:00:02+00:00', - 'source.ip': '107.141.1.2', - 'destination.port': 389, - 'source.reverse_dns': '192-0-2-10.example.net', - 'source.asn': 7018, - 'source.geolocation.cc': 'US', - 'source.geolocation.region': 'VISALIA', - 'source.geolocation.city': 'VISALIA', - 'source.geolocation.region': 'CALIFORNIA', - 'extra.end_time': '2021-03-28T00:20:22+00:00', - 'extra.public_source': 'CISPA', - 'extra.source.naics': 517311, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'malware.name': 'ddos-amplification', - 'source.reverse_dns': '107-141-x-x.lightspeed.frsnca.sbcglobal.net', - }, - {'__type': 'Event', - 'feed.name': 'Amplification DDoS Victim', - 'classification.identifier': 'amplification-ddos-victim', - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'time.observation': '2019-01-01T00:00:00+00:00', - 'time.source': '2021-03-28T00:00:02+00:00', - 'source.ip': '74.59.3.4', - 'destination.port': 389, - 'source.reverse_dns': 'modemcablex-x-59-74.mc.videotron.ca', - 'source.asn': 5769, - 'source.geolocation.cc': 'CA', - 'source.geolocation.city': 'CHICOUTIMI', - 'source.geolocation.region': 'QUEBEC', - 'extra.end_time': '2021-03-28T00:13:50+00:00', - 'extra.public_source': 'CISPA', - 'extra.source.naics': 517311, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'malware.name': 'ddos-amplification', - }, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_malware_url.py b/intelmq/tests/bots/parsers/shadowserver/test_malware_url.py deleted file mode 100644 index b19b200b5f..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_malware_url.py +++ /dev/null @@ -1,107 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/malware_url.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Malware URL', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-malware_url-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'malware-url', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'extra.application' : 'http', - 'source.url' : 'http://41.86.0.0:50008/Mozi.m', - 'feed.name' : 'Malware URL', - 'malware.hash.sha256' : '12013662c71da69de977c04cd7021f13a70cf7bed4ca6c82acbc100464d4b0ef', - 'malware.name' : 'cve-2016-10372', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 37203, - 'source.geolocation.cc' : 'LR', - 'source.geolocation.city' : 'MONROVIA', - 'source.geolocation.region' : 'MONTSERRADO', - 'source.ip' : '41.86.0.0', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-07T00:02:07+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'malware-url', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'extra.application' : 'http', - 'extra.source.naics' : 517311, - 'source.url' : 'http://42.225.0.0:38173/Mozi.m', - 'feed.name' : 'Malware URL', - 'malware.name' : 'cve-2018-10562', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 4837, - 'source.geolocation.cc' : 'CN', - 'source.geolocation.city' : 'ZHUMADIAN', - 'source.geolocation.region' : 'HENAN SHENG', - 'source.ip' : '42.225.0.0', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-07T00:03:14+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'malware-url', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'extra.application' : 'http', - 'extra.source.naics' : 517311, - 'source.url' : 'http://211.52.0.0:53029/Mozi.m', - 'feed.name' : 'Malware URL', - 'malware.name' : 'cve-2018-10562', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 4766, - 'source.geolocation.cc' : 'KR', - 'source.geolocation.city' : 'SAGOK-MYEON', - 'source.geolocation.region' : 'CHUNGCHEONGNAM-DO', - 'source.ip' : '211.52.0.0', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-07T00:10:26+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_phish_url.py b/intelmq/tests/bots/parsers/shadowserver/test_phish_url.py deleted file mode 100644 index 0783372f91..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_phish_url.py +++ /dev/null @@ -1,106 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/phish_url.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Phish URL', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-phish_url-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'phish-url', - 'classification.taxonomy' : 'fraud', - 'classification.type' : 'phishing', - 'source.fqdn' : 'priceless-pare.example.net', - 'extra.source' : 'openphish.com', - 'extra.source.naics' : 518210, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'source.url' : 'https://priceless-pare.example.net/Postal-/acec6/', - 'feed.name' : 'Phish URL', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'BUFFALO', - 'source.geolocation.region' : 'NEW YORK', - 'source.ip' : '172.245.0.0', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-02-01T08:00:07+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'phish-url', - 'classification.taxonomy' : 'fraud', - 'classification.type' : 'phishing', - 'source.fqdn' : 'mailyahooattt.example.net', - 'extra.source' : 'openphish.com', - 'extra.source.sector' : 'Professional, Scientific, and Technical Services', - 'source.url' : 'https://mailyahooattt.example.net/', - 'feed.name' : 'Phish URL', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'SAN FRANCISCO', - 'source.geolocation.region' : 'CALIFORNIA', - 'source.ip' : '199.34.0.0', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-02-01T08:00:07+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'phish-url', - 'classification.taxonomy' : 'fraud', - 'classification.type' : 'phishing', - 'source.fqdn' : 'www.example.net', - 'extra.source' : 'openphish.com', - 'extra.source.naics' : 519130, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'source.url' : 'https://www.example.net/viewer/vbid-730ec2b1-omsttuer', - 'feed.name' : 'Phish URL', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'DRAPER', - 'source.geolocation.region' : 'UTAH', - 'source.ip' : '216.58.0.0', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-02-01T08:00:07+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_population_http_proxy.py b/intelmq/tests/bots/parsers/shadowserver/test_population_http_proxy.py deleted file mode 100644 index e9f11a47c3..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_population_http_proxy.py +++ /dev/null @@ -1,130 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/population_http_proxy.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible HTTP Proxy', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-population_http_proxy-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'accessible-http-proxy', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'extra.connection': 'keep-alive', - 'extra.content_length': 3741, - 'extra.content_type': 'text/html;charset=utf-8', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 407, - 'extra.http_date': '2010-02-10T00:00:00+00:00', - 'extra.http_reason': 'Proxy Authentication Required', - 'extra.proxy_authenticate': 'Basic realm=\\\\"Squid proxy-caching web ' - 'server\\"\\""', - 'extra.server': 'squid/4.10', - 'feed.name': 'Accessible HTTP Proxy', - 'malware.name': 'http-connect-proxy-closed', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 3128, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-http-proxy', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'extra.connection': 'keep-alive', - 'extra.content_length': 3833, - 'extra.content_type': 'text/html;charset=utf-8', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 407, - 'extra.http_date': '2010-02-10T00:00:01+00:00', - 'extra.http_reason': 'Proxy Authentication Required', - 'extra.proxy_authenticate': 'Basic realm=\\\\"00:23:24:43:1c:34\\"\\""', - 'feed.name': 'Accessible HTTP Proxy', - 'malware.name': 'http-connect-proxy-closed', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 3128, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-http-proxy', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'extra.connection': 'keep-alive', - 'extra.content_length': 179, - 'extra.content_type': 'text/html;charset=utf-8', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 407, - 'extra.http_date': '2010-02-10T00:00:02+00:00', - 'extra.http_reason': 'Proxy Authentication Required', - 'extra.proxy_authenticate': 'Basic realm=\\\\"Proxy\\"\\""', - 'feed.name': 'Accessible HTTP Proxy', - 'malware.name': 'http-connect-proxy-closed', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 3128, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_sandbox_conn.py b/intelmq/tests/bots/parsers/shadowserver/test_sandbox_conn.py deleted file mode 100644 index c5da823465..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_sandbox_conn.py +++ /dev/null @@ -1,99 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/sandbox_conn.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Sandbox Connections', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-sandbox_conn-test.csv", - } -EVENTS = [ - { - '__type' : 'Event', - 'classification.identifier' : 'sandbox-conn', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'destination.fqdn' : 'time.windows.com', - 'feed.name' : 'Sandbox Connections', - 'malware.hash.md5' : 'b575ce6dcce6502a8431db5610135c25', - 'protocol.transport' : 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 8075, - 'source.geolocation.cc' : 'US', - 'source.ip' : '40.119.6.228', - 'source.port' : 123, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:00:03+00:00' -}, - { - '__type' : 'Event', - 'classification.identifier' : 'sandbox-conn', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'feed.name' : 'Sandbox Connections', - 'malware.hash.md5' : 'c0d947f9a8685b0d9f3efdba966389c2', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 3356, - 'source.geolocation.cc' : 'US', - 'source.ip' : '8.252.70.126', - 'source.port' : 80, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:00:03+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'sandbox-conn', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'feed.name' : 'Sandbox Connections', - 'malware.hash.md5' : 'c0d947f9a8685b0d9f3efdba966389c2', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 8075, - 'source.geolocation.cc' : 'US', - 'source.ip' : '52.109.8.22', - 'source.port' : 443, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:00:03+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_sandbox_dns.py b/intelmq/tests/bots/parsers/shadowserver/test_sandbox_dns.py deleted file mode 100644 index 70cf1eee5e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_sandbox_dns.py +++ /dev/null @@ -1,95 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/sandbox_dns.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Sandbox DNS', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-sandbox_dns-test.csv", - } -EVENTS = [ - { - '__type' : 'Event', - 'classification.identifier' : 'sandbox-dns', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.request' : 'time.windows.com', - 'extra.response' : '40.119.6.228', - 'extra.dns_query_type' : 'A', - 'feed.name' : 'Sandbox DNS', - 'malware.hash.md5' : 'b575ce6dcce6502a8431db5610135c25', - 'protocol.application' : 'dns', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:00:02+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'sandbox-dns', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.request' : 'time.windows.com', - 'extra.response' : '40.119.6.228', - 'extra.dns_query_type' : 'A', - 'feed.name' : 'Sandbox DNS', - 'malware.hash.md5' : '807679198a39c80d3ca07e60fd51b581', - 'protocol.application' : 'dns', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:00:08+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'sandbox-dns', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.request' : 'client-office365-tas.msedge.net', - 'extra.response' : '13.107.5.88', - 'extra.dns_query_type' : 'A', - 'feed.name' : 'Sandbox DNS', - 'malware.hash.md5' : 'd97e973b9bf073bd3a217425259cea26', - 'protocol.application' : 'dns', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:00:20+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_sandbox_url.py b/intelmq/tests/bots/parsers/shadowserver/test_sandbox_url.py deleted file mode 100644 index 91b0154b84..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_sandbox_url.py +++ /dev/null @@ -1,104 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/sandbox_url.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Sandbox URL', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-sandbox_url-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'sandbox-url', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'destination.fqdn' : 'www.msftncsi.com', - 'extra.http_request_method' : 'GET', - 'destination.url' : 'http://www.msftncsi.com/ncsi.txt', - 'extra.user_agent' : 'Microsoft NCSI', - 'feed.name' : 'Sandbox URL', - 'malware.hash.md5' : '37514b54e679a5313334e830ad780ec7', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 20940, - 'source.geolocation.cc' : 'US', - 'source.ip' : '23.196.47.89', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:13+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'sandbox-url', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'destination.fqdn' : 'www.download.windowsupdate.com', - 'extra.http_request_method' : 'GET', - 'destination.url' : 'http://www.download.windowsupdate.com/msdownload/update/v3/static/trustedr/en/authrootstl.cab', - 'extra.user_agent' : 'Microsoft-CryptoAPI/6.1', - 'feed.name' : 'Sandbox URL', - 'malware.hash.md5' : '37514b54e679a5313334e830ad780ec7', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 15133, - 'source.geolocation.cc' : 'US', - 'source.ip' : '72.21.81.240', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:28+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'sandbox-url', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'destination.fqdn' : 'crl.microsoft.com', - 'extra.http_request_method' : 'GET', - 'destination.url' : 'http://crl.microsoft.com/pki/crl/products/MicTimStaPCA_2010-07-01.crl', - 'extra.user_agent' : 'Microsoft-CryptoAPI/6.1', - 'feed.name' : 'Sandbox URL', - 'malware.hash.md5' : 'e97ea2820c0d79f3f3ca241d4dcd1060', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 20940, - 'source.geolocation.cc' : 'US', - 'source.ip' : '23.56.4.57', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:08:24+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_adb.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_adb.py deleted file mode 100644 index 6bc6e61461..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_adb.py +++ /dev/null @@ -1,98 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_adb.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible ADB', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2018-07-30T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_adb-test-test.csv", - - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Accessible ADB', - 'time.observation': '2018-07-30T00:00:00+00:00', - 'time.source': '2018-07-26T02:07:16+00:00', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-adb', - 'protocol.application': 'adb', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 3462, - 'source.geolocation.cc': 'TW', - 'source.geolocation.city': 'TAOYUAN CITY', - 'source.geolocation.region': 'TAOYUAN COUNTY', - 'source.ip': '36.239.124.210', - 'source.port': 5555, - 'extra.name': 'hlteuc', - 'extra.model': 'SAMSUNG-SM-N900A', - 'extra.device': 'hlteatt', - 'extra.tag': 'adb', - 'extra.naics': 518210, - 'extra.sic': 737415, - 'source.reverse_dns': '36-239-124-210.dynamic-ip.hinet.net', - }, - {'__type': 'Event', - 'feed.name': 'Accessible ADB', - 'time.observation': '2018-07-30T00:00:00+00:00', - 'time.source': '2018-07-26T02:07:16+00:00', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-adb', - 'protocol.application': 'adb', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn': 3462, - 'source.geolocation.cc': 'TW', - 'source.geolocation.city': 'TAIPEI', - 'source.geolocation.region': 'TAIPEI CITY', - 'source.ip': '36.236.108.107', - 'source.port': 5555, - 'extra.name': 'marlin', - 'extra.model': 'Pixel XL', - 'extra.device': 'marlin', - 'extra.features': 'cmd,shell_v2', - 'extra.naics': 518210, - 'extra.sic': 737415, - 'extra.tag': 'adb', - 'source.reverse_dns': '36-236-108-107.dynamic-ip.hinet.net', - }, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_afp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_afp.py deleted file mode 100644 index cc30b1e4c0..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_afp.py +++ /dev/null @@ -1,106 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_afp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible AFP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2018-07-30T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_afp-test-test.csv", - - } -EVENTS = [{ - '__type': 'Event', - 'feed.name': 'Accessible AFP', - "classification.identifier": "accessible-afp", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.afp_versions": "AFP3.3,AFP3.2,AFP3.1", - "extra.flags": "SupportsCopyFile,SupportsChgPwd,SupportsServerMessages,SupportsServerSignature,SupportsTCP/IP,SupportsSrvrNotifications,SupportsReconnect,SupportsOpenDirectory,SupportsUTF8Servername,SupportsUUIDs,SupportsSuperClient", - "extra.machine_type": "TimeCapsule8,119", - "extra.naics": 517311, - "extra.network_address": "198.33.24.165:548,10.0.1.1:548,fe80:0008:0000:0000:6e70:9fff:fed4::548,fe80:0009:0000:0000:6e70:9fff:fed4::548,179.24.24.165 (DNS address),", - "extra.server_name": "airport-time-capsule-de-jack", - "extra.signature": "4338364e37364442463948350069672d", - "extra.tag": "afp", - "extra.uams": "DHCAST128,DHX2,SRP,Recon1", - "extra.utf8_servername": "AirPort Time Capsule de jack", - "protocol.application": "afp", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 6057, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.13.34.22", - "source.port": 548, - "source.reverse_dns": "host.local", - "time.observation": "2018-07-30T00:00:00+00:00", - "time.source": "2019-09-04T05:05:53+00:00" -}, -{ - '__type': 'Event', - 'feed.name': 'Accessible AFP', - "classification.identifier": "accessible-afp", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.afp_versions": "AFP3.3,AFP3.2,AFP3.1", - "extra.flags": "SupportsCopyFile,SupportsChgPwd,SupportsServerMessages,SupportsServerSignature,SupportsTCP/IP,SupportsSrvrNotifications,SupportsReconnect,SupportsOpenDirectory,SupportsUTF8Servername,SupportsUUIDs,SupportsSuperClient", - "extra.machine_type": "TimeCapsule8,119", - "extra.naics": 517311, - "extra.network_address": "0.0.0.1:548,10.0.1.1:548,198.33.42.1:548,fe80:000b:0000:0000:dea4:caff:feba::548,fe80:000c:0000:0000:dea4:caff:feba::548,fe80:000d:0000:0000:4c7d:ffff:fec7::548,0.0.0.1 (DNS address),", - "extra.server_name": "time-capsule-del-jack", - "extra.signature": "433836544b303147463948360069672d", - "extra.tag": "afp", - "extra.uams": "DHCAST128,DHX2,SRP,Recon1", - "extra.utf8_servername": "Time Capsule del Jack", - "protocol.application": "afp", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 6057, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.40.27.212", - "source.port": 548, - "source.reverse_dns": "host.local", - "time.observation": "2018-07-30T00:00:00+00:00", - "time.source": "2019-09-04T05:05:56+00:00" - },] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_amqp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_amqp.py deleted file mode 100644 index df707f30b0..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_amqp.py +++ /dev/null @@ -1,144 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_amqp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible AMQP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_amqp-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'accessible-amqp', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.capabilities' : 'publisher_confirms,exchange_exchange_bindings,basic.nack,consumer_cancel_notify,connection.blocked,consumer_priorities,authentication_failure_close,per_consumer_qos', - 'extra.class' : '10', - 'extra.cluster_name' : 'rabbit@iZuf63m0nnq9bwf7lhjxrkZ', - 'extra.locales' : 'en_US', - 'extra.mechanisms' : 'PLAIN AMQPLAIN', - 'extra.message_length' : 509, - 'extra.method' : '10', - 'extra.platform' : 'Erlang/OTP', - 'extra.product' : 'RabbitMQ', - 'extra.product_version' : '3.3.5', - 'extra.naics' : 518210, - 'extra.tag' : 'amqp', - 'extra.version_minor' : '9', - 'feed.name' : 'Accessible AMQP', - 'protocol.application' : 'amqp', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 37963, - 'source.geolocation.cc' : 'CN', - 'source.geolocation.city' : 'SHANGHAI', - 'source.geolocation.region' : 'SHANGHAI SHI', - 'source.ip' : '47.103.0.0', - 'source.port' : 5672, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T04:32:13+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'accessible-amqp', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.capabilities' : 'publisher_confirms,exchange_exchange_bindings,basic.nack,consumer_cancel_notify,connection.blocked,consumer_priorities,authentication_failure_close,per_consumer_qos,direct_reply_to', - 'extra.class' : '10', - 'extra.cluster_name' : 'rabbit@mtk-breizh', - 'extra.locales' : 'en_US', - 'extra.mechanisms' : 'AMQPLAIN PLAIN', - 'extra.message_length' : 509, - 'extra.method' : '10', - 'extra.platform' : 'Erlang/OTP 24.0.3', - 'extra.product' : 'RabbitMQ', - 'extra.product_version' : '3.8.19', - 'extra.naics' : 518210, - 'extra.tag' : 'amqp', - 'extra.version_minor' : '9', - 'feed.name' : 'Accessible AMQP', - 'protocol.application' : 'amqp', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 16276, - 'source.geolocation.cc' : 'DE', - 'source.geolocation.city' : 'SAARBRUCKEN', - 'source.geolocation.region' : 'SAARLAND', - 'source.ip' : '141.95.0.0', - 'source.port' : 5672, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T04:32:13+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'accessible-amqp', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.capabilities' : 'publisher_confirms,exchange_exchange_bindings,basic.nack,consumer_cancel_notify,connection.blocked,consumer_priorities,authentication_failure_close,per_consumer_qos,direct_reply_to', - 'extra.class' : '10', - 'extra.cluster_name' : 'rabbit@1397a0e9629b', - 'extra.locales' : 'en_US', - 'extra.mechanisms' : 'PLAIN AMQPLAIN', - 'extra.message_length' : 509, - 'extra.method' : '10', - 'extra.platform' : 'Erlang/OTP 24.2', - 'extra.product' : 'RabbitMQ', - 'extra.product_version' : '3.9.11', - 'extra.naics' : 454110, - 'extra.tag' : 'amqp', - 'extra.version_minor' : '9', - 'feed.name' : 'Accessible AMQP', - 'protocol.application' : 'amqp', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 14618, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'ASHBURN', - 'source.geolocation.region' : 'VIRGINIA', - 'source.ip' : '54.234.0.0', - 'source.port' : 5672, - 'source.reverse_dns' : 'ec2-54.234.0.0.compute-1.amazonaws.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T04:32:13+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ard.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ard.py deleted file mode 100644 index 4d8420c3bb..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ard.py +++ /dev/null @@ -1,111 +0,0 @@ -# SPDX-FileCopyrightText: 2020 Tomas Bellus -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ard.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible ARD', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2020-07-20T00:00:00+00:00", - "extra.file_name": "2020-01-01-scan_ard-test-test.csv", - - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'accessible-ard', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 201.2, - 'extra.machine_name': 'Macmini (radio)', - 'extra.response_size': 1006, - 'extra.tag': 'ard', - 'feed.name': 'Accessible ARD', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 3283, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-ard', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 201.2, - 'extra.machine_name': 'biuro-rip-org-pl', - 'extra.response_size': 1006, - 'extra.tag': 'ard', - 'feed.name': 'Accessible ARD', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 3283, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-ard', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 201.2, - 'extra.machine_name': '127.0.0.1', - 'extra.response_size': 1006, - 'extra.tag': 'ard', - 'feed.name': 'Accessible ARD', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 3283, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_chargen.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_chargen.py deleted file mode 100644 index 3b72baa8db..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_chargen.py +++ /dev/null @@ -1,110 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_chargen.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open Chargen', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_chargen-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-chargen', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 74.0, - 'extra.response_size': 74, - 'extra.tag': 'chargen', - 'feed.name': 'Open Chargen', - 'protocol.application': 'chargen', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 19, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-chargen', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 74.0, - 'extra.response_size': 74, - 'extra.tag': 'chargen', - 'feed.name': 'Open Chargen', - 'protocol.application': 'chargen', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 19, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-chargen', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 74.0, - 'extra.response_size': 74, - 'extra.sector': 'Government', - 'extra.tag': 'chargen', - 'feed.name': 'Open Chargen', - 'protocol.application': 'chargen', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 19, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_cisco_smart_install.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_cisco_smart_install.py deleted file mode 100644 index 46c963a79e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_cisco_smart_install.py +++ /dev/null @@ -1,82 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_cisco_smart_install.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible Cisco Smart Install', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_cisco_smart_install-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Accessible Cisco Smart Install', - 'classification.identifier': 'accessible-cisco-smart-install', - 'classification.type': 'vulnerable-system', - 'classification.taxonomy': 'vulnerable', - 'protocol.application': 'cisco-smart-install', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 8559, - 'source.geolocation.cc': 'AT', - 'source.geolocation.city': 'VIENNA', - 'source.geolocation.region': 'WIEN', - 'source.ip': '198.51.100.103', - 'source.port': 4786, - 'extra.tag': 'cisco-smart-install', - 'source.reverse_dns': '198-51-100-103.example.net', - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2017-11-18T08:42:45+00:00'}, - {'__type': 'Event', - 'feed.name': 'Accessible Cisco Smart Install', - 'classification.identifier': 'accessible-cisco-smart-install', - 'classification.type': 'vulnerable-system', - 'classification.taxonomy': 'vulnerable', - 'protocol.application': 'cisco-smart-install', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn': 35609, - 'source.geolocation.cc': 'AT', - 'source.geolocation.city': 'VIENNA', - 'source.geolocation.region': 'WIEN', - 'source.ip': '198.51.100.218', - 'source.port': 4786, - 'extra.tag': 'cisco-smart-install', - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2017-11-18T08:47:54+00:00'}, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_coap.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_coap.py deleted file mode 100644 index 773fc04d51..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_coap.py +++ /dev/null @@ -1,121 +0,0 @@ -# SPDX-FileCopyrightText: 2020 Thomas Hungenberg -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_coap.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible-CoAP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2020-06-29T00:00:00+00:00", - "extra.file_name": "2020-06-28-scan_coap-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'accessible-coap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 2.05, - 'extra.response': ',,', - 'extra.response_size': 43, - 'extra.tag': 'coap', - 'extra.version': '2', - 'feed.name': 'Accessible-CoAP', - 'protocol.application': 'coap', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 5683, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-coap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 5.38, - 'extra.response': ',,,,,,,,,', - 'extra.response_size': 113, - 'extra.tag': 'coap', - 'extra.version': '2', - 'feed.name': 'Accessible-CoAP', - 'protocol.application': 'coap', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 5683, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-coap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 113.5, - 'extra.response': '`EsjAy************************************************************|CoAP ' - 'RFC 7252 ' - '|************************************************************|This ' - 'server is using the Eclipse Californium (Cf) CoAP ' - 'framework|published under EPL+EDL: ' - 'http://www.eclipse.org/californium/||(c) 2014, 2015, 2016 ' - 'Institute for Pervasive Computing, ETH Zurich and ' - 'others|************************************************************', - 'extra.response_size': 454, - 'extra.tag': 'coap', - 'extra.version': '1', - 'feed.name': 'Accessible-CoAP', - 'protocol.application': 'coap', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 5683, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_couchdb.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_couchdb.py deleted file mode 100644 index 1bf6f321c6..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_couchdb.py +++ /dev/null @@ -1,128 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_couchdb.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible CouchDB Server', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_couchdb-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-couchdb', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.couchdb_message' : 'Welcome', - 'extra.couchdb_version' : '1.6.1', - 'extra.server_version' : 'CouchDB/1.6.1 (Erlang OTP/18)', - 'extra.tag' : 'couchdb', - 'extra.vendor' : 'Ubuntu 16.04', - 'extra.visible_databases' : '_replicator;_users;test;shops;god', - 'feed.name' : 'Accessible CouchDB Server', - 'protocol.application' : 'couchdb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 5984, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-couchdb', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.couchdb_message' : 'Welcome', - 'extra.couchdb_version' : '3.2.1', - 'extra.features' : 'access-ready,partitioned,pluggable-storage-engines,reshard,scheduler', - 'extra.git_sha' : '244d428af', - 'extra.server_version' : 'CouchDB/3.2.1 (Erlang OTP/23)', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'couchdb', - 'extra.vendor' : 'The Apache Software Foundation', - 'feed.name' : 'Accessible CouchDB Server', - 'protocol.application' : 'couchdb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 5984, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-couchdb', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.couchdb_message' : 'Welcome', - 'extra.couchdb_version' : '3.2.1', - 'extra.features' : 'access-ready,partitioned,pluggable-storage-engines,reshard,scheduler', - 'extra.git_sha' : '244d428af', - 'extra.server_version' : 'CouchDB/3.2.1 (Erlang OTP/20)', - 'extra.source.sector' : 'Retail Trade', - 'extra.tag' : 'couchdb', - 'extra.vendor' : 'The Apache Software Foundation', - 'feed.name' : 'Accessible CouchDB Server', - 'protocol.application' : 'couchdb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 5984, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_cwmp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_cwmp.py deleted file mode 100644 index b508b64508..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_cwmp.py +++ /dev/null @@ -1,103 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_cwmp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible CWMP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2018-07-30T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_cwmp-test-test.csv", - - } -EVENTS = [{ - '__type': 'Event', - 'feed.name': 'Accessible CWMP', - "classification.identifier": "open-cwmp", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.connection": "keep-alive", - "extra.content_length": 5678, - "extra.content_type": "text/html", - "extra.date": "Wed, 04 Sep 2019 07:42:37 GMT", - "extra.http": "HTTP/1.1", - "extra.http_code": 200, - "extra.http_reason": "OK", - "extra.naics": 517311, - "extra.server": "DNVRS-Webs", - "extra.tag": "cwmp", - "protocol.application": "cwmp", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 5678, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.142", - "source.port": 30005, - "time.observation": "2018-07-30T00:00:00+00:00", - "time.source": "2019-09-04T10:44:55+00:00" -}, -{ - '__type': 'Event', - 'feed.name': 'Accessible CWMP', - "classification.identifier": "open-cwmp", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.content_type": "text/html", - "extra.http": "HTTP/1.1", - "extra.http_code": 404, - "extra.http_reason": "Not Found", - "extra.naics": 517311, - "extra.server": "RomPager/4.07 UPnP/1.0", - "extra.tag": "cwmp", - "extra.transfer_encoding": "chunked", - "protocol.application": "cwmp", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 5678, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.162", - "source.port": 5678, - "source.reverse_dns": "localhost.localdomain", - "time.observation": "2018-07-30T00:00:00+00:00", - "time.source": "2019-09-04T11:06:50+00:00" - },] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_db2.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_db2.py deleted file mode 100644 index 423ebe8c53..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_db2.py +++ /dev/null @@ -1,91 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Sebastian Wagner -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_db2.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "ShadowServer Open-DB2-Discovery-Service", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_db2-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-db2-discovery-service', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 14.9, - 'extra.db2_hostname': 'NOWAK_SERWER', - 'extra.servername': 'node01.example.com', - 'extra.size': 298, - 'extra.tag': 'db2', - 'feed.name': 'ShadowServer Open-DB2-Discovery-Service', - 'protocol.application': 'db2', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 523, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-db2-discovery-service', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 14.9, - 'extra.db2_hostname': 'SPZOZ-DZIEWIN', - 'extra.servername': 'node02.example.com', - 'extra.size': 298, - 'extra.tag': 'db2', - 'feed.name': 'ShadowServer Open-DB2-Discovery-Service', - 'protocol.application': 'db2', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 523, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ddos_middlebox.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ddos_middlebox.py deleted file mode 100644 index 9038a79ef1..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ddos_middlebox.py +++ /dev/null @@ -1,119 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ddos_middlebox.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'DDoS Middlebox', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_ddos_middlebox-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ddos-middlebox', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.amplification' : 2, - 'extra.bytes' : 99, - 'extra.method' : 'SYN+ACK:PSH', - 'extra.source_port' : '49002', - 'feed.name' : 'DDoS Middlebox', - 'protocol.application' : 'ddos-middlebox', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 80, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ddos-middlebox', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.amplification' : 2, - 'extra.bytes' : 99, - 'extra.method' : 'SYN+ACK:PSH', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.source_port' : '41200', - 'feed.name' : 'DDoS Middlebox', - 'protocol.application' : 'ddos-middlebox', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 80, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ddos-middlebox', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.amplification' : 2, - 'extra.bytes' : 99, - 'extra.method' : 'SYN+ACK:PSH', - 'extra.source_port' : '47492', - 'feed.name' : 'DDoS Middlebox', - 'protocol.application' : 'ddos-middlebox', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 80, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_dns.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_dns.py deleted file mode 100644 index 3492f82cec..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_dns.py +++ /dev/null @@ -1,91 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_dns.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'DNS Open Resolvers', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2018-07-30T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_dns-test-test.csv", - } -EVENTS = [{ - '__type': 'Event', - 'feed.name': 'DNS Open Resolvers', - "classification.identifier": "dns-open-resolver", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.dns_version": "dnsmasq-2.66", - "extra.min_amplification": 4.619, - "extra.tag": "openresolver", - "protocol.application": "dns", - "protocol.transport": "udp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 25255, - "source.geolocation.cc": "AT", - "source.geolocation.city": "VIENNA", - "source.geolocation.region": "WIEN", - "source.ip": "198.51.100.179", - "source.port": 53, - "source.reverse_dns": "198-51-100-189.example.net", - "time.observation": "2018-07-30T00:00:00+00:00", - "time.source": "2018-04-14T00:14:34+00:00" -}, -{ - '__type': 'Event', - 'feed.name': 'DNS Open Resolvers', - "classification.identifier": "dns-open-resolver", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.dns_version": "dnsmasq-2.51", - "extra.min_amplification": 4.619, - "extra.tag": "openresolver", - "protocol.application": "dns", - "protocol.transport": "udp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 25255, - "source.geolocation.cc": "AT", - "source.geolocation.city": "VIENNA", - "source.geolocation.region": "WIEN", - "source.ip": "198.51.100.8", - "source.port": 53, - "source.reverse_dns": "198-51-100-111.example.net", - "time.observation": "2018-07-30T00:00:00+00:00", - "time.source": "2018-04-14T00:14:36+00:00" -},] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_docker.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_docker.py deleted file mode 100644 index 31d0e4417e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_docker.py +++ /dev/null @@ -1,159 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_docker.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible Docker Service', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_docker-test.csv", - } -EVENTS = [ - { - '__type' : 'Event', - 'classification.identifier' : 'open-docker', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.api_version' : '1.37', - 'extra.arch' : 'amd64', - 'extra.build_time' : '2018-05-09T22:18:36.000000000+00:00', - 'extra.content_type' : 'application/json; charset=UTF-8', - 'extra.date' : 'Fri, 06 May 2022 14:06:30 GMT', - 'extra.experimental' : 'false', - 'extra.git_commit' : 'f150324', - 'extra.go_version' : 'go1.9.5', - 'extra.http' : 'HTTP/1.1', - 'extra.http_code' : 200, - 'extra.http_reason' : 'OK', - 'extra.kernel_version' : '3.10.0-514.26.2.el7.x86_64', - 'extra.min_api_version' : '1.12', - 'extra.os.name' : 'linux', - 'extra.server' : 'Docker/18.05.0-ce (linux)', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'docker', - 'extra.version' : '18.05.0-ce', - 'feed.name' : 'Accessible Docker Service', - 'protocol.application' : 'docker', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 2375, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-docker', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.api_version' : '1.26', - 'extra.arch' : 'amd64', - 'extra.build_time' : '2022-03-02T15:25:43.414574467+00:00', - 'extra.content_type' : 'application/json', - 'extra.date' : 'Fri, 06 May 2022 14:08:07 GMT', - 'extra.experimental' : 'false', - 'extra.git_commit' : '7d71120/1.13.1', - 'extra.go_version' : 'go1.10.3', - 'extra.http' : 'HTTP/1.1', - 'extra.http_code' : 200, - 'extra.http_reason' : 'OK', - 'extra.kernel_version' : '3.10.0-693.2.2.el7.x86_64', - 'extra.min_api_version' : '1.12', - 'extra.os.name' : 'linux', - 'extra.pkg_version' : 'docker-1.13.1-209.git7d71120.el7.centos.x86_64', - 'extra.server' : 'Docker/1.13.1 (linux)', - 'extra.tag' : 'docker', - 'extra.version' : '1.13.1', - 'feed.name' : 'Accessible Docker Service', - 'protocol.application' : 'docker', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 2375, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-docker', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.api_version' : '1.37', - 'extra.arch' : 'amd64', - 'extra.build_time' : '2018-05-09T22:18:36.000000000+00:00', - 'extra.content_type' : 'application/json; charset=UTF-8', - 'extra.date' : 'Fri, 06 May 2022 14:08:06 GMT', - 'extra.experimental' : 'false', - 'extra.git_commit' : 'f150324', - 'extra.go_version' : 'go1.9.5', - 'extra.http' : 'HTTP/1.1', - 'extra.http_code' : 200, - 'extra.http_reason' : 'OK', - 'extra.kernel_version' : '3.10.0-514.26.2.el7.x86_64', - 'extra.min_api_version' : '1.12', - 'extra.os.name' : 'linux', - 'extra.server' : 'Docker/18.05.0-ce (linux)', - 'extra.tag' : 'docker', - 'extra.version' : '18.05.0-ce', - 'feed.name' : 'Accessible Docker Service', - 'protocol.application' : 'docker', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 2375, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_dvr_dhcpdiscover.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_dvr_dhcpdiscover.py deleted file mode 100644 index 01e68db94b..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_dvr_dhcpdiscover.py +++ /dev/null @@ -1,178 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_dvr_dhcpdiscover.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible DVR DHCPDiscover', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_dvr_dhcpdiscover-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-dvr-dhcpdiscover', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.alarm_input_channels': 0, - 'extra.alarm_output_channels': 0, - 'extra.amplification': 794.0, - 'extra.device_model': 'BCS-TIP3401IR-E-V', - 'extra.device_serial': '6J0E022PAG35073', - 'extra.device_type': 'IPC', - 'extra.device_vendor': 'General', - 'extra.device_version': '2.800.106F004.0.R', - 'extra.http_port': 80, - 'extra.internal_port': 37777, - 'extra.ipv4_address': '192.168.0.1', - 'extra.ipv4_dhcp_enable': False, - 'extra.ipv4_gateway': '192.168.0.240', - 'extra.ipv4_subnet_mask': '255.255.255.0', - 'extra.ipv6_address': 'fd09:4ab5:dae9:b078::1', - 'extra.ipv6_dhcp_enable': False, - 'extra.ipv6_gateway': 'fd09:4ab5:dae9:b078::ff', - 'extra.ipv6_link_local': 'fe80::3ac4:e8ff:fe03:b3e2/64', - 'extra.mac_address': '38:c4:e8:03:b3:e2', - 'extra.machine_name': '6J0E022PAG35073', - 'extra.manufacturer': 'General', - 'extra.method': 'client.notifyDevInfo', - 'extra.remote_video_input_channels': 0, - 'extra.response_size': 794, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.video_input_channels': 1, - 'extra.video_output_channels': 0, - 'feed.name': 'Accessible DVR DHCPDiscover', - 'protocol.application': 'dvrdhcpdiscover', - 'protocol.transport': 'udp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 37810, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, - {'__type': 'Event', - 'classification.identifier': 'open-dvr-dhcpdiscover', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.alarm_input_channels': 0, - 'extra.alarm_output_channels': 0, - 'extra.amplification': 761.0, - 'extra.device_model': 'HCVR', - 'extra.device_serial': '2K0488CPAGS0ND6', - 'extra.device_type': 'HCVR', - 'extra.device_vendor': 'Private', - 'extra.device_version': '3.210.1.4', - 'extra.http_port': 80, - 'extra.internal_port': 37777, - 'extra.ipv4_address': '192.168.0.2', - 'extra.ipv4_dhcp_enable': False, - 'extra.ipv4_gateway': '192.168.0.240', - 'extra.ipv4_subnet_mask': '255.255.255.0', - 'extra.ipv6_address': 'fd09:4ab5:dae9:b078::2', - 'extra.ipv6_gateway': 'fd09:4ab5:dae9:b078::ff', - 'extra.ipv6_link_local': 'fe80::3eef:8cff:fe18:a507/64', - 'extra.mac_address': '3c:ef:8c:18:a5:07', - 'extra.machine_name': 'HCVR', - 'extra.manufacturer': 'Private', - 'extra.method': 'client.notifyDevInfo', - 'extra.remote_video_input_channels': 9, - 'extra.response_size': 761, - 'extra.video_input_channels': 3, - 'extra.video_output_channels': 0, - 'feed.name': 'Accessible DVR DHCPDiscover', - 'protocol.application': 'dvrdhcpdiscover', - 'protocol.transport': 'udp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 37810, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-dvr-dhcpdiscover', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.alarm_input_channels': 0, - 'extra.alarm_output_channels': 0, - 'extra.amplification': 711.0, - 'extra.device_model': 'BCS-XVR0401-IV', - 'extra.device_serial': '5L034FAPAZA0E30', - 'extra.device_type': 'HCVR', - 'extra.device_vendor': 'General', - 'extra.device_version': '4.000.0000002.11', - 'extra.http_port': 80, - 'extra.internal_port': 37777, - 'extra.ipv4_address': '192.168.0.3', - 'extra.ipv4_dhcp_enable': False, - 'extra.ipv4_gateway': '192.168.0.240', - 'extra.ipv4_subnet_mask': '255.255.255.0', - 'extra.ipv6_address': 'fd09:4ab5:dae9:b078::3', - 'extra.ipv6_gateway': 'fd09:4ab5:dae9:b078::ff', - 'extra.ipv6_link_local': 'fe80::3ac4:e8ff:fe02:74da/64', - 'extra.mac_address': '38:c4:e8:02:74:da', - 'extra.machine_name': 'XVR', - 'extra.manufacturer': 'General', - 'extra.method': 'client.notifyDevInfo', - 'extra.remote_video_input_channels': 0, - 'extra.response_size': 711, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.video_input_channels': 4, - 'extra.video_output_channels': 0, - 'feed.name': 'Accessible DVR DHCPDiscover', - 'protocol.application': 'dvrdhcpdiscover', - 'protocol.transport': 'udp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 37810, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_elasticsearch.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_elasticsearch.py deleted file mode 100644 index 4e12a1b076..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_elasticsearch.py +++ /dev/null @@ -1,126 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_elasticsearch.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open Elasticsearch', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_elasticsearch-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-elasticsearch', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.build_hash': '90f439ff60a3c0f497f91663701e64ccd01edbb4', - 'extra.build_snapshot': False, - 'extra.build_timestamp': '2016-07-27T10:36:52Z', - 'extra.cluster_name': 'elasticsearch', - 'extra.lucene_version': '5.5.0', - 'extra.name': 'Red Skull', - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'elasticsearch', - 'extra.tagline': 'You Know, for Search', - 'extra.version': '2.3.5', - 'feed.name': 'Open Elasticsearch', - 'protocol.application': 'elasticsearch', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 9200, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, - {'__type': 'Event', - 'classification.identifier': 'open-elasticsearch', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.build_hash': 'bee86328705acaa9a6daede7140defd4d9ec56bd', - 'extra.build_snapshot': False, - 'extra.cluster_name': 'docker-cluster', - 'extra.lucene_version': '8.11.1', - 'extra.name': 'allinonepod', - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'elasticsearch', - 'extra.tagline': 'You Know, for Search', - 'extra.version': '7.17.0', - 'feed.name': 'Open Elasticsearch', - 'protocol.application': 'elasticsearch', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 9200, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-elasticsearch', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.build_hash': '79d65f6e357953a5b3cbcc5e2c7c21073d89aa29', - 'extra.build_snapshot': False, - 'extra.cluster_name': 'docker-cluster', - 'extra.lucene_version': '8.9.0', - 'extra.name': 'f547c2952610', - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'elasticsearch', - 'extra.tagline': 'You Know, for Search', - 'extra.version': '7.15.0', - 'feed.name': 'Open Elasticsearch', - 'protocol.application': 'elasticsearch', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 9200, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_exchange.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_exchange.py deleted file mode 100644 index aeeffa3c29..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_exchange.py +++ /dev/null @@ -1,149 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), "testdata/scan_exchange.csv")) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = { - "feed.name": "Shadowserver CVE-2021-26855", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2020-08-19T00:00:00+00:00", - "extra.file_name": "2020-08-19-scan_exchange.csv", -} - -EVENTS = [ - { - "__type": "Event", - "feed.name": "Shadowserver CVE-2021-26855", - "time.source": "2021-05-14T00:11:30+00:00", - "source.ip": "12.237.1.2", - "source.port": 443, - "source.asn": 7018, - "source.geolocation.cc": "US", - "source.geolocation.region": "CALIFORNIA", - "source.geolocation.city": "TURLOCK", - "source.reverse_dns": 'afs-exch-cas2.xxx.com', - "extra.version": '15.2.721', - "extra.source.sector": "Communications, Service Provider, and Hosting Service", - "extra.source.naics": 517311, - "classification.identifier": "vulnerable-exchange-server", - "extra.tag": "exchange;cve-2021-26855", - "classification.taxonomy": "vulnerable", - "classification.type": "infected-system", - "extra.servername": "AFS-EXCH2019", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[1]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Shadowserver CVE-2021-26855", - "time.source": "2021-05-14T00:11:37+00:00", - "source.ip": "98.153.3.4", - "source.port": 443, - "source.asn": 20001, - "source.geolocation.cc": "US", - "source.geolocation.region": "CALIFORNIA", - "source.geolocation.city": "LOS ANGELES", - "source.reverse_dns": 'rrcs-98-153-x-x.west.biz.rr.com', - "extra.version": '15.0.847', - "extra.source.sector": "Communications, Service Provider, and Hosting Service", - "extra.source.naics": 517311, - "extra.tag": "exchange;webshell", - "classification.taxonomy": "intrusions", - "classification.type": "system-compromise", - "classification.identifier": "exchange-server-webshell", - "extra.servername": "SSAMAIL", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[2]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Shadowserver CVE-2021-26855", - "time.source": "2021-05-14T00:11:38+00:00", - "source.ip": "206.210.5.6", - "source.port": 443, - "source.asn": 17054, - "source.geolocation.cc": "US", - "source.geolocation.region": "PENNSYLVANIA", - "source.geolocation.city": "PITTSBURGH", - "source.reverse_dns": 'webmail.xxx.com', - "extra.source.naics": 518210, - "extra.version": '15.0.1178', - "extra.servername": "OMNYXEXCH02", - "extra.tag": "exchange;webshell", - "classification.taxonomy": "intrusions", - "classification.type": "system-compromise", - "classification.identifier": "exchange-server-webshell", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[3]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Shadowserver CVE-2021-26855", - "time.source": "2021-05-14T00:11:38+00:00", - "source.ip": "12.33.7.8", - "source.port": 443, - "source.asn": 7018, - "source.geolocation.cc": "US", - "source.geolocation.region": "ARKANSAS", - "source.geolocation.city": "LITTLE ROCK", - "source.reverse_dns": 'mail.xxx.org', - "extra.version": '15.1.2176', - "extra.source.sector": "Communications, Service Provider, and Hosting Service", - "extra.source.naics": 921120, - "extra.servername": "MHASVR02", - "classification.identifier": "vulnerable-exchange-server", - "extra.tag": "exchange;cve-2021-26855", - "classification.taxonomy": "vulnerable", - "classification.type": "infected-system", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[4]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Shadowserver CVE-2021-26855", - "time.source": "2021-05-14T00:11:38+00:00", - "source.ip": "41.204.9.10", - "source.port": 443, - "source.asn": 21042, - "source.geolocation.cc": 'MG', - "source.geolocation.city": 'ANTANANARIVO', - "source.geolocation.region": 'ANTANANARIVO', - "source.reverse_dns": 'mail.xxx.mg', - "extra.servername": "SABMHQE0232", - "classification.identifier": "vulnerable-exchange-server", - "extra.tag": "exchange;cve-2021-26855", - "classification.taxonomy": "vulnerable", - "classification.type": "infected-system", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[5]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, -] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == "__main__": - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ftp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ftp.py deleted file mode 100644 index 33daefd75e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ftp.py +++ /dev/null @@ -1,120 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ftp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible FTP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2019-03-25T00:00:00+00:00", - "extra.file_name": "2019-03-25-scan_ftp-test-test.csv", - } -EVENTS = [{ - '__type': 'Event', - 'feed.name': 'Accessible FTP', - 'time.observation': '2019-03-25T00:00:00+00:00', - 'time.source': '2019-03-06T06:37:00+00:00', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-ftp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.ip': '61.126.3.70', - 'source.port': 21, - 'protocol.transport': 'tcp', - 'protocol.application': 'ftp', - 'source.reverse_dns': 'arcus-net.co.jp', - 'extra.tag': 'ftp', - 'source.asn': 4713, - 'source.geolocation.cc': 'JP', - 'source.geolocation.region': 'TOKYO', - 'source.geolocation.city': 'TOKYO', - 'extra.naics': 517311, - 'extra.sic': 737401, - 'extra.banner': '220 FTP Server ready.|', - 'extra.handshake': 'TLSv1.2', - 'extra.cipher_suite': 'TLS_RSA_WITH_AES_128_CBC_SHA', - 'extra.cert_length': 2048, - 'extra.subject_common_name': '*.bizmw.com', - 'extra.issuer_common_name': 'GlobalSign Organization Validation CA - SHA256 - G2', - 'extra.cert_issue_date': 'Jan 14 08:04:50 2015 GMT', - 'extra.cert_expiration_date': 'Jan 14 08:04:50 2020 GMT', - 'extra.sha1_fingerprint': 'D9:98:3F:2E:F9:D1:BE:9A:10:1E:DE:51:2C:C1:DF:01:18:0A:20:65', - 'extra.cert_serial_number': '1121DC7421AB7924C3B1D396AEA3707E9E29', - 'extra.ssl_version': 2, - 'extra.signature_algorithm': 'sha256WithRSAEncryption', - 'extra.key_algorithm': 'rsaEncryption', - 'extra.subject_organization_name': 'NTT Communications Corporation', - 'extra.subject_country': 'JP', - 'extra.subject_state_or_province_name': 'Tokyo', - 'extra.subject_locality_name': 'Minato-ku', - 'extra.issuer_organization_name': 'GlobalSign nv-sa', - 'extra.issuer_country': 'BE', - 'extra.sha256_fingerprint': '27:4A:8A:3A:A7:DF:82:D0:43:03:0E:6F:48:30:30:C9:24:77:11:1A:08:EF:F7:B9:74:0C:CE:40:87:03:D2:51', - 'extra.sha512_fingerprint': 'E5:93:8B:72:84:0F:35:52:8E:7A:6C:E3:EF:36:90:4C:F2:86:A7:4D:B2:DD:C0:C6:23:83:18:EF:DD:86:34:92:91:57:22:29:75:45:71:8B:3A:CD:F1:27:A9:CA:5F:70:5E:AC:15:A5:E6:63:FD:6F:BB:C5:E2:45:99:73:E9:E6', - 'extra.md5_fingerprint': 'D1:A7:BC:96:78:1D:16:D0:24:A8:62:7C:3A:95:5A:4A', - 'extra.cert_valid': False, - 'extra.self_signed': False, - 'extra.cert_expired': False, - 'extra.validation_level': 'OV', - 'extra.auth_tls_response': '234 AUTH TLS successful', - }, - { - '__type': 'Event', - 'feed.name': 'Accessible FTP', - 'time.observation': '2019-03-25T00:00:00+00:00', - 'time.source': '2019-03-06T06:37:00+00:00', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-ftp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.ip': '62.48.156.65', - 'source.port': 21, - 'protocol.transport': 'tcp', - 'protocol.application': 'ftp', - 'source.reverse_dns': 'dial-62-48-156-65.ptprime.net', - 'extra.tag': 'ftp', - 'source.asn': 15525, - 'source.geolocation.cc': 'PT', - 'source.geolocation.region': 'LISBOA', - 'source.geolocation.city': 'FRIELAS', - 'extra.banner': '220-================================================================| PT Empresas| Acesso Reservado| Acesso nao autorizado punido por lei: 109/91; 67/98| ----------------------------------------------------------------| HENNES & MAURITZ LDA - 149093| SITE: PT303 - Cascais Shopping| MORADA: | NIR: EWS1822940| ================================================================|220 FTP server ready, 1 active clients of 4 simultaneous clients allowed.|', - 'extra.auth_tls_response': '500 Syntax error, command unrecognized.', - 'extra.auth_ssl_response': '500 Syntax error, command unrecognized.' - } - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_hadoop.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_hadoop.py deleted file mode 100644 index 0b5794cb7b..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_hadoop.py +++ /dev/null @@ -1,94 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Sebastian Wagner -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_hadoop.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "ShadowServer Accessible-Hadoop", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_hadoop-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'ShadowServer Accessible-Hadoop', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-hadoop', - 'extra.version': '2.7.3, rbaa91f7c6bc9cb92be5982de4719c1c8af91ccff', - 'extra.server_type': 'namenode', - 'extra.clusterid': 'CID-64471a53-60cb-4302-9832-92f321f111fe', - 'extra.total_disk': 41567956992, - 'extra.used_disk': 53248, - 'extra.free_disk': 25160089600, - 'extra.livenodes': 'edmonton:50010', - 'protocol.application': 'hadoop', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 15296, - 'source.geolocation.cc': 'CA', - 'source.geolocation.city': 'CALGARY', - 'source.geolocation.region': 'ALBERTA', - 'source.ip': '199.116.235.200', - 'source.port': 50070, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2017-09-13T02:06:05+00:00'}, - {'__type': 'Event', - 'feed.name': 'ShadowServer Accessible-Hadoop', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-hadoop', - 'extra.version': '2.7.1.2.4.0.0-169', - 'extra.naics': 334111, - 'extra.sic': 357101, - 'extra.server_type': 'datanode', - 'extra.clusterid': 'CID-771bae52-9e4f-4ec4-bc1a-c867585751f0', - 'extra.namenodeaddress': 'sandbox.hortonworks.com', - 'extra.volumeinfo': '/hadoop/hdfs/data/current', - 'protocol.application': 'hadoop', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn': 8075, - 'source.geolocation.cc': 'US', - 'source.geolocation.city': 'DES MOINES', - 'source.geolocation.region': 'IOWA', - 'source.ip': '104.43.235.92', - 'source.port': 50075, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2017-09-13T02:07:48+00:00'}, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_http.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_http.py deleted file mode 100644 index 793a95f221..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_http.py +++ /dev/null @@ -1,100 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_http.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible HTTP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-03-25-scan_http-test-test.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Accessible HTTP', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'classification.identifier': 'accessible-http', - 'extra.source.naics': 518111, - 'extra.source.sic': 737401, - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 200, - 'extra.http_reason': 'OK', - 'extra.content_type': 'text/html', - 'extra.server': 'lighttpd', - 'extra.transfer_encoding': 'chunked', - 'extra.http_date': '2018-04-19T00:02:28+00:00', - 'extra.tag': 'http', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.reverse_dns': 'c-75-74-78-113.hsd1.fl.comcast.net', - 'source.asn': 7922, - 'source.geolocation.cc': 'US', - 'source.geolocation.city': 'MIAMI', - 'source.geolocation.region': 'FLORIDA', - 'source.ip': '75.74.78.113', - 'source.port': 8080, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2018-04-19T00:02:26+00:00'}, - {'__type': 'Event', - 'feed.name': 'Accessible HTTP', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'classification.identifier': 'accessible-http', - 'extra.source.naics': 518210, - 'extra.source.sic': 737415, - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 200, - 'extra.http_reason': 'OK', - 'extra.content_type': 'text/html', - 'extra.content_length': 17729, - 'extra.http_date': '2018-04-19T02:02:28+00:00', - 'extra.tag': 'http', - 'protocol.transport': 'tcp', - 'protocol.application': 'http', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.reverse_dns': 'sto95-3-88-162-174-130.fbx.proxad.net', - 'source.asn': 12322, - 'source.geolocation.cc': 'FR', - 'source.geolocation.city': 'SAINT-OUEN-LAUMONE', - 'source.ip': '88.162.174.130', - 'source.port': 8080, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2018-04-19T00:02:26+00:00'}, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_http_proxy.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_http_proxy.py deleted file mode 100644 index dc5e94e5ec..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_http_proxy.py +++ /dev/null @@ -1,118 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_http_proxy.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open HTTP Proxy', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_http_proxy-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-http-proxy', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 200, - 'extra.http_date': '2010-02-10T00:00:00+00:00', - 'extra.http_reason': 'Connection established', - 'extra.tag': 'http-connect-proxy', - 'feed.name': 'Open HTTP Proxy', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 3128, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-http-proxy', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 200, - 'extra.http_date': '2010-02-10T00:00:01+00:00', - 'extra.http_reason': 'Connection established', - 'extra.tag': 'http-connect-proxy', - 'extra.via': 'HTTP/1.1 s_proxy_den1', - 'feed.name': 'Open HTTP Proxy', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 3128, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-http-proxy', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 200, - 'extra.http_date': '2010-02-10T00:00:02+00:00', - 'extra.http_reason': 'Connection established', - 'extra.tag': 'http-connect-proxy', - 'extra.via': 'HTTP/1.1 s_proxy_yvr', - 'feed.name': 'Open HTTP Proxy', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 3128, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_http_vulnerable.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_http_vulnerable.py deleted file mode 100644 index d15232eaf7..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_http_vulnerable.py +++ /dev/null @@ -1,125 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Mikk Margus Möll -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_http_vulnerable.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Vulnerable HTTP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2021-08-01T09:00:00+00:00", - "extra.file_name": "2021-08-01-scan_http_vulnerable-test-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'accessible-http', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.content_length': 149, - 'extra.content_type': 'text/html; charset=utf-8', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 401, - 'extra.http_date': '2010-02-10T00:00:00+00:00', - 'extra.http_reason': 'Unauthorized', - 'extra.server': 'TwistedWeb/19.7.0', - 'extra.set_cookie': 'TWISTED_SESSION=5473ad3faa3de66685fb3a53bffb390b4fcec2039893009a06caf38e1bec8aa8', - 'extra.tag': 'basic-auth,http', - 'extra.www_authenticate': 'Basic realm=\\\\"OpenWebif\\"\\""', - 'feed.name': 'Vulnerable HTTP', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 8080, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-http', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.content_length': 149, - 'extra.content_type': 'text/html; charset=utf-8', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 401, - 'extra.http_date': '2010-02-10T00:00:01+00:00', - 'extra.http_reason': 'Unauthorized', - 'extra.server': 'TwistedWeb/19.7.0', - 'extra.set_cookie': 'TWISTED_SESSION=d2460d37b7fdbdd6c27dd74423ead5704e553d4f2c230672313edc5602059e33', - 'extra.tag': 'basic-auth,http', - 'extra.www_authenticate': 'Basic realm=\\\\"OpenWebif\\"\\""', - 'feed.name': 'Vulnerable HTTP', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 80, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-http', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.detail': 'repositoryformatversion = 0;filemode = false;bare = ' - 'false;logallrefupdates = true;symlinks = false;ignorecase = ' - 'true', - 'extra.http_date': '2010-02-10T00:00:02+00:00', - 'extra.tag': 'git-config-file', - 'feed.name': 'Vulnerable HTTP', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 443, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ics.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ics.py deleted file mode 100644 index f673f40c80..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ics.py +++ /dev/null @@ -1,125 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ics.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Acessible ICS', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_ics-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ics', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.device_model' : 'device_model', - 'extra.device_type' : 'device_type', - 'extra.device_vendor' : 'Vendor 1', - 'extra.device_version' : 'device_version', - 'extra.raw_response' : 'dGVzdDE=', - 'extra.response_size' : 5, - 'extra.source.sector' : 'Sector', - 'feed.name' : 'Acessible ICS', - 'protocol.application' : 'modbus', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'CITY', - 'source.geolocation.region' : 'REGION', - 'source.ip' : '192.168.0.1', - 'source.port' : 502, - 'source.reverse_dns' : 'host1.example.net', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-03-02T00:34:22+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ics', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.device_model' : 'device_model', - 'extra.device_type' : 'device_type', - 'extra.device_vendor' : 'Vendor 2', - 'extra.device_version' : 'device_version', - 'extra.raw_response' : 'dGVzdDI=', - 'extra.response_size' : 5, - 'extra.source.sector' : 'Sector', - 'feed.name' : 'Acessible ICS', - 'protocol.application' : 'modbus', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64513, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'CITY', - 'source.geolocation.region' : 'REGION', - 'source.ip' : '192.168.0.2', - 'source.port' : 502, - 'source.reverse_dns' : 'host2.example.net', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-03-02T00:34:22+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ics', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.device_model' : 'device_model', - 'extra.device_type' : 'device_type', - 'extra.device_vendor' : 'Vendor 3', - 'extra.device_version' : 'device_version', - 'extra.raw_response' : 'dGVzdDM=', - 'extra.response_size' : 5, - 'extra.source.sector' : 'Sector', - 'feed.name' : 'Acessible ICS', - 'protocol.application' : 'modbus', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64514, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'CITY', - 'source.geolocation.region' : 'REGION', - 'source.ip' : '192.168.0.3', - 'source.port' : 502, - 'source.reverse_dns' : 'host3.example.net', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-03-02T00:34:22+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ipmi.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ipmi.py deleted file mode 100644 index 08a9082af9..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ipmi.py +++ /dev/null @@ -1,106 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_ipmi.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open IPMI', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ipmi-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Open IPMI', - "classification.identifier": "open-ipmi", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.anon_login": False, - "extra.defaultkg": "-", - "extra.ipmi_version": "1.5", - "extra.md2_auth": False, - "extra.md5_auth": True, - "extra.none_auth": True, - "extra.nulluser": True, - "extra.oem_auth": False, - "extra.passkey_auth": True, - "extra.permessage_auth": True, - "extra.tag": "ipmi", - "extra.userlevel_auth": True, - "extra.usernames": False, - "protocol.application": "ipmi", - "protocol.transport": "udp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 2914, - "source.geolocation.cc": "DE", - "source.geolocation.city": "BERLIN", - "source.geolocation.region": "BERLIN", - "source.ip": "198.51.100.4", - "source.port": 623, - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2016-07-24T00:09:42+00:00" - }, - {'__type': 'Event', - 'feed.name': 'Open IPMI', - "classification.identifier": "open-ipmi", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.anon_login": False, - "extra.defaultkg": "default", - "extra.ipmi_version": "2.0", - "extra.md2_auth": False, - "extra.md5_auth": False, - "extra.none_auth": False, - "extra.nulluser": False, - "extra.oem_auth": False, - "extra.passkey_auth": False, - "extra.permessage_auth": False, - "extra.tag": "ipmi", - "extra.userlevel_auth": True, - "extra.usernames": True, - "protocol.application": "ipmi", - "protocol.transport": "udp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 28753, - "source.geolocation.cc": "DE", - "source.geolocation.city": "FRANKFURT AM MAIN", - "source.geolocation.region": "HESSEN", - "source.ip": "198.51.100.182", - "source.port": 623, - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2016-07-24T00:09:43+00:00" - }, - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ipp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ipp.py deleted file mode 100644 index 9adc8485e0..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ipp.py +++ /dev/null @@ -1,79 +0,0 @@ -# SPDX-FileCopyrightText: 2020 Thomas Hungenberg -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_ipp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open-IPP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2020-06-09T00:00:00+00:00", - "extra.file_name": "2020-06-08-scan_ipp-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Open-IPP', - "classification.identifier": "open-ipp", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517311, - "extra.tag": "ipp", - "extra.ipp_version": "IPP/2.1", - "extra.cups_version": "CUPS/2.0", - "extra.printer_uris": "ipp://123.45.67.89:631/ipp/print", - "extra.printer_name": "NPI3F0D22", - "extra.printer_info": "HP Color LaserJet MFP M277dw", - "extra.printer_more_info": "http://123.45.67.89:631/hp/device/info_config_AirPrint.html?tab=Networking&menu=AirPrintStatus", - "extra.printer_make_and_model": "HP Color LaserJet MFP M277dw", - "extra.printer_firmware_name": "20191203", - "extra.printer_firmware_string_version": "20191203", - "extra.printer_firmware_version": "20191203", - "extra.printer_organization": "org", - "extra.printer_organization_unit": "unit", - "extra.printer_uuid": "urn:uuid:456e4238-4a44-4643-4c42-10e1813f0a18", - "extra.printer_wifi_ssid": "wifissid", - "protocol.application": "ipp", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 12345, - "source.geolocation.cc": "AA", - "source.geolocation.city": "CITY", - "source.geolocation.region": "REGION", - "source.ip": "123.45.67.89", - "source.port": 631, - 'source.reverse_dns': 'some.host.com', - "time.observation": "2020-06-09T00:00:00+00:00", - "time.source": "2020-06-08T11:30:14+00:00" - }, - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_isakmp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_isakmp.py deleted file mode 100644 index 3192f508f8..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_isakmp.py +++ /dev/null @@ -1,105 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_isakmp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Vulnerable ISAKMP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_isakmp-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Vulnerable ISAKMP', - "classification.identifier": "open-ike", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.domain_of_interpretation": '00', - "extra.exchange_type": '05', - "extra.flags": '00', - "extra.initiator_spi": "3e35c70729dfedef", - "extra.message_id": "00000000", - "extra.naics": 517311, - "extra.next_payload": '11', - "extra.next_payload2": '00', - "extra.notify_message_type": '14', - "extra.responder_spi": "253acab7cbfda607", - "extra.spi_size": 0, - "extra.tag": "isakmp-vulnerable", - "protocol.application": "ipsec", - "protocol.transport": "udp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 5678, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.42", - "source.port": 500, - "source.reverse_dns": "example.local", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T00:17:25+00:00" - }, - {'__type': 'Event', - 'feed.name': 'Vulnerable ISAKMP', - "classification.identifier": "open-ike", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.domain_of_interpretation": '00', - "extra.exchange_type": '05', - "extra.flags": '00', - "extra.initiator_spi": "3e35c70729dfedef", - "extra.message_id": "00000000", - "extra.next_payload": '11', - "extra.next_payload2": '00', - "extra.notify_message_type": '14', - "extra.responder_spi": "b274460e7adc1bf0", - "extra.spi_size": 0, - "extra.tag": "isakmp-vulnerable", - "protocol.application": "ipsec", - "protocol.transport": "udp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 20255, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.67", - "source.port": 500, - "source.reverse_dns": "example.local", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T00:17:28+00:00" - }, - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_kubernetes.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_kubernetes.py deleted file mode 100644 index 2bac336a79..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_kubernetes.py +++ /dev/null @@ -1,214 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_kubernetes.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible Kubernetes API Server', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_kubernetes-test.csv", - } -EVENTS = [ - { - '__type' : 'Event', - 'classification.identifier' : 'open-kubernetes', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.browser_error' : 'x509: failed to load system roots and no roots provided', - 'extra.browser_trusted' : False, - 'extra.build_date' : '2021-11-17T13:00:29Z', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.compiler' : 'gc', - 'extra.content_type' : 'application/json', - 'extra.date' : 'Tue, 10 May 2022 14:24:13 GMT', - 'extra.git_commit' : '2444b3347a2c45eb965b182fb836e1f51dc61b70', - 'extra.git_tree_state' : 'clean', - 'extra.git_version' : 'v1.20.13', - 'extra.go_version' : 'go1.15.15', - 'extra.handshake' : 'TLSv1.2', - 'extra.http' : 'HTTP/1.1', - 'extra.http_code' : 200, - 'extra.http_reason' : 'OK', - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.major' : '1', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.minor' : '20', - 'extra.platform' : 'linux/amd64', - 'extra.self_signed' : False, - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.tag' : 'kubernetes', - 'feed.name' : 'Accessible Kubernetes API Server', - 'protocol.application' : 'kubernetes', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 6443, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-kubernetes', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.browser_error' : 'x509: failed to load system roots and no roots provided', - 'extra.browser_trusted' : False, - 'extra.build_date' : '2022-02-25T06:26:46Z', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.compiler' : 'gc', - 'extra.content_type' : 'application/json', - 'extra.date' : 'Tue, 10 May 2022 14:24:12 GMT', - 'extra.git_commit' : '6f5a5295923a614a4202a7ad274b38b69f9ca8c0', - 'extra.git_tree_state' : 'clean', - 'extra.git_version' : 'v1.23.3+e419edf', - 'extra.go_version' : 'go1.17.5', - 'extra.handshake' : 'TLSv1.2', - 'extra.http' : 'HTTP/1.1', - 'extra.http_code' : 200, - 'extra.http_reason' : 'OK', - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.major' : '1', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.minor' : '23', - 'extra.platform' : 'linux/amd64', - 'extra.self_signed' : False, - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.source.sector' : 'Retail Trade', - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.tag' : 'kubernetes', - 'feed.name' : 'Accessible Kubernetes API Server', - 'protocol.application' : 'kubernetes', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 6443, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-kubernetes', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.browser_error' : 'x509: failed to load system roots and no roots provided', - 'extra.browser_trusted' : False, - 'extra.build_date' : '2020-05-08T07:29:59Z', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.compiler' : 'gc', - 'extra.content_type' : 'application/json', - 'extra.date' : 'Tue, 10 May 2022 14:24:12 GMT', - 'extra.git_commit' : '4f7ea78', - 'extra.git_version' : 'v1.16.9-aliyun.1', - 'extra.go_version' : 'go1.13.9', - 'extra.handshake' : 'TLSv1.2', - 'extra.http' : 'HTTP/1.1', - 'extra.http_code' : 200, - 'extra.http_reason' : 'OK', - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.major' : '1', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.minor' : '16+', - 'extra.platform' : 'linux/amd64', - 'extra.self_signed' : False, - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.tag' : 'kubernetes', - 'feed.name' : 'Accessible Kubernetes API Server', - 'protocol.application' : 'kubernetes', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 6443, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_tcp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_tcp.py deleted file mode 100644 index b6abf6eba9..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_tcp.py +++ /dev/null @@ -1,154 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_ldap_tcp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open LDAP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ldap_tcp-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.configuration_naming_context': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.default_naming_context': 'DC=ad,DC=example,DC=com', - 'extra.domain_controller_functionality': 7, - 'extra.ds_service_name': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.forest_functionality': 2, - 'extra.ldap_service_name': 'node01.example.com', - 'extra.naming_contexts': 'DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.root_domain_naming_context': 'DC=example,DC=com', - 'extra.schema_naming_context': 'CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.server_name': 'CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com', - 'extra.size': 0, - 'extra.subschema_subentry': 'CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.supported_control': '1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|', - 'extra.supported_ldap_policies': 'MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent', - 'extra.supported_ldap_version': '3|2', - 'extra.supported_sasl_mechanisms': 'GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5', - 'extra.tag': 'ldap-tcp', - 'feed.name': 'Open LDAP', - 'protocol.application': 'ldap', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.local_hostname': 'node01.example.com', - 'source.port': 389, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, - {'__type': 'Event', - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.configuration_naming_context': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.current_time': '20220821124435.0Z', - 'extra.default_naming_context': 'DC=ad,DC=example,DC=com', - 'extra.domain_controller_functionality': 7, - 'extra.domain_functionality': 7, - 'extra.ds_service_name': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.forest_functionality': 7, - 'extra.highest_committed_usn': 25029662, - 'extra.is_global_catalog_ready': True, - 'extra.is_synchronized': True, - 'extra.ldap_service_name': 'node02.example.com', - 'extra.naming_contexts': 'DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.root_domain_naming_context': 'DC=example,DC=com', - 'extra.schema_naming_context': 'CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.server_name': 'CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com', - 'extra.size': 0, - 'extra.subschema_subentry': 'CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.supported_capabilities': '1.2.840.113556.1.4.800|1.2.840.113556.1.4.1670|1.2.840.113556.1.4.1791|1.2.840.113556.1.4.1935|1.2.840.113556.1.4.2080|1.2.840.113556.1.4.2237', - 'extra.supported_control': '1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|1.2.840.113556.1.4.2206|1.2.840.113556.1.4.2211|1.2.840.113556.1.4.2239|1.2.840.113556.1.4.2255|1.2.840.113556.1.4.2256|1.2.840.113556.1.4.2309', - 'extra.supported_ldap_policies': 'MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent', - 'extra.supported_ldap_version': '3|2', - 'extra.supported_sasl_mechanisms': 'GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5', - 'extra.tag': 'ldap-tcp', - 'feed.name': 'Open LDAP', - 'protocol.application': 'ldap', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.local_hostname': 'node02.example.com', - 'source.port': 389, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.configuration_naming_context': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.current_time': '20220821124539.0Z', - 'extra.default_naming_context': 'DC=ad,DC=example,DC=com', - 'extra.ds_service_name': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.ldap_service_name': 'node03.example.com', - 'extra.naming_contexts': 'DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.root_domain_naming_context': 'DC=example,DC=com', - 'extra.schema_naming_context': 'CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.server_name': 'CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com', - 'extra.size': 0, - 'extra.subschema_subentry': 'CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.supported_control': '1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|', - 'extra.tag': 'ldap-tcp', - 'feed.name': 'Open LDAP', - 'protocol.application': 'ldap', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.local_hostname': 'node03.example.com', - 'source.port': 389, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_udp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_udp.py deleted file mode 100644 index aa4deefb87..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_udp.py +++ /dev/null @@ -1,162 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_ldap_udp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open LDAP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ldap_udp-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 58.42, - 'extra.configuration_naming_context': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.current_time': '20220821044533.0Z', - 'extra.default_naming_context': 'DC=ad,DC=example,DC=com', - 'extra.domain_controller_functionality': 7, - 'extra.domain_functionality': 7, - 'extra.ds_service_name': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.forest_functionality': 7, - 'extra.highest_committed_usn': 222537, - 'extra.is_global_catalog_ready': True, - 'extra.is_synchronized': True, - 'extra.ldap_service_name': 'node01.example.com', - 'extra.naming_contexts': 'DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.root_domain_naming_context': 'DC=example,DC=com', - 'extra.schema_naming_context': 'CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.server_name': 'CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com', - 'extra.size': 3038, - 'extra.subschema_subentry': 'CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.supported_capabilities': '1.2.840.113556.1.4.800|1.2.840.113556.1.4.1670|1.2.840.113556.1.4.1791|1.2.840.113556.1.4.1935|1.2.840.113556.1.4.2080|1.2.840.113556.1.4.2237', - 'extra.supported_control': '1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|1.2.840.113556.1.4.2206|1.2.840.113556.1.4.2211|1.2.840.113556.1.4.2239|1.2.840.113556.1.4.2255|1.2.840.113556.1.4.2256|1.2.840.113556.1.4.2309|1.2.840.113556.1.4.2330|1.2.840.113556.1.4.2354', - 'extra.supported_ldap_policies': 'MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent', - 'extra.supported_ldap_version': '3|2', - 'extra.supported_sasl_mechanisms': 'GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5', - 'extra.tag': 'ldap-udp', - 'feed.name': 'Open LDAP', - 'protocol.application': 'ldap', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.local_hostname': 'node01.example.com', - 'source.port': 389, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 58.88, - 'extra.configuration_naming_context': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.current_time': '20220821044948.0Z', - 'extra.default_naming_context': 'DC=ad,DC=example,DC=com', - 'extra.domain_controller_functionality': 7, - 'extra.domain_functionality': 7, - 'extra.ds_service_name': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.forest_functionality': 7, - 'extra.highest_committed_usn': 1478714, - 'extra.is_global_catalog_ready': True, - 'extra.is_synchronized': True, - 'extra.ldap_service_name': 'node02.example.com', - 'extra.naming_contexts': 'DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.root_domain_naming_context': 'DC=example,DC=com', - 'extra.schema_naming_context': 'CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.server_name': 'CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com', - 'extra.size': 3062, - 'extra.subschema_subentry': 'CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.supported_capabilities': '1.2.840.113556.1.4.800|1.2.840.113556.1.4.1670|1.2.840.113556.1.4.1791|1.2.840.113556.1.4.1935|1.2.840.113556.1.4.2080|1.2.840.113556.1.4.2237', - 'extra.supported_control': '1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|1.2.840.113556.1.4.2206|1.2.840.113556.1.4.2211|1.2.840.113556.1.4.2239|1.2.840.113556.1.4.2255|1.2.840.113556.1.4.2256|1.2.840.113556.1.4.2309|1.2.840.113556.1.4.2330|1.2.840.113556.1.4.2354', - 'extra.supported_ldap_policies': 'MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent', - 'extra.supported_ldap_version': '3|2', - 'extra.supported_sasl_mechanisms': 'GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5', - 'extra.tag': 'ldap-udp', - 'feed.name': 'Open LDAP', - 'protocol.application': 'ldap', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.local_hostname': 'node02.example.com', - 'source.port': 389, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 0.69, - 'extra.configuration_naming_context': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.default_naming_context': 'DC=ad,DC=example,DC=com', - 'extra.ds_service_name': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.ldap_service_name': 'node03.example.com', - 'extra.naming_contexts': 'DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.root_domain_naming_context': 'DC=example,DC=com', - 'extra.schema_naming_context': 'CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.server_name': 'CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com', - 'extra.size': 36, - 'extra.subschema_subentry': 'CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.tag': 'ldap-udp', - 'feed.name': 'Open LDAP', - 'protocol.application': 'ldap', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.local_hostname': 'node03.example.com', - 'source.port': 389, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_mdns.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_mdns.py deleted file mode 100644 index 9207aaf365..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_mdns.py +++ /dev/null @@ -1,127 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_mdns.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open mDNS', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_mdns-test-geo.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mdns', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.http_ipv4' : '192.168.0.1', - 'extra.http_ipv6' : 'fd09:4ab5:dae9:b078::1', - 'extra.services' : '_smb._tcp.local.; _device-info._tcp.local.; _http._tcp.local.; _dacp._tcp.local.;', - 'extra.tag' : 'mdns', - 'extra.workstation_ipv4' : '192.168.0.1', - 'extra.workstation_ipv6' : 'fd09:4ab5:dae9:b078::1', - 'feed.name' : 'Open mDNS', - 'protocol.application' : 'mdns', - 'protocol.transport' : 'udp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 5353, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mdns', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.http_ipv4' : '192.168.0.2', - 'extra.http_ipv6' : 'fd09:4ab5:dae9:b078::2', - 'extra.services' : '_home-assistant._tcp.local.;', - 'extra.tag' : 'mdns', - 'extra.workstation_ipv4' : '192.168.0.2', - 'extra.workstation_ipv6' : 'fd09:4ab5:dae9:b078::2', - 'feed.name' : 'Open mDNS', - 'protocol.application' : 'mdns', - 'protocol.transport' : 'udp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 5353, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mdns', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.http_info' : '\\\\\"vendor=Synology\\\"\\\" \\\"\\\"model=DS218+\\\"\\\" \\\"\\\"serial=17A0PCN482002\\\"\\\" \\\"\\\"version_major=6\\\"\\\" \\\"\\\"version_minor=2\\\"\\\" \\\"\\\"version_build=25556\\\"\\\" \\\"\\\"admin_port=5000\\\"\\\" \\\"\\\"secure_admin_port=5001\\\"\\\" \\\"\\\"mac_address=00:11:32:80:fd:b5\\\"\\\"\"', - 'extra.http_ipv4' : '192.168.0.3', - 'extra.http_ipv6' : 'fd09:4ab5:dae9:b078::3', - 'extra.http_name' : 'snmeijer.local.', - 'extra.http_port' : 5000, - 'extra.http_ptr' : 'snmeijer._http._tcp.local.', - 'extra.http_target' : 'snmeijer.local.', - 'extra.services' : '_webdav._tcp.local.; _adisk._tcp.local.; _smb._tcp.local.; _http._tcp.local.; _dacp._tcp.local.; _afpovertcp._tcp.local.; _device-info._tcp.local.;', - 'extra.tag' : 'mdns,iot', - 'extra.workstation_ipv4' : '192.168.0.3', - 'extra.workstation_ipv6' : 'fd09:4ab5:dae9:b078::3', - 'feed.name' : 'Open mDNS', - 'protocol.application' : 'mdns', - 'protocol.transport' : 'udp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 5353, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_memcached.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_memcached.py deleted file mode 100644 index b54fc0ea53..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_memcached.py +++ /dev/null @@ -1,130 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_memcached.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open Memcached', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_memcached-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-memcached', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 81.71, - 'extra.curr_connections': 243, - 'extra.pid': 1010, - 'extra.pointer_size': 64, - 'extra.response_size': 1144, - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'memcached', - 'extra.time': '2022-08-21 10:34:06', - 'extra.total_connections': 6106, - 'extra.uptime': 32908114, - 'extra.version': '1.4.15', - 'feed.name': 'Open Memcached', - 'protocol.application': 'memcached', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 50260, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-memcached', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 75.21, - 'extra.curr_connections': 9, - 'extra.pid': 5316, - 'extra.pointer_size': 64, - 'extra.response_size': 1053, - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'memcached', - 'extra.time': '2022-08-21 10:39:21', - 'extra.total_connections': 2962, - 'extra.uptime': 9618498, - 'extra.version': '1.4.13', - 'feed.name': 'Open Memcached', - 'protocol.application': 'memcached', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 11211, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, - {'__type': 'Event', - 'classification.identifier': 'open-memcached', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 31.57, - 'extra.curr_connections': 2, - 'extra.pid': 1460, - 'extra.pointer_size': 32, - 'extra.response_size': 442, - 'extra.tag': 'memcached', - 'extra.time': '2022-08-21 10:39:39', - 'extra.total_connections': 534, - 'extra.uptime': 1375159, - 'extra.version': '1.2.6', - 'feed.name': 'Open Memcached', - 'protocol.application': 'memcached', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 11211, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_mongodb.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_mongodb.py deleted file mode 100644 index 3ecf7b21f9..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_mongodb.py +++ /dev/null @@ -1,103 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_mongodb.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open MongoDB', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_mongodb-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Open MongoDB', - "classification.identifier": "open-mongodb", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.allocator": "tcmalloc", - "extra.bits": "64", - "extra.gitversion": "a2ddc68ba7c9cee17bfe69ed840383ec3506602b", - "extra.javascriptengine": "V8", - "extra.maxbsonobjectsize": "16777216", - "extra.ok": True, - "extra.sysinfo": "Linux ip-198-51-100-100 198.51.100.103-2.ec2.v1.2.fc8xen #1 SMP Fri Nov 20 17:48:28 EST 2009 x86_64 BOOST_LIB_VERSION=1_49", - "extra.tag": "mongodb", - "extra.version": "2.4.5", - "extra.visible_databases": "local | countly | admin", - "protocol.application": "mongodb", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 20773, - "source.geolocation.cc": "DE", - "source.geolocation.city": "WEEZE", - "source.geolocation.region": "NORDRHEIN-WESTFALEN", - "source.ip": "198.51.100.203", - "source.port": 27017, - "source.reverse_dns": "198-51-100-203.example.net", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2016-07-24T00:40:07+00:00" - }, - {'__type': 'Event', - 'feed.name': 'Open MongoDB', - "classification.identifier": "open-mongodb", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.allocator": "tcmalloc", - "extra.bits": "64", - "extra.gitversion": "d73c92b1c85703828b55c2916a5dd4ad46535f6a", - "extra.javascriptengine": "V8", - "extra.maxbsonobjectsize": "16777216", - "extra.ok": True, - "extra.sector": "Information Technology", - "extra.sysinfo": "Linux build5.ny.cbi.10gen.cc 2.6.32-431.3.1.el6.x86_64 #1 SMP Fri Jan 3 21:39:27 UTC 2014 x86_64 BOOST_LIB_VERSION=1_49", - "extra.tag": "mongodb", - "extra.version": "2.6.12", - "extra.visible_databases": "none visible", - "protocol.application": "mongodb", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 24940, - "source.geolocation.cc": "DE", - "source.geolocation.city": "GUNZENHAUSEN", - "source.geolocation.region": "BAYERN", - "source.ip": "198.51.100.42", - "source.port": 27017, - "source.reverse_dns": "198-51-100-208.example.net", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2016-07-24T00:40:07+00:00" - }, - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt.py deleted file mode 100644 index 45d19f9eea..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt.py +++ /dev/null @@ -1,89 +0,0 @@ -# SPDX-FileCopyrightText: 2020 Thomas Hungenberg -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_mqtt.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open-MQTT', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2020-03-15T00:00:00+00:00", - "extra.file_name": "2020-03-14-scan_mqtt-test-geo.csv", - } -EVENTS = [ - { - '__type' : 'Event', - 'classification.identifier' : 'open-mqtt', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.anonymous_access' : False, - 'extra.cert_expiration_date' : '2022-11-14 00:00:00', - 'extra.cert_issue_date' : '2020-08-12 00:00:00', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : '085699743A23114C9B6B8DC975A8AF42', - 'extra.cipher_suite' : 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', - 'extra.code' : 'Connection Refused, not authorized', - 'extra.hex_code' : '05', - 'extra.issuer_common_name' : 'Sectigo RSA Domain Validation Secure Server CA', - 'extra.issuer_country' : 'GB', - 'extra.issuer_locality_name' : 'Salford', - 'extra.issuer_organization_name' : 'Sectigo Limited', - 'extra.issuer_state_or_province_name' : 'Greater Manchester', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'DE:2C:98:30:27:2E:7D:C9:ED:A3:9D:AF:9E:CE:14:CC', - 'extra.raw_response' : '20020005', - 'extra.sha1_fingerprint' : '70:84:F1:6D:28:DA:B6:E6:27:60:13:8B:2C:93:52:B6:7B:4B:13:7B', - 'extra.sha256_fingerprint' : 'D2:D7:54:52:EB:86:4E:2D:34:4D:FC:CE:CD:CF:39:41:E1:06:5C:8B:B8:54:E6:0C:DF:FD:6E:E3:F1:B5:41:00', - 'extra.sha512_fingerprint' : '17:57:FB:88:9D:BE:A7:F0:29:A5:31:FC:79:DF:F7:8A:1C:D6:4A:DF:1B:4A:DC:BF:05:E7:E8:2F:79:9A:FA:FE:F7:E8:66:22:CB:B9:4C:72:F7:FB:6C:1D:59:8C:54:63:70:05:DE:7F:3C:2F:BA:B8:37:18:CE:29:6F:11:E8:AB', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.naics' : 454110, - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : '*.tracesafe.io', - 'extra.tag' : 'mqtt', - 'feed.name' : 'Open-MQTT', - 'protocol.application' : 'mqtt', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 12345, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'COLUMBUS', - 'source.geolocation.region' : 'OHIO', - 'source.ip' : '18.220.0.0', - 'source.port' : 8883, - 'source.reverse_dns' : '18-220-0-0.example.com', - 'time.observation' : '2020-03-15T00:00:00+00:00', - 'time.source' : '2022-02-07T12:56:53+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt_anon.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt_anon.py deleted file mode 100644 index 4618957240..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt_anon.py +++ /dev/null @@ -1,173 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_mqtt_anon.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open Anonymous MQTT', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_mqtt_anon-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mqtt-anon', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.cert_expiration_date' : '2030-05-06 08:07:05', - 'extra.cert_issue_date' : '2020-05-08 08:07:05', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : '02', - 'extra.cipher_suite' : 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', - 'extra.code' : 'Connection Accepted', - 'extra.hex_code' : '00', - 'extra.issuer_common_name' : 'RootCA', - 'extra.issuer_country' : 'CN', - 'extra.issuer_organization_name' : 'EMQ', - 'extra.issuer_state_or_province_name' : 'hangzhou', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'AB:A8:E0:2C:EF:AE:BF:9D:DD:FA:70:BA:2F:F2:CA:5C', - 'extra.raw_response' : '20020000', - 'extra.sha1_fingerprint' : '70:1A:1E:1F:EC:5F:7E:A9:12:32:B2:C9:8A:C9:EE:91:8E:0B:82:45', - 'extra.sha256_fingerprint' : '85:26:A2:F2:A2:50:CD:96:33:19:A6:2D:12:2E:97:6B:D3:06:3C:11:EA:01:B4:B7:25:2A:B7:4F:0A:8F:45:40', - 'extra.sha512_fingerprint' : '72:50:07:30:9A:6F:CB:FD:E2:80:69:02:65:62:77:16:C3:B4:0C:98:44:4E:D4:2C:AC:6B:AF:F8:9E:AB:51:C2:FA:A8:72:A3:45:DF:81:09:50:08:18:EB:03:34:FC:92:33:A7:12:46:FE:90:20:91:86:C5:4D:89:48:86:4C:CD', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.source.naics' : 518210, - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'Server', - 'extra.subject_country' : 'CN', - 'extra.subject_organization_name' : 'EMQ', - 'extra.subject_state_or_province_name' : 'hangzhou', - 'extra.tag' : 'mqtt,mqtt-anon', - 'feed.name' : 'Open Anonymous MQTT', - 'protocol.application' : 'mqtt', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 37963, - 'source.geolocation.cc' : 'CN', - 'source.geolocation.city' : 'SHENZHEN', - 'source.geolocation.region' : 'GUANGDONG SHENG', - 'source.ip' : '47.106.0.0', - 'source.port' : 8883, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:59:34+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mqtt-anon', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.cert_expiration_date' : '2022-03-06 13:48:03', - 'extra.cert_issue_date' : '2021-12-06 13:48:04', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : '06B25BEAD1F43266ABCFCDDE408D3544D04B', - 'extra.cipher_suite' : 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', - 'extra.code' : 'Connection Accepted', - 'extra.hex_code' : '00', - 'extra.issuer_common_name' : 'R3', - 'extra.issuer_country' : 'US', - 'extra.issuer_organization_name' : 'Lets Encrypt', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : '23:99:39:C6:77:D8:9F:55:90:FC:A5:FB:BA:72:8B:42', - 'extra.raw_response' : '20020000', - 'extra.sha1_fingerprint' : '20:0E:AC:E7:AF:07:8D:D3:16:7C:63:D1:B9:12:AD:1D:2C:F0:46:86', - 'extra.sha256_fingerprint' : 'DD:7A:4C:8A:1D:66:1D:7C:F5:17:04:5B:A0:B4:C4:E0:80:58:44:B4:DB:A7:5E:61:AE:43:9D:85:4C:9E:DC:83', - 'extra.sha512_fingerprint' : '55:B6:3D:56:A4:39:6E:99:B6:AF:72:AF:4D:3C:7C:C5:A8:C5:4F:A1:79:92:D0:46:8A:A2:9B:2A:48:0D:00:68:39:F0:B8:67:B4:E0:88:51:2A:D7:55:46:83:BD:ED:1E:09:6E:DB:3D:21:E2:AA:DB:42:6A:33:45:1A:2A:DB:4C', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.source.naics' : 518210, - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'example.com', - 'extra.tag' : 'mqtt,mqtt-anon', - 'feed.name' : 'Open Anonymous MQTT', - 'protocol.application' : 'mqtt', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 24940, - 'source.geolocation.cc' : 'DE', - 'source.geolocation.city' : 'WERNIGERODE', - 'source.geolocation.region' : 'SACHSEN-ANHALT', - 'source.ip' : '144.76.0.0', - 'source.port' : 8883, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:59:34+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mqtt-anon', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.cert_expiration_date' : '2030-08-05 16:51:57', - 'extra.cert_issue_date' : '2020-08-07 16:51:57', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'A71541EFAE529B03', - 'extra.cipher_suite' : 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA', - 'extra.code' : 'Connection Accepted', - 'extra.hex_code' : '00', - 'extra.issuer_common_name' : 'ClearView2Dev', - 'extra.issuer_organization_name' : 'Sohonet', - 'extra.issuer_organization_unit_name' : 'ClearView2Dev', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : '43:0D:A7:89:9E:76:8D:6E:D5:AD:95:CC:F2:91:87:56', - 'extra.raw_response' : '20020000', - 'extra.sha1_fingerprint' : '32:4B:66:98:FA:5B:D2:D1:F2:53:83:21:19:11:5A:A9:BE:85:56:16', - 'extra.sha256_fingerprint' : 'AE:0D:65:34:2F:51:F7:32:1E:DF:B1:DA:12:C7:6A:DE:42:B5:4B:FF:80:2C:E5:EF:99:F6:CC:01:4B:C9:77:68', - 'extra.sha512_fingerprint' : '44:C4:B8:19:FA:39:55:51:EC:E4:6D:C4:6D:0F:A5:46:BB:D5:F9:FD:A6:8D:DF:F3:2D:D2:92:6C:0B:D5:D3:25:CB:19:50:9D:A6:A4:D4:D3:2E:53:10:F5:8D:77:F7:90:F8:65:A7:79:AB:14:62:72:01:F3:EA:38:E2:68:C7:25', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.ssl_version' : 0, - 'extra.subject_common_name' : 'foo.example.com', - 'extra.subject_locality_name' : '<', - 'extra.subject_organization_name' : 'Sohonet', - 'extra.tag' : 'mqtt,mqtt-anon', - 'feed.name' : 'Open Anonymous MQTT', - 'protocol.application' : 'mqtt', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 5555, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'BURBANK', - 'source.geolocation.region' : 'CALIFORNIA', - 'source.ip' : '173.0.0.0', - 'source.port' : 8883, - 'source.reverse_dns' : 'example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:59:34+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_mssql.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_mssql.py deleted file mode 100644 index 0f12014e68..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_mssql.py +++ /dev/null @@ -1,123 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_mssql.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open MSSQL', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_mssql-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-mssql', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 310.0, - 'extra.instance_name': 'OPTIMA', - 'extra.named_pipe': '\\\\\\\\ERPOPTIMA\\\\pipe\\\\MSSQL$OPTIMA\\\\sql\\\\query', - 'extra.response_size': 310, - 'extra.tag': 'mssql', - 'extra.tcp_port': 49729, - 'extra.version': '13.2.5026.0', - 'feed.name': 'Open MSSQL', - 'protocol.application': 'mssql', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.local_hostname': 'ERPOPTIMA', - 'source.port': 1434, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-mssql', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 226.0, - 'extra.instance_name': 'MSSQLSERVER', - 'extra.response_size': 226, - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'mssql', - 'extra.tcp_port': 1433, - 'extra.version': '13.0.1601.5', - 'feed.name': 'Open MSSQL', - 'protocol.application': 'mssql', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.local_hostname': 'SERWER', - 'source.port': 1434, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-mssql', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 304.0, - 'extra.instance_name': 'INSERTGT', - 'extra.named_pipe': '\\\\\\\\ILONY\\\\pipe\\\\MSSQL$INSERTGT\\\\sql\\\\query', - 'extra.response_size': 304, - 'extra.tag': 'mssql', - 'extra.tcp_port': 49358, - 'extra.version': '10.50.2500.0', - 'feed.name': 'Open MSSQL', - 'protocol.application': 'mssql', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.local_hostname': 'ILONY', - 'source.port': 1434, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_mysql.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_mysql.py deleted file mode 100644 index 3e008f9502..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_mysql.py +++ /dev/null @@ -1,258 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_mysql.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible MySQL Server', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_mysql-test.csv", - } -EVENTS = [ - { - '__type' : 'Event', - 'classification.identifier' : 'open-mysql', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.browser_error' : 'x509: failed to load system roots and no roots provided', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.client_can_handle_expired_passwords' : True, - 'extra.client_compress' : True, - 'extra.client_connect_attrs' : True, - 'extra.client_connect_with_db' : True, - 'extra.client_deprecated_eof' : True, - 'extra.client_found_rows' : True, - 'extra.client_ignore_sigpipe' : True, - 'extra.client_ignore_space' : True, - 'extra.client_interactive' : True, - 'extra.client_local_files' : True, - 'extra.client_long_flag' : True, - 'extra.client_long_password' : True, - 'extra.client_multi_results' : True, - 'extra.client_multi_statements' : True, - 'extra.client_no_schema' : True, - 'extra.client_odbc' : True, - 'extra.client_plugin_auth' : True, - 'extra.client_plugin_auth_len_enc_client_data' : True, - 'extra.client_protocol_41' : True, - 'extra.client_ps_multi_results' : True, - 'extra.client_reserved' : True, - 'extra.client_secure_connection' : True, - 'extra.client_session_track' : False, - 'extra.client_ssl' : False, - 'extra.client_transactions' : False, - 'extra.error_code' : '1', - 'extra.error_id' : '1', - 'extra.error_message' : '1', - 'extra.handshake' : 'TLSv1.2', - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.mysql_protocol_version' : '10', - 'extra.server_version' : '5.7.37-0ubuntu0.18.04.1', - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.tag' : 'mysql', - 'feed.name' : 'Accessible MySQL Server', - 'protocol.application' : 'mysql', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 3306, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mysql', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.browser_error' : 'x509: failed to load system roots and no roots provided', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.client_can_handle_expired_passwords' : True, - 'extra.client_compress' : True, - 'extra.client_connect_attrs' : True, - 'extra.client_connect_with_db' : True, - 'extra.client_deprecated_eof' : True, - 'extra.client_found_rows' : True, - 'extra.client_ignore_sigpipe' : True, - 'extra.client_ignore_space' : True, - 'extra.client_interactive' : True, - 'extra.client_local_files' : True, - 'extra.client_long_flag' : True, - 'extra.client_long_password' : True, - 'extra.client_multi_results' : True, - 'extra.client_multi_statements' : True, - 'extra.client_no_schema' : True, - 'extra.client_odbc' : True, - 'extra.client_plugin_auth' : True, - 'extra.client_plugin_auth_len_enc_client_data' : True, - 'extra.client_protocol_41' : True, - 'extra.client_ps_multi_results' : True, - 'extra.client_reserved' : True, - 'extra.client_secure_connection' : True, - 'extra.client_session_track' : False, - 'extra.client_ssl' : False, - 'extra.client_transactions' : False, - 'extra.error_code' : '1', - 'extra.error_id' : '1', - 'extra.error_message' : '1', - 'extra.handshake' : 'TLSv1.2', - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.mysql_protocol_version' : '10', - 'extra.server_version' : '5.7.30-0ubuntu0.18.04.1-log', - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.tag' : 'mysql', - 'feed.name' : 'Accessible MySQL Server', - 'protocol.application' : 'mysql', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 3306, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mysql', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.browser_error' : 'x509: failed to load system roots and no roots provided', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.client_can_handle_expired_passwords' : True, - 'extra.client_compress' : True, - 'extra.client_connect_attrs' : True, - 'extra.client_connect_with_db' : True, - 'extra.client_deprecated_eof' : True, - 'extra.client_found_rows' : True, - 'extra.client_ignore_sigpipe' : True, - 'extra.client_ignore_space' : True, - 'extra.client_interactive' : True, - 'extra.client_local_files' : True, - 'extra.client_long_flag' : True, - 'extra.client_long_password' : True, - 'extra.client_multi_results' : True, - 'extra.client_multi_statements' : True, - 'extra.client_no_schema' : True, - 'extra.client_odbc' : True, - 'extra.client_plugin_auth' : True, - 'extra.client_plugin_auth_len_enc_client_data' : True, - 'extra.client_protocol_41' : True, - 'extra.client_ps_multi_results' : True, - 'extra.client_reserved' : True, - 'extra.client_secure_connection' : True, - 'extra.client_session_track' : False, - 'extra.client_ssl' : False, - 'extra.client_transactions' : False, - 'extra.error_code' : '1', - 'extra.error_id' : '1', - 'extra.error_message' : '1', - 'extra.handshake' : 'TLSv1.2', - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.mysql_protocol_version' : '10', - 'extra.server_version' : '8.0.23', - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.source.sector' : 'Retail Trade', - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.tag' : 'mysql', - 'feed.name' : 'Accessible MySQL Server', - 'protocol.application' : 'mysql', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 3306, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_nat_pmp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_nat_pmp.py deleted file mode 100644 index beeac2717f..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_nat_pmp.py +++ /dev/null @@ -1,116 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_nat_pmp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open NATPMP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_nat_pmp-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-natpmp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 6.0, - 'extra.external_ip': '192.168.0.1', - 'extra.opcode': '128', - 'extra.response_size': 12, - 'extra.tag': 'nat-pmp', - 'extra.uptime': 291278940, - 'feed.name': 'Open NATPMP', - 'protocol.application': 'natpmp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 5351, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-natpmp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 6.0, - 'extra.external_ip': '192.168.0.2', - 'extra.opcode': '128', - 'extra.response_size': 12, - 'extra.tag': 'nat-pmp', - 'extra.uptime': 768416, - 'feed.name': 'Open NATPMP', - 'protocol.application': 'natpmp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 5351, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, - {'__type': 'Event', - 'classification.identifier': 'open-natpmp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 6.0, - 'extra.external_ip': '192.168.0.3', - 'extra.opcode': '128', - 'extra.response_size': 12, - 'extra.tag': 'nat-pmp', - 'extra.uptime': 19629454, - 'feed.name': 'Open NATPMP', - 'protocol.application': 'natpmp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 5351, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_netbios.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_netbios.py deleted file mode 100644 index febe8305c1..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_netbios.py +++ /dev/null @@ -1,121 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_netbios.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Netbios', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_netbios-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-netbios-nameservice', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 4.58, - 'extra.mac_address': '00-00-00-00-00-00', - 'extra.machine_name': 'NBG6503', - 'extra.response_size': 229, - 'extra.tag': 'netbios', - 'feed.name': 'Netbios', - 'protocol.application': 'netbios-nameservice', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.account': 'NBG6503', - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 137, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-netbios-nameservice', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3.86, - 'extra.mac_address': '00-00-00-00-00-00', - 'extra.machine_name': 'NAS-OLD', - 'extra.response_size': 193, - 'extra.tag': 'netbios', - 'extra.workgroup': 'PRACOWNIAELN.', - 'feed.name': 'Netbios', - 'protocol.application': 'netbios-nameservice', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.account': 'NAS-OLD', - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 137, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-netbios-nameservice', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3.14, - 'extra.mac_address': '00-25-90-F0-64-64', - 'extra.machine_name': 'HR-SRV01', - 'extra.response_size': 157, - 'extra.sector': 'Government', - 'extra.tag': 'netbios', - 'extra.workgroup': 'HRSIGMA', - 'feed.name': 'Netbios', - 'protocol.application': 'netbios-nameservice', - 'protocol.transport': 'udp', - 'raw': 'InRpbWVzdGFtcCIsImlwIiwicHJvdG9jb2wiLCJwb3J0IiwiaG9zdG5hbWUiLCJ0YWciLCJtYWNfYWRkcmVzcyIsImFzbiIsImdlbyIsInJlZ2lvbiIsImNpdHkiLCJ3b3JrZ3JvdXAiLCJtYWNoaW5lX25hbWUiLCJ1c2VybmFtZSIsIm5haWNzIiwic2ljIiwic2VjdG9yIiwicmVzcG9uc2Vfc2l6ZSIsImFtcGxpZmljYXRpb24iCiIyMDEwLTAyLTEwIDAwOjAwOjAyIiwxOTIuMTY4LjAuMyx1ZHAsMTM3LG5vZGUwMy5leGFtcGxlLmNvbSxuZXRiaW9zLDAwLTI1LTkwLUYwLTY0LTY0LDY0NTEyLFpaLFJlZ2lvbixDaXR5LEhSU0lHTUEsSFItU1JWMDEsLDAsMCxHb3Zlcm5tZW50LDE1NywzLjE0', - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 137, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_netis_router.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_netis_router.py deleted file mode 100644 index 043cdf1aad..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_netis_router.py +++ /dev/null @@ -1,107 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Sebastian Wagner -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_netis_router.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_netis_router-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-netis', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 18.0, - 'extra.response': 'Login:', - 'extra.response_size': 18, - 'extra.tag': 'netis_vulnerability', - 'feed.name': 'Open-Netis', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 53413, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-netis', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 18.0, - 'extra.response': 'Login:', - 'extra.response_size': 18, - 'extra.tag': 'netis_vulnerability', - 'feed.name': 'Open-Netis', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 53413, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-netis', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 18.0, - 'extra.response': 'Login:', - 'extra.response_size': 18, - 'extra.tag': 'netis_vulnerability', - 'feed.name': 'Open-Netis', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 53413, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ntp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ntp.py deleted file mode 100644 index 85ef710d4e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ntp.py +++ /dev/null @@ -1,161 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_ntp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'NTP Version', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ntp-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'ntp-version', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 27.0, - 'extra.clock': '0xe6ac3809.363028e7', - 'extra.frequency': 2.018, - 'extra.jitter': 0.977, - 'extra.leap': 0.0, - 'extra.noise': '0.984', - 'extra.offset': 0.557, - 'extra.peer': 18986, - 'extra.poll': 10, - 'extra.precision': -10, - 'extra.refid': '81.15.252.130', - 'extra.reftime': '0xe6ac35ba.2d2e8f2b', - 'extra.response_size': 324, - 'extra.rootdelay': 17.685, - 'extra.rootdispersion': 61.254, - 'extra.stability': '0.027', - 'extra.state': '4', - 'extra.stratum': 4, - 'extra.system': 'UNIX', - 'extra.tag': 'ntpversion', - 'extra.version': '4', - 'feed.name': 'NTP Version', - 'protocol.application': 'ntp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 123, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'ntp-version', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 27.33, - 'extra.clk_wander': 0.007, - 'extra.clock': '0xE6AC3806.7DF3B7A0', - 'extra.frequency': -20.407, - 'extra.jitter': 8.776, - 'extra.leap': 0.0, - 'extra.mintc': '3', - 'extra.offset': -14.502, - 'extra.peer': 19244, - 'extra.precision': -10, - 'extra.refid': '10.48.21.21', - 'extra.reftime': '0xE6AC3431.B3B64790', - 'extra.response_size': 328, - 'extra.rootdelay': 32.25, - 'extra.rootdispersion': 105.778, - 'extra.sector': 'Transportation and Warehousing', - 'extra.stratum': 8, - 'extra.system': 'UNIX', - 'extra.tag': 'ntpversion', - 'extra.tc': 10, - 'extra.version': '4', - 'feed.name': 'NTP Version', - 'protocol.application': 'ntp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 123, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'ntp-version', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 27.0, - 'extra.clk_wander': 0.001, - 'extra.clock': '0xE6AC380A.5A1CAD00', - 'extra.frequency': -24.01, - 'extra.jitter': 2.343, - 'extra.leap': 0.0, - 'extra.mintc': '3', - 'extra.offset': 0.49, - 'extra.peer': 51892, - 'extra.precision': -10, - 'extra.refid': '172.28.0.1', - 'extra.reftime': '0xE6AC3020.0C49BA80', - 'extra.response_size': 324, - 'extra.rootdelay': 7.749, - 'extra.rootdispersion': 81.612, - 'extra.stratum': 4, - 'extra.system': 'UNIX', - 'extra.tag': 'ntpversion', - 'extra.tc': 10, - 'extra.version': '4', - 'feed.name': 'NTP Version', - 'protocol.application': 'ntp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 123, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ntpmonitor.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ntpmonitor.py deleted file mode 100644 index ff0e95f3ea..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ntpmonitor.py +++ /dev/null @@ -1,108 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_ntpmonitor.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'NTP Monitor', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ntpmonitor-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'ntp-monitor', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 55.33, - 'extra.packets': 2, - 'extra.size': 664, - 'feed.name': 'NTP Monitor', - 'protocol.application': 'ntp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 123, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'ntp-monitor', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3666.67, - 'extra.packets': 100, - 'extra.size': 44000, - 'feed.name': 'NTP Monitor', - 'protocol.application': 'ntp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 123, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'ntp-monitor', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3666.67, - 'extra.packets': 100, - 'extra.size': 44000, - 'feed.name': 'NTP Monitor', - 'protocol.application': 'ntp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 123, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_portmapper.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_portmapper.py deleted file mode 100644 index 11caec78a1..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_portmapper.py +++ /dev/null @@ -1,120 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_portmapper.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open Portmapper', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_portmapper-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-portmapper', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3.7, - 'extra.exports': '/mnt/export 192.168.0.0', - 'extra.programs': '100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp; ' - '100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp;', - 'extra.response_size': 148, - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'portmapper', - 'feed.name': 'Open Portmapper', - 'protocol.application': 'portmapper', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 111, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-portmapper', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3.7, - 'extra.exports': '/mnt/export 192.168.0.0', - 'extra.programs': '100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp; ' - '100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp;', - 'extra.response_size': 148, - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'portmapper', - 'feed.name': 'Open Portmapper', - 'protocol.application': 'portmapper', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 111, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-portmapper', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3.7, - 'extra.exports': '/mnt/export 192.168.0.0', - 'extra.programs': '100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp; ' - '100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp;', - 'extra.response_size': 148, - 'extra.sector': 'Government', - 'extra.tag': 'portmapper', - 'feed.name': 'Open Portmapper', - 'protocol.application': 'portmapper', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 111, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_postgres.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_postgres.py deleted file mode 100644 index 43a297f787..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_postgres.py +++ /dev/null @@ -1,199 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_postgres.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible-PostgreSQL', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_postgres-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-postgres', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.client_ssl' : False, - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.protocol_error_code' : '0A000', - 'extra.protocol_error_file' : 'postmaster.c', - 'extra.protocol_error_line' : '1798', - 'extra.protocol_error_message' : 'unsupported frontend protocol 255.255: server supports 1.0 to 3.0', - 'extra.protocol_error_routine' : 'ProcessStartupPacket', - 'extra.protocol_error_severity' : 'FATAL', - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.startup_error_code' : '28000', - 'extra.startup_error_file' : 'postmaster.c', - 'extra.startup_error_line' : 1893, - 'extra.startup_error_message' : 'no PostgreSQL user name specified in startup packet', - 'extra.startup_error_routine' : 'ProcessStartupPacket', - 'extra.startup_error_severity' : 'FATAL', - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.supported_protocols' : '1.0-3.0', - 'extra.tag' : 'postgres', - 'feed.name' : 'Accessible-PostgreSQL', - 'protocol.application' : 'postgres', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 5432, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-postgres', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.client_ssl' : False, - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.protocol_error_code' : '0A000', - 'extra.protocol_error_file' : 'postmaster.c', - 'extra.protocol_error_line' : '1798', - 'extra.protocol_error_message' : 'unsupported frontend protocol 255.255: server supports 1.0 to 3.0', - 'extra.protocol_error_routine' : 'ProcessStartupPacket', - 'extra.protocol_error_severity' : 'FATAL', - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.startup_error_code' : '28000', - 'extra.startup_error_file' : 'postmaster.c', - 'extra.startup_error_line' : 1893, - 'extra.startup_error_message' : 'no PostgreSQL user name specified in startup packet', - 'extra.startup_error_routine' : 'ProcessStartupPacket', - 'extra.startup_error_severity' : 'FATAL', - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.supported_protocols' : '1.0-3.0', - 'extra.tag' : 'postgres', - 'feed.name' : 'Accessible-PostgreSQL', - 'protocol.application' : 'postgres', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 5432, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-postgres', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.client_ssl' : False, - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.protocol_error_code' : '0A000', - 'extra.protocol_error_file' : 'postmaster.c', - 'extra.protocol_error_line' : '1798', - 'extra.protocol_error_message' : 'unsupported frontend protocol 255.255: server supports 1.0 to 3.0', - 'extra.protocol_error_routine' : 'ProcessStartupPacket', - 'extra.protocol_error_severity' : 'FATAL', - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.startup_error_code' : '28000', - 'extra.startup_error_file' : 'postmaster.c', - 'extra.startup_error_line' : 1893, - 'extra.startup_error_message' : 'no PostgreSQL user name specified in startup packet', - 'extra.startup_error_routine' : 'ProcessStartupPacket', - 'extra.startup_error_severity' : 'FATAL', - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.supported_protocols' : '1.0-3.0', - 'extra.tag' : 'postgres', - 'feed.name' : 'Accessible-PostgreSQL', - 'protocol.application' : 'postgres', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 5432, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_qotd.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_qotd.py deleted file mode 100644 index de52af6259..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_qotd.py +++ /dev/null @@ -1,119 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_qotd.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open QOTD', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_qotd-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-qotd', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 166.0, - 'extra.quote': '_The secret of being miserable is to have leisure to bother ' - 'about whether?? you are happy or not. The cure for it is ' - 'occupation._?? George Bernard Shaw (1856-1950)?', - 'extra.response_size': 166, - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'qotd', - 'feed.name': 'Open QOTD', - 'protocol.application': 'qotd', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 17, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-qotd', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 162.0, - 'extra.quote': '_Oh the nerves, the nerves; the mysteries of this machine ' - 'called man!?? Oh the little that unhinges it, poor creatures ' - 'that we are!_?? Charles Dickens (1812-70)?', - 'extra.response_size': 162, - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'qotd', - 'feed.name': 'Open QOTD', - 'protocol.application': 'qotd', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 17, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-qotd', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 162.0, - 'extra.quote': '_Oh the nerves, the nerves; the mysteries of this machine ' - 'called man!?? Oh the little that unhinges it, poor creatures ' - 'that we are!_?? Charles Dickens (1812-70)?', - 'extra.response_size': 162, - 'extra.tag': 'qotd', - 'feed.name': 'Open QOTD', - 'protocol.application': 'qotd', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 17, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_quic.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_quic.py deleted file mode 100644 index 23d11ce996..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_quic.py +++ /dev/null @@ -1,118 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_quic.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible QUIC Report', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_quic-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-quic', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.source.naics' : 517311, - 'extra.tag' : 'quic', - 'extra.version_field_1' : 'Q050', - 'extra.version_field_3' : 'Q046', - 'extra.version_field_4' : 'Q043', - 'feed.name' : 'Accessible QUIC Report', - 'protocol.transport' : 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 5607, - 'source.geolocation.cc' : 'UK', - 'source.geolocation.city' : 'LONDON', - 'source.geolocation.region' : 'LONDON', - 'source.ip' : '176.255.0.0', - 'source.port' : 443, - 'source.reverse_dns' : 'test1.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T14:31:17+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-quic', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.source.naics' : 517311, - 'extra.tag' : 'quic', - 'extra.version_field_1' : 'Q050', - 'extra.version_field_2' : 'Q046', - 'extra.version_field_4' : 'Q043', - 'feed.name' : 'Accessible QUIC Report', - 'protocol.transport' : 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 6327, - 'source.geolocation.cc' : 'CA', - 'source.geolocation.city' : 'MEACHAM', - 'source.geolocation.region' : 'SASKATCHEWAN', - 'source.ip' : '24.244.0.0', - 'source.port' : 443, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T14:31:17+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-quic', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.source.naics' : 517919, - 'extra.tag' : 'quic', - 'extra.version_field_2' : 'Q050', - 'extra.version_field_3' : 'Q046', - 'extra.version_field_4' : 'Q043', - 'feed.name' : 'Accessible QUIC Report', - 'protocol.transport' : 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 20940, - 'source.geolocation.cc' : 'JP', - 'source.geolocation.city' : 'OSAKA', - 'source.geolocation.region' : 'OSAKA', - 'source.ip' : '23.60.0.0', - 'source.port' : 443, - 'source.reverse_dns' : 'test3.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T14:31:17+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_radmin.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_radmin.py deleted file mode 100644 index 7c052c451c..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_radmin.py +++ /dev/null @@ -1,236 +0,0 @@ -# SPDX-FileCopyrightText: 2020 sinus-x -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), "testdata/scan_radmin.csv")) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = { - "feed.name": "Accessible Radmin", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2020-08-19T00:00:00+00:00", - "extra.file_name": "2020-08-19-scan_radmin-test-test.csv", -} - -EVENTS = [ - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517312, - "extra.tag": "radmin", - "extra.version": "Radmin (Details Unknown)", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.asn": 701, - "source.geolocation.cc": "US", - "source.geolocation.city": "BROOKLYN", - "source.geolocation.region": "NEW YORK", - "source.ip": "74.101.218.75", - "source.port": 4899, - "source.reverse_dns": "static-74-101-218-75.nycmny.fios.verizon.net", - "time.source": "2020-07-06T13:55:26+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[1]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.tag": "radmin", - "extra.version": "Radmin v3.X Radmin Authentication", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.asn": 56618, - "source.geolocation.cc": "RU", - "source.geolocation.city": "MURMANSK", - "source.geolocation.region": "MURMANSKAYA OBLAST", - "source.ip": "192.162.189.171", - "source.port": 4899, - "source.reverse_dns": "rubin.an.ru", - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[2]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517311, - "extra.tag": "radmin", - "extra.version": "Radmin (Details Unknown)", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.geolocation.cc": "CN", - "source.geolocation.city": "BEIJING", - "source.geolocation.region": "BEIJING SHI", - "source.asn": 4808, - "source.ip": "111.197.143.69", - "source.port": 4899, - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[3]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517311, - "extra.tag": "radmin", - "extra.version": "Radmin v3.X Radmin Authentication", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.geolocation.cc": "KR", - "source.geolocation.city": "DAEIN-DONG", - "source.geolocation.region": "GWANGJU-GWANGYEOKSI", - "source.asn": 4766, - "source.ip": "121.147.215.220", - "source.port": 4899, - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[4]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517311, - "extra.tag": "radmin", - "extra.version": "Radmin v3.X Radmin Authentication", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.geolocation.cc": "KR", - "source.geolocation.city": "DAEIN-DONG", - "source.geolocation.region": "GWANGJU-GWANGYEOKSI", - "source.asn": 4766, - "source.ip": "121.147.215.178", - "source.port": 4899, - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[5]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517312, - "extra.tag": "radmin", - "extra.version": "Radmin v3.X Radmin Authentication", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.geolocation.cc": "CN", - "source.geolocation.city": "CHONGQING", - "source.geolocation.region": "CHONGQING SHI", - "source.asn": 9808, - "source.ip": "183.230.5.219", - "source.port": 4899, - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[6]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.tag": "radmin", - "extra.version": "Radmin v3.X Radmin Authentication", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.geolocation.cc": "RU", - "source.geolocation.city": "MOSCOW", - "source.geolocation.region": "MOSKVA", - "source.asn": 34300, - "source.ip": "85.93.154.74", - "source.port": 4899, - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[7]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517311, - "extra.tag": "radmin", - "extra.version": "Radmin v3.X Radmin Authentication", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.geolocation.cc": "BE", - "source.geolocation.city": "BRASSCHAAT", - "source.geolocation.region": "ANTWERPEN", - "source.asn": 5432, - "source.ip": "81.246.135.247", - "source.port": 4899, - "source.reverse_dns": "247.135-246-81.adsl-dyn.isp.belgacom.be", - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[8]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517312, - "extra.tag": "radmin", - "extra.version": "Radmin v3.X Radmin Authentication", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.geolocation.cc": "ES", - "source.geolocation.city": "LAS PALMAS DE GRAN CANARIA", - "source.geolocation.region": "LAS PALMAS", - "source.asn": 12430, - "source.ip": "46.27.146.22", - "source.port": 4899, - "source.reverse_dns": "static-22-146-27-46.ipcom.comunitel.net", - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[9]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == "__main__": - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_rdp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_rdp.py deleted file mode 100644 index 28a4a02c23..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_rdp.py +++ /dev/null @@ -1,117 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_rdp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible RDP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_rdp-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Accessible RDP', - "classification.identifier": "open-rdp", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.bluekeep_vulnerable": False, - "extra.cert_expiration_date": "2019-10-29 02:22:06", - "extra.cert_issue_date": "2019-04-29 02:22:06", - "extra.cert_length": 5678, - "extra.cert_serial_number": "1EF2B37AF850C9BF4E88F18177001D6B", - "extra.cve20190708_vulnerable": False, - "extra.issuer_common_name": "KABESRV.KABE.local", - "extra.key_algorithm": "rsaEncryption", - "extra.md5_fingerprint": "BC:6E:C3:E2:98:22:EC:BA:5B:30:E2:53:FD:4A:9D:FF", - "extra.naics": 517311, - "extra.rdp_protocol": "RDP", - "extra.sha1_fingerprint": "EC:BB:4D:DB:9F:0C:D3:FF:5B:49:EA:B1:56:62:B6:A7:5D:60:54:42", - "extra.sha256_fingerprint": "B7:C9:F4:07:D5:C0:75:1D:EA:0C:40:E7:26:39:C2:30:C6:13:83:7E:18:46:D8:E9:4C:45:3F:88:1B:0B:70:76", - "extra.sha512_fingerprint": "08:AC:75:FA:EB:A3:2B:44:15:DE:6D:A7:0B:C0:AE:17:94:F3:55:D9:EC:70:AC:5B:B7:94:79:F0:D7:84:83:89:CB:A9:11:E0:08:D7:54:4D:33:85:89:D2:A8:DD:9D:15:F4:CC:95:DE:6A:E3:DF:6B:FA:8B:27:E3:DA:16:AF:0A", - "extra.signature_algorithm": "sha256WithRSAEncryption", - "extra.ssl_version": 2, - "extra.subject_common_name": "KABESRV.KABE.local", - "extra.tag": "rdp", - "protocol.application": "rdp", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 5678, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.178", - "source.port": 5678, - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T15:45:51+00:00" - }, - {'__type': 'Event', - 'feed.name': 'Accessible RDP', - "classification.identifier": "open-rdp", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.bluekeep_vulnerable": False, - "extra.cert_expiration_date": "2019-10-16 06:15:20", - "extra.cert_issue_date": "2019-04-16 06:15:20", - "extra.cert_length": 5678, - "extra.cert_serial_number": "3FF3EBC5CF154BA54D128A8548C8AAF5", - "extra.cve20190708_vulnerable": False, - "extra.issuer_common_name": "RAMBLA01.rambla.local", - "extra.key_algorithm": "rsaEncryption", - "extra.md5_fingerprint": "38:73:6A:B3:AA:41:69:C9:BA:E7:3D:D7:40:16:F8:AA", - "extra.naics": 517311, - "extra.rdp_protocol": "RDP", - "extra.sector": "Information Technology", - "extra.sha1_fingerprint": "7A:67:1F:F8:87:C6:B0:AC:A9:84:15:B7:40:EC:CB:19:AA:E3:19:52", - "extra.sha256_fingerprint": "8F:CD:7D:C4:80:2D:8D:9B:06:A0:40:18:9F:ED:73:7A:BA:83:55:BE:1B:56:83:A2:97:DF:BB:B4:06:57:CB:F1", - "extra.sha512_fingerprint": "E8:9B:9A:93:69:B4:58:01:D8:46:C2:DC:01:20:1E:DD:93:E1:EB:E3:9D:6B:65:A0:C5:00:6C:A4:44:08:FE:A4:A6:19:FF:55:79:F2:AA:61:68:C8:1C:B0:CE:78:EB:84:DD:29:9D:64:2F:4E:25:31:3A:6C:B8:02:C9:AF:F5:1F", - "extra.signature_algorithm": "sha1WithRSAEncryption", - "extra.ssl_version": 2, - "extra.subject_common_name": "RAMBLA01.rambla.local", - "extra.tag": "rdp", - "protocol.application": "rdp", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 5678, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.233", - "source.port": 5678, - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T15:45:51+00:00" - }, - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_rdpeudp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_rdpeudp.py deleted file mode 100644 index 54be35a26f..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_rdpeudp.py +++ /dev/null @@ -1,109 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Sebastian Waldbauer -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_rdpeudp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible MS RDPEUDP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_rdpeudp-test-geo.csv", - } - -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'accessible-msrdpeudp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 77.0, - 'extra.response_size': 1232, - 'extra.sessionid': '05b28c0c', - 'extra.tag': 'rdpeudp', - 'feed.name': 'Accessible MS RDPEUDP', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 3389, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-msrdpeudp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 77.0, - 'extra.response_size': 1232, - 'extra.sessionid': '053d355f', - 'extra.tag': 'rdpeudp', - 'feed.name': 'Accessible MS RDPEUDP', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 3389, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-msrdpeudp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 77.0, - 'extra.response_size': 1232, - 'extra.sessionid': '0567a8cb', - 'extra.tag': 'rdpeudp', - 'feed.name': 'Accessible MS RDPEUDP', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 3389, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_redis.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_redis.py deleted file mode 100644 index 04552e2ec0..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_redis.py +++ /dev/null @@ -1,107 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_redis.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open Redis', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_redis-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Open Redis', - "classification.identifier": "open-redis", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.build_id": "26069fb482f6334b", - "extra.connected_clients": "50", - "extra.gcc_version": "4.7.2", - "extra.git_sha1": "00000000", - "extra.mode": "standalone", - "extra.multiplexing_api": "epoll", - "extra.naics": 541512, - "extra.os.name": "Linux 3.2.0-4-amd64 x86_64", - "extra.process_id": "2127", - "extra.run_id": "d440b0b2fb3d1db655ad607e11e6f38011a0f599", - "extra.sic": 737999, - "extra.tag": "redis", - "extra.uptime": 27946314, - "extra.version": "2.8.19", - "protocol.application": "redis", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 201229, - "source.geolocation.cc": "DE", - "source.geolocation.city": "FRANKFURT AM MAIN", - "source.geolocation.region": "HESSEN", - "source.ip": "198.51.100.152", - "source.port": 6379, - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2016-07-24T00:42:33+00:00" - }, - {'__type': 'Event', - 'feed.name': 'Open Redis', - "classification.identifier": "open-redis", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.build_id": "e41bf84a0cecf09d", - "extra.connected_clients": "25376", - "extra.gcc_version": "4.8.4", - "extra.git_sha1": "00000000", - "extra.mode": "standalone", - "extra.multiplexing_api": "epoll", - "extra.os.name": "Linux 3.18.24-sirzion x86_64", - "extra.process_id": "343519", - "extra.run_id": "53d63f23511dc0080b49aaa8e8203d65619f1c8c", - "extra.tag": "redis", - "extra.uptime": 310556, - "extra.version": "3.0.6", - "protocol.application": "redis", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 12586, - "source.geolocation.cc": "DE", - "source.geolocation.city": "FRANKFURT AM MAIN", - "source.geolocation.region": "HESSEN", - "source.ip": "198.51.100.67", - "source.port": 6379, - "source.reverse_dns": "198-51-100-67.example.net", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2016-07-24T00:42:43+00:00" - }, - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_rsync.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_rsync.py deleted file mode 100644 index e2a961f710..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_rsync.py +++ /dev/null @@ -1,116 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_rsync.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible Rsync', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_rsync-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'accessible-rsync', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.has_password': 'N', - 'extra.module': 'system|Backup system;system_full|Backup full ' - 'system;mysql|Backup virtual mysql;netadmin|Backup virtual ' - 'netadmin;', - 'extra.tag': 'rsync', - 'feed.name': 'Accessible Rsync', - 'protocol.application': 'rsync', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 873, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-rsync', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.has_password': 'N', - 'extra.module': 'system|Backup system;system_full|Backup full ' - 'system;mysql|Backup virtual mysql;netadmin|Backup virtual ' - 'netadmin;', - 'extra.tag': 'rsync', - 'feed.name': 'Accessible Rsync', - 'protocol.application': 'rsync', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 873, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-rsync', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.has_password': 'N', - 'extra.module': 'system|Backup system;system_full|Backup full ' - 'system;mysql|Backup virtual mysql;netadmin|Backup virtual ' - 'netadmin;', - 'extra.tag': 'rsync', - 'feed.name': 'Accessible Rsync', - 'protocol.application': 'rsync', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 873, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_sip.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_sip.py deleted file mode 100644 index 6b972ec5d5..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_sip.py +++ /dev/null @@ -1,124 +0,0 @@ -# SPDX-FileCopyrightText: 2023 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_sip.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible-SIP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_sip-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-sip', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.sip_allow': 'INVITE,ACK,BYE,CANCEL,REGISTER', - 'extra.amplification': 15.57, - 'extra.content_length': 0, - 'extra.response_size': 109, - 'extra.sip': 'SIP/2.0', - 'extra.sip_code': '489', - 'extra.sip_reason': 'Event Package Not Supported', - 'extra.tag': 'sip', - 'feed.name': 'Accessible-SIP', - 'protocol.application': 'sip', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 5060, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-sip', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 62.57, - 'extra.content_length': 364, - 'extra.content_type': 'text/plain', - 'extra.response_size': 438, - 'extra.sip': 'SIP/2.0', - 'extra.sip_code': '400', - 'extra.sip_reason': 'Bad Request', - 'extra.tag': 'sip', - 'feed.name': 'Accessible-SIP', - 'protocol.application': 'sip', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 5060, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-sip', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 6.57, - 'extra.content_length': 0, - 'extra.response_size': 46, - 'extra.sip': 'SIP/2.0', - 'extra.sip_code': '400', - 'extra.sip_reason': 'Bad Request', - 'extra.tag': 'sip', - 'feed.name': 'Accessible-SIP', - 'protocol.application': 'sip', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 5060, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_slp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_slp.py deleted file mode 100644 index f05973cf5c..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_slp.py +++ /dev/null @@ -1,137 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_slp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible SLP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_slp-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-slp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.error_code': '5', - 'extra.error_code_text': 'Unsupported SLP SPI', - 'extra.flags': '0x0000', - 'extra.function': '2', - 'extra.function_text': 'Service reply', - 'extra.language_tag': 'en', - 'extra.language_tag_length': '2', - 'extra.raw_response': 'MDIwMjAwMDAxNDAwMDAwMDAwMDAwMDA1MDAwMjY1NmUwMDA1MDAwMA==', - 'extra.response_size': 40, - 'extra.tag': 'slp', - 'extra.version': '2', - 'extra.xid': '5', - 'feed.name': 'Accessible SLP', - 'protocol.application': 'slp', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 427, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-slp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.error_code': '5', - 'extra.error_code_text': 'Unsupported SLP SPI', - 'extra.flags': '0x0000', - 'extra.function': '2', - 'extra.function_text': 'Service reply', - 'extra.language_tag': 'en', - 'extra.language_tag_length': '2', - 'extra.raw_response': 'MDIwMjAwMDAxNDAwMDAwMDAwMDAwMDA1MDAwMjY1NmUwMDA1MDAwMA==', - 'extra.response_size': 40, - 'extra.tag': 'slp', - 'extra.version': '2', - 'extra.xid': '5', - 'feed.name': 'Accessible SLP', - 'protocol.application': 'slp', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 427, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-slp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.error_code': '5', - 'extra.error_code_text': 'Unsupported SLP SPI', - 'extra.flags': '0x0000', - 'extra.function': '2', - 'extra.function_text': 'Service reply', - 'extra.language_tag': 'en', - 'extra.language_tag_length': '2', - 'extra.raw_response': 'MDIwMjAwMDAxNDAwMDAwMDAwMDAwMDA1MDAwMjY1NmUwMDA1MDAwMA==', - 'extra.response_size': 40, - 'extra.tag': 'slp', - 'extra.version': '2', - 'extra.xid': '5', - 'feed.name': 'Accessible SLP', - 'protocol.application': 'slp', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 427, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_smb.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_smb.py deleted file mode 100644 index 921525122c..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_smb.py +++ /dev/null @@ -1,124 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_smb.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible SMB', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_smb-test-geo.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Accessible SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 445, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Accessible SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 445, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Accessible SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 445, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_smb_json.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_smb_json.py deleted file mode 100644 index cae83d2733..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_smb_json.py +++ /dev/null @@ -1,123 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Sebastian Waldbauer -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest -import json - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser_json import ShadowserverJSONParserBot -from intelmq.tests.bots.parsers.shadowserver.test_testdata import csvtojson - -EXAMPLE_FILE = csvtojson(os.path.join(os.path.dirname(__file__), 'testdata/scan_smb.csv')) - -EXAMPLE_REPORT = { - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_smb-test-geo.json", - } - -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Accessible-SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode(json.dumps([json.loads(EXAMPLE_FILE)[0]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 445, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Accessible-SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode(json.dumps([json.loads(EXAMPLE_FILE)[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 445, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Accessible-SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode(json.dumps([json.loads(EXAMPLE_FILE)[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 445, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverJSONParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverJSONParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverJSONParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_smtp_vulnerable.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_smtp_vulnerable.py deleted file mode 100644 index 4428420cfb..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_smtp_vulnerable.py +++ /dev/null @@ -1,92 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Mikk Margus Möll -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_smtp_vulnerable.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Vulnerable SMTP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2021-07-08T00:00:00+00:00", - "extra.file_name": "2021-07-08-scan_smtp_vulnerable-test-test.csv", - } - -EVENTS = [ - { - '__type' : 'Event', - 'classification.identifier' : 'vulnerable-smtp', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.banner' : '220 smtp-server.invalid ESMTP Exim 4.80 Wed, 11 Jun 2021 10:00:00 +0300|', - 'extra.tag' : 'smtp;21nails', - 'feed.name' : 'Vulnerable SMTP', - 'protocol.application' : 'smtp', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 12345, - 'source.geolocation.cc' : 'EE', - 'source.geolocation.city' : 'TALLINN', - 'source.geolocation.region' : 'HARJUMAA', - 'source.ip' : '1.2.3.4', - 'source.port' : 25, - 'source.reverse_dns' : 'smtp-server.invalid', - 'time.observation' : '2021-07-08T00:00:00+00:00', - 'time.source' : '2021-07-08T11:58:42+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'vulnerable-smtp', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.banner' : '220 smtp-out.invalid, ESMTP EXIM 4.86_2|', - 'extra.tag' : 'smtp;21nails', - 'feed.name' : 'Vulnerable SMTP', - 'protocol.application' : 'smtp', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 23456, - 'source.geolocation.cc' : 'EE', - 'source.geolocation.city' : 'TALLINN', - 'source.geolocation.region' : 'HARJUMAA', - 'source.ip' : '5.6.7.8', - 'source.port' : 25, - 'source.reverse_dns' : 'smtp-out.invalid', - 'time.observation' : '2021-07-08T00:00:00+00:00', - 'time.source' : '2021-07-08T11:58:44+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_snmp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_snmp.py deleted file mode 100644 index e6da5b34f9..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_snmp.py +++ /dev/null @@ -1,120 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_snmp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open SNMP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_snmp-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-snmp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 1.94, - 'extra.community': 'public', - 'extra.response_size': 165, - 'extra.sysdesc': 'Linux localhost 3.18.20 #1 SMP Mon Jul 9 14:11:21 CST 2018 ' - 'armv7l', - 'extra.tag': 'snmp', - 'extra.version': '2', - 'feed.name': 'Open SNMP', - 'protocol.application': 'snmp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 161, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-snmp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 1.35, - 'extra.community': 'public', - 'extra.device_sector': 'consumer', - 'extra.device_type': 'router', - 'extra.device_vendor': 'MikroTik', - 'extra.response_size': 115, - 'extra.sysdesc': 'RouterOS CCR1009-8G-1S-1S+', - 'extra.tag': 'snmp,iot', - 'extra.version': '2', - 'feed.name': 'Open SNMP', - 'protocol.application': 'snmp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 161, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-snmp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 1.0, - 'extra.community': 'public', - 'extra.response_size': 85, - 'extra.tag': 'snmp', - 'extra.version': '2', - 'feed.name': 'Open SNMP', - 'protocol.application': 'snmp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 161, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_socks.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_socks.py deleted file mode 100644 index 067602aa10..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_socks.py +++ /dev/null @@ -1,107 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_socks.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open SOCKS', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_socks-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-socks', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'feed.name' : 'Open SOCKS', - 'protocol.application' : 'socks4', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 1080, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-socks', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'feed.name' : 'Open SOCKS', - 'protocol.application' : 'socks5', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 1080, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-socks', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.source.sector' : 'Retail Trade', - 'feed.name' : 'Open SOCKS', - 'protocol.application' : 'socks4', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 1080, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssdp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ssdp.py deleted file mode 100644 index 0811f15eda..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssdp.py +++ /dev/null @@ -1,136 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ssdp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open SSDP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ssdp-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-ssdp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3.35, - 'extra.cache_control': 'max-age=100', - 'extra.header': 'HTTP/1.1 200 OK', - 'extra.host': 'node01.example.com', - 'extra.location': 'http://192.168.200.254:49152/description.xml', - 'extra.response_size': 325, - 'extra.search_target': 'upnp:rootdevice', - 'extra.sector': 'Government', - 'extra.server': 'Linux/2.6.26, UPnP/1.0, Portable SDK for UPnP devices/1.3.1', - 'extra.systime': 'Sun, 21 Aug 2022 09:51:13 GMT', - 'extra.tag': 'ssdp', - 'extra.unique_service_name': 'uuid:28802880-2880-1880-a880-001bc502f600::upnp:rootdevice', - 'feed.name': 'Open SSDP', - 'protocol.application': 'ssdp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 60194, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-ssdp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 2.71, - 'extra.cache_control': 'max-age = 1800', - 'extra.header': 'HTTP/1.1 200 OK', - 'extra.host': 'node02.example.com', - 'extra.location': 'http://95.160.216.14:52235/dmr/SamsungMRDesc.xml', - 'extra.response_size': 263, - 'extra.search_target': 'upnp:rootdevice', - 'extra.server': 'Linux/9.0 UPnP/1.0 PROTOTYPE/1.0', - 'extra.tag': 'ssdp', - 'extra.unique_service_name': 'uuid:f144ca92-6816-94b5-b95f-b58180834044::upnp:rootdevice', - 'feed.name': 'Open SSDP', - 'protocol.application': 'ssdp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 38732, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-ssdp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 4.79, - 'extra.cache_control': 'max-age=1800', - 'extra.header': 'HTTP/1.1 200 OK', - 'extra.host': 'node03.example.com', - 'extra.location': 'http://192.168.1.3:8008/ssdp/device-desc.xml', - 'extra.response_size': 465, - 'extra.search_target': 'upnp:rootdevice', - 'extra.sector': 'Government', - 'extra.server': 'Linux/3.10.79, UPnP/1.0, Portable SDK for UPnP ' - 'devices/1.6.18', - 'extra.systime': 'Sun, 03 Jan 2016 21:37:50 GMT', - 'extra.tag': 'ssdp', - 'extra.unique_service_name': 'uuid:62fa0fc8-079d-d00f-2e22-59b49fb488f9::upnp:rootdevice', - 'feed.name': 'Open SSDP', - 'protocol.application': 'ssdp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 57626, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssh.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ssh.py deleted file mode 100644 index a01383713b..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssh.py +++ /dev/null @@ -1,182 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ssh.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible SSH', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_ssh-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ssh', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.algorithm' : 'ecdsa-sha2-nistp256', - 'extra.available_ciphers' : 'chacha20-poly1305@openssh.com, aes128-ctr, aes192-ctr, aes256-ctr, aes128-gcm@openssh.com, aes256-gcm@openssh.com, aes128-cbc, aes192-cbc, aes256-cbc, blowfish-cbc, cast128-cbc, 3des-cbc', - 'extra.available_compression' : 'none, zlib@openssh.com', - 'extra.available_kex' : 'curve25519-sha256, curve25519-sha256@libssh.org, ecdh-sha2-nistp256, ecdh-sha2-nistp384, ecdh-sha2-nistp521, diffie-hellman-group-exchange-sha256, diffie-hellman-group16-sha512, diffie-hellman-group18-sha512, diffie-hellman-group-exchange-sha1, diffie-hellman-group14-sha256, diffie-hellman-group14-sha1, diffie-hellman-group1-sha1', - 'extra.available_mac' : 'umac-64-etm@openssh.com, umac-128-etm@openssh.com, hmac-sha2-256-etm@openssh.com, hmac-sha2-512-etm@openssh.com, hmac-sha1-etm@openssh.com, umac-64@openssh.com, umac-128@openssh.com, hmac-sha2-256, hmac-sha2-512, hmac-sha1', - 'extra.ecdsa_curve' : 'P-256', - 'extra.ecdsa_curve25519' : '1xx7ASut7BF4ED8b592bebZBMBKTCzOsmbH4cjwx/0U=', - 'extra.ecdsa_public_key_b' : 'WsY12Ko6k+ez671VdpiGvGUdBrDMU7D2O848PifSYEs=', - 'extra.ecdsa_public_key_gx' : 'axfR8uEsQkf4vOblY6RA8ncDfYEt6zOg9KE5RdiYwpY=', - 'extra.ecdsa_public_key_gy' : 'T+NC4v4af5uO5+tKfA+eFivOM1drMV7Oy7ZAaDe/UfU=', - 'extra.ecdsa_public_key_length' : '256', - 'extra.ecdsa_public_key_n' : '/////wAAAAD//////////7zm+q2nF56E87nKwvxjJVE=', - 'extra.ecdsa_public_key_p' : '/////wAAAAEAAAAAAAAAAAAAAAD///////////////8=', - 'extra.ecdsa_public_key_x' : 'NIQdotpy2HAHfSu0DjEmA3cb3NeQKabZaFX9OU0GsPY=', - 'extra.ecdsa_public_key_y' : '0fuNQAZX7XciX2YkqIHtK2dWLBYwVCCqvl//zoM42kI=', - 'extra.selected_cipher' : 'aes128-ctr', - 'extra.selected_compression' : 'none', - 'extra.selected_kex' : 'curve25519-sha256@libssh.org', - 'extra.selected_mac' : 'hmac-sha2-256', - 'extra.server_cookie' : 'bGjsifbPIDWT7tAu8BMjyg==', - 'extra.server_host_key' : 'AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDSEHaLacthwB30rtA4xJgN3G9zXkCmm2WhV/TlNBrD20fuNQAZX7XciX2YkqIHtK2dWLBYwVCCqvl//zoM42kI=', - 'extra.server_host_key_sha256' : 'a6e4e1c16ba25d51bcddc58a6e16797144575dd18d02d9dedf75093d2b15c557', - 'extra.server_signature_raw' : 'AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAABKAAAAIQCd+X/B/OEx+FrwJSlVecOvNMuS5w2vTRz0z4prM+5VBwAAACEArU60b9CHs/d5BgyaOd7vmFygTMK5SyL90bS8VIztX/4=', - 'extra.server_signature_value' : 'AAAAIQCd+X/B/OEx+FrwJSlVecOvNMuS5w2vTRz0z4prM+5VBwAAACEArU60b9CHs/d5BgyaOd7vmFygTMK5SyL90bS8VIztX/4=', - 'extra.serverid_raw' : 'SSH-2.0-OpenSSH_7.4', - 'extra.serverid_software' : 'OpenSSH_7.4', - 'extra.serverid_version' : '2.0', - 'extra.source.naics' : 454110, - 'extra.tag' : 'ssh', - 'extra.userauth_methods' : 'publickey', - 'feed.name' : 'Accessible SSH', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 16509, - 'source.geolocation.cc' : 'JP', - 'source.geolocation.city' : 'TOKYO', - 'source.geolocation.region' : 'TOKYO', - 'source.ip' : '18.179.0.0', - 'source.port' : 22, - 'source.reverse_dns' : 'ec2-18-179-0-0.ap-northeast-1.compute.amazonaws.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T02:20:37+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ssh', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.algorithm' : 'ssh-rsa', - 'extra.available_ciphers' : 'aes128-cbc, 3des-cbc, aes256-cbc, twofish256-cbc, twofish-cbc, twofish128-cbc, blowfish-cbc', - 'extra.available_compression' : 'none', - 'extra.available_kex' : 'diffie-hellman-group1-sha1', - 'extra.available_mac' : 'hmac-sha1-96, hmac-sha1, hmac-md5', - 'extra.device_vendor' : 'Arris', - 'extra.rsa_exponent' : '65537', - 'extra.rsa_length' : '1040', - 'extra.rsa_modulus' : 'g6tZBqqsHUFa8gjXKMg2waZy/6JLzNnfTZkdDo6a3E6Amdk2eVlGbdgRsDxNUCyWLJI4b+xsJX0OKoRKhw4N+cDK5Du4/S00Auylt8YrDi7lIpeBZjMT5KJerN/feOcwRqY8NJfwZiZ1A5brcWr5HXfOH6aRTMFzadr248VdUmFXPQ==', - 'extra.selected_cipher' : 'aes128-cbc', - 'extra.selected_compression' : 'none', - 'extra.selected_kex' : 'diffie-hellman-group1-sha1', - 'extra.selected_mac' : 'hmac-sha1', - 'extra.server_cookie' : 'Y4RQS9sdRgEFwNJKVP6bZg==', - 'extra.server_host_key' : 'AAAAB3NzaC1yc2EAAAADAQABAAAAgwCDq1kGqqwdQVryCNcoyDbBpnL/okvM2d9NmR0OjprcToCZ2TZ5WUZt2BGwPE1QLJYskjhv7GwlfQ4qhEqHDg35wMrkO7j9LTQC7KW3xisOLuUil4FmMxPkol6s39945zBGpjw0l/BmJnUDlutxavkdd84fppFMwXNp2vbjxV1SYVc9', - 'extra.server_host_key_sha256' : 'd53fedbfe92e631264629882b2e85bfd213ca4b07b824cd31f8de1fcb8d0ddcb', - 'extra.server_signature_raw' : 'AAAAB3NzaC1yc2EAAACCLQj+UTJEQqdb/p/c/19yVc63eo+rnedwXKjP6eNNxxijN2cFoOjVMeqT2QTBjyoN7yRWBU2EID+3y2jUYT8mCqmqfyUv1eEbiCfLVlUyQ0X/CY9I5DDb5l6yEjNkuH2xVNNV6R7GFRwyYKAsYzfy+i9o1OORlUh3tozkkPfA9z/NlA==', - 'extra.server_signature_value' : 'LQj+UTJEQqdb/p/c/19yVc63eo+rnedwXKjP6eNNxxijN2cFoOjVMeqT2QTBjyoN7yRWBU2EID+3y2jUYT8mCqmqfyUv1eEbiCfLVlUyQ0X/CY9I5DDb5l6yEjNkuH2xVNNV6R7GFRwyYKAsYzfy+i9o1OORlUh3tozkkPfA9z/NlA==', - 'extra.serverid_raw' : 'SSH-2.0-ARRIS_0.50', - 'extra.serverid_software' : 'ARRIS_0.50', - 'extra.serverid_version' : '2.0', - 'extra.tag' : 'ssh', - 'extra.userauth_methods' : 'publickey, password', - 'feed.name' : 'Accessible SSH', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 11976, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'MARSHALL', - 'source.geolocation.region' : 'TEXAS', - 'source.ip' : '170.10.0.0', - 'source.port' : 22, - 'source.reverse_dns' : '170-10-0-0.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T02:20:37+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ssh', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.algorithm' : 'ssh-rsa', - 'extra.available_ciphers' : 'aes128-cbc, 3des-cbc, aes192-cbc, aes256-cbc', - 'extra.available_compression' : 'none', - 'extra.available_kex' : 'diffie-hellman-group-exchange-sha1, diffie-hellman-group14-sha1, diffie-hellman-group1-sha1', - 'extra.available_mac' : 'hmac-sha1, hmac-sha1-96, hmac-md5, hmac-md5-96', - 'extra.device_sector' : 'enterprise', - 'extra.device_vendor' : 'Cisco', - 'extra.rsa_exponent' : '65537', - 'extra.rsa_length' : '4096', - 'extra.rsa_modulus' : 'yFVwcChoYt+YGm8BzWYugcZbNQRrQ1VWRYcL4U6SSkyoVeE9h5wxRu/hQaWHo3PdsB9Nuln/riRyKZypFUEZ5zlffMyl1uvE8/jp8E/GgUHSyPkGAwu8C8BkX/nDolxAJKTK6djiZnvhsEPe6AXHBMHbto/b3GABUNPngjzX8D63GYcFW9NJLf5qC1UsVkXbAzM0IjQ2X9s3pfhUCAJeXAn2i0gEGtUyF8vEjNdwdG655aXciKrpEEtM1L/zy/+gLH4YC13kAYI7NVyH+qi/mXbULLOQClA7iYK1g3Et58jWUIPwgLfF3SLC57bt2wp/lRgNTv4FBi0tWvRqBnf5UQK5ZjgzbW3bO+Ju4cWgH/4M4NCxSceh4cLm5lQs01xB5feSh2ByqA7wrVDoFJu81LoMVo4bCz30+lH2QsLwmNtUhlWLKBD4k09g4bgBa4jPj0/Nya3rBR4GQ6LG6ltFQotm8wCkgbv76YWqk20nQ6NMYZFvSQm981JFtoHv3vxq48VeHDV0QvV0P12BCFprRf4B0otIvSsHl+LDeUxJAf+Nbw78gzncjyfCbWtCPbwaJQ8CeqnTBzj5TluaFvN8goG5lCTWJGfjIrwAZXOokv9NOqmIiMJJx3s22OX6GHfJAzje2ALLDsAiXBub4iCOdGdTfVbBpFL+bGTK9qfa8vE=', - 'extra.selected_cipher' : 'aes128-cbc', - 'extra.selected_compression' : 'none', - 'extra.selected_kex' : 'diffie-hellman-group14-sha1', - 'extra.selected_mac' : 'hmac-sha1', - 'extra.server_cookie' : 'Z2fOfWsrLlh76Y0bOqa1cw==', - 'extra.server_host_key' : 'AAAAB3NzaC1yc2EAAAADAQABAAACAQDIVXBwKGhi35gabwHNZi6Bxls1BGtDVVZFhwvhTpJKTKhV4T2HnDFG7+FBpYejc92wH026Wf+uJHIpnKkVQRnnOV98zKXW68Tz+OnwT8aBQdLI+QYDC7wLwGRf+cOiXEAkpMrp2OJme+GwQ97oBccEwdu2j9vcYAFQ0+eCPNfwPrcZhwVb00kt/moLVSxWRdsDMzQiNDZf2zel+FQIAl5cCfaLSAQa1TIXy8SM13B0brnlpdyIqukQS0zUv/PL/6AsfhgLXeQBgjs1XIf6qL+ZdtQss5AKUDuJgrWDcS3nyNZQg/CAt8XdIsLntu3bCn+VGA1O/gUGLS1a9GoGd/lRArlmODNtbds74m7hxaAf/gzg0LFJx6HhwubmVCzTXEHl95KHYHKoDvCtUOgUm7zUugxWjhsLPfT6UfZCwvCY21SGVYsoEPiTT2DhuAFriM+PT83JresFHgZDosbqW0VCi2bzAKSBu/vphaqTbSdDo0xhkW9JCb3zUkW2ge/e/GrjxV4cNXRC9XQ/XYEIWmtF/gHSi0i9KweX4sN5TEkB/41vDvyDOdyPJ8Jta0I9vBolDwJ6qdMHOPlOW5oW83yCgbmUJNYkZ+MivABlc6iS/006qYiIwknHezbY5foYd8kDON7YAssOwCJcG5viII50Z1N9VsGkUv5sZMr2p9ry8Q==', - 'extra.server_host_key_sha256' : '06ff3cce443ed832927576d982b69d5a526d0e63334c72e87201deda61679406', - 'extra.server_signature_raw' : 'AAAAB3NzaC1yc2EAAAIAlrzL2DY9fVvwYg6CgB75uf2s8CLo+rL8Mp9tU1Ja3sDfBzj9QJjVDykupiy8s3usHfxMrHS2v3DhTiZjz/b5K6tVTgUBTXL94JfM4lwB+3EbLggPzKnlm1jQgnnU9c+tb7RX3IhBqU9Yj1gqxhErv9NFotgajQOOLgY0Ua5C0Ee+AIaMlLaNZe3LTejMsNUZMN5tl+sEmtutMHkGQsmjJxiJ3feF+Pys0I2+ojiiAfzqlMYar/5xOPl4Dj+HO+h91xVQ1/8nQRBc082fM7+ZJtDbRLtt4G8srlB5gew26jqfVASc/ui5gx4+BR9DG9VH8w+rJWBGfhOAaWqLFE2M3YuEWkjEmQMR1SQK1WFQ/oNiWJO2K5L3rk2LcAmyR6nQMtClVxYZ7CQOwa3uFL+JNXp9AhiiAtVaqhrEK81NJrJNh/+egTBl5STphxIShXd4KI9wyvkGlCIvNIMO94iXPVaWUXXbsGnU03+dsUkBzGf0eJ4DePInCk/RtunlSmOsjGld+rpS9g0VRxPrzbQRWuhpkgpV+CldyrI3C/rOxJRs2vSAKXocRsGwhqEKseAJzHXmiZ5ncsaGKoeB5lUkWLwcKjyok2tHVCDlzDUpE4aA/JHNEhT48det9RqtjC71yz8m0PeK2ySI/I+Qb7eBgevgduBmt+OUxgvfKi2UB6s=', - 'extra.server_signature_value' : 'lrzL2DY9fVvwYg6CgB75uf2s8CLo+rL8Mp9tU1Ja3sDfBzj9QJjVDykupiy8s3usHfxMrHS2v3DhTiZjz/b5K6tVTgUBTXL94JfM4lwB+3EbLggPzKnlm1jQgnnU9c+tb7RX3IhBqU9Yj1gqxhErv9NFotgajQOOLgY0Ua5C0Ee+AIaMlLaNZe3LTejMsNUZMN5tl+sEmtutMHkGQsmjJxiJ3feF+Pys0I2+ojiiAfzqlMYar/5xOPl4Dj+HO+h91xVQ1/8nQRBc082fM7+ZJtDbRLtt4G8srlB5gew26jqfVASc/ui5gx4+BR9DG9VH8w+rJWBGfhOAaWqLFE2M3YuEWkjEmQMR1SQK1WFQ/oNiWJO2K5L3rk2LcAmyR6nQMtClVxYZ7CQOwa3uFL+JNXp9AhiiAtVaqhrEK81NJrJNh/+egTBl5STphxIShXd4KI9wyvkGlCIvNIMO94iXPVaWUXXbsGnU03+dsUkBzGf0eJ4DePInCk/RtunlSmOsjGld+rpS9g0VRxPrzbQRWuhpkgpV+CldyrI3C/rOxJRs2vSAKXocRsGwhqEKseAJzHXmiZ5ncsaGKoeB5lUkWLwcKjyok2tHVCDlzDUpE4aA/JHNEhT48det9RqtjC71yz8m0PeK2ySI/I+Qb7eBgevgduBmt+OUxgvfKi2UB6s=', - 'extra.serverid_raw' : 'SSH-1.99-Cisco-1.25', - 'extra.serverid_software' : 'Cisco-1.25', - 'extra.serverid_version' : '1.99', - 'extra.source.naics' : 517311, - 'extra.tag' : 'ssh', - 'extra.userauth_methods' : 'publickey, keyboard-interactive, password', - 'feed.name' : 'Accessible SSH', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 33363, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'ORLANDO', - 'source.geolocation.region' : 'FLORIDA', - 'source.ip' : '72.17.0.0', - 'source.port' : 22, - 'source.reverse_dns' : '072-017-0-0.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T02:20:37+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl.py deleted file mode 100644 index f96c03e567..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl.py +++ /dev/null @@ -1,218 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ssl.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible SSL', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_ssl-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ssl', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.browser_error' : 'x509: unknown error', - 'extra.browser_trusted' : False, - 'extra.cert_expiration_date' : '2038-01-19 03:14:07', - 'extra.cert_expired' : False, - 'extra.cert_issue_date' : '2014-06-23 09:56:32', - 'extra.cert_length' : 1024, - 'extra.cert_serial_number' : '168CAE', - 'extra.cert_valid' : True, - 'extra.cipher_suite' : 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA', - 'extra.content_length' : 131, - 'extra.content_type' : 'text/html', - 'extra.freak_vulnerable' : False, - 'extra.handshake' : 'TLSv1.2', - 'extra.http_code' : 200, - 'extra.http_date' : '2022-01-10T00:01:44+00:00', - 'extra.http_reason' : 'OK', - 'extra.http_response_type' : 'HTTP/1.1', - 'extra.issuer_common_name' : 'support', - 'extra.issuer_country' : 'US', - 'extra.issuer_email_address' : 'support@fortinet.com', - 'extra.issuer_locality_name' : 'Sunnyvale', - 'extra.issuer_organization_name' : 'Fortinet', - 'extra.issuer_organization_unit_name' : 'Certificate Authority', - 'extra.issuer_state_or_province_name' : 'California', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : '99:45:1F:2E:AE:EB:88:91:27:43:33:79:FA:93:7D:CA', - 'extra.self_signed' : False, - 'extra.server_type' : 'xxxxxxxx-xxxxx', - 'extra.sha1_fingerprint' : '5A:3D:FF:06:F9:E9:25:37:57:F9:09:52:33:A4:85:15:24:2D:88:7F', - 'extra.sha256_fingerprint' : '35:AB:B6:76:2A:3D:17:B2:FB:40:45:1B:FC:0A:99:0A:6E:48:57:F7:30:0A:3B:B1:1A:E6:99:70:5B:7C:32:41', - 'extra.sha512_fingerprint' : '88:7B:16:DB:39:44:0C:47:0E:4A:8F:0B:C5:FB:4D:45:BC:93:5A:00:43:A1:D9:7F:05:1D:86:33:02:F8:FC:57:67:A6:1D:C0:FF:F7:D2:40:D8:9A:21:AE:4E:6D:DC:E7:FF:72:BF:13:CB:EE:A7:5F:CD:83:EA:8A:5E:FB:87:DD', - 'extra.signature_algorithm' : 'sha1WithRSAEncryption', - 'extra.source.naics' : 517311, - 'extra.ssl_poodle' : False, - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'FGT60D4614030700', - 'extra.subject_country' : 'US', - 'extra.subject_email_address' : 'support@fortinet.com', - 'extra.subject_locality_name' : 'Sunnyvale', - 'extra.subject_organization_name' : 'Fortinet', - 'extra.subject_organization_unit_name' : 'FortiGate', - 'extra.subject_state_or_province_name' : 'California', - 'extra.tag' : 'ssl,vpn', - 'feed.name' : 'Accessible SSL', - 'protocol.application': 'https', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 4181, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'MILWAUKEE', - 'source.geolocation.region' : 'WISCONSIN', - 'source.ip' : '96.60.0.0', - 'source.port' : 10443, - 'source.reverse_dns' : '96-60-0-0.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:42+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ssl', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.browser_error' : 'x509: unknown error', - 'extra.browser_trusted' : False, - 'extra.cert_expiration_date' : '2023-02-06 01:01:34', - 'extra.cert_expired' : False, - 'extra.cert_issue_date' : '2022-01-04 01:01:34', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : '36974C4C6B1B3785', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', - 'extra.content_type' : 'text/html; charset=UTF-8', - 'extra.freak_vulnerable' : False, - 'extra.handshake' : 'TLSv1.2', - 'extra.http_code' : 200, - 'extra.http_connection' : 'keep-alive', - 'extra.http_date' : '2022-01-10T00:01:44+00:00', - 'extra.http_reason' : 'OK', - 'extra.http_response_type' : 'HTTP/1.1', - 'extra.issuer_common_name' : '1078-btb-tbi-HungHa-61d39c6d5a7e2', - 'extra.issuer_organization_name' : 'pfSense webConfigurator Self-Signed Certificate', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : '16:93:9A:F4:35:7F:9A:85:45:71:91:C7:7C:80:88:00', - 'extra.self_signed' : True, - 'extra.server_type' : 'nginx', - 'extra.set_cookie' : 'PHPSESSID=e15bdfa5739c36877608eb4cf46cc388; path=/; secure; HttpO', - 'extra.sha1_fingerprint' : 'A9:00:BB:E1:54:4D:56:54:59:F1:B7:EA:F1:1A:D5:36:5C:63:90:8E', - 'extra.sha256_fingerprint' : '38:85:F0:44:1E:AD:84:B8:2F:43:68:BA:AC:EE:17:13:A4:BF:86:1D:48:75:7E:22:FA:08:4C:28:5F:AC:3E:5F', - 'extra.sha512_fingerprint' : 'AE:1B:4F:D1:E4:C0:35:9D:2A:4F:7A:37:B8:7B:11:9D:84:25:23:21:AB:EF:B2:0F:DC:C9:F2:A3:72:28:92:E1:74:72:FA:E1:09:6C:E1:F6:B6:E3:A7:61:1C:58:89:34:D7:06:5C:3D:0A:A7:F6:CC:8A:D6:24:D0:04:4C:03:02', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.source.naics' : 517311, - 'extra.ssl_poodle' : False, - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : '1078-btb-tbi-HungHa-61d39c6d5a7e2', - 'extra.subject_organization_name' : 'pfSense webConfigurator Self-Signed Certificate', - 'extra.tag' : 'ssl', - 'extra.transfer_encoding' : 'chunked', - 'feed.name' : 'Accessible SSL', - 'protocol.application': 'https', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 45899, - 'source.geolocation.cc' : 'VN', - 'source.geolocation.city' : 'THAI BINH', - 'source.geolocation.region' : 'THAI BINH', - 'source.ip' : '113.160.0.0', - 'source.port' : 10443, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:42+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ssl', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.browser_trusted' : True, - 'extra.cert_expiration_date' : '2022-11-06 15:30:28', - 'extra.cert_expired' : False, - 'extra.cert_issue_date' : '2021-10-07 15:30:28', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : '7B388364A24B88E77E5553B5C6748100', - 'extra.cert_valid' : True, - 'extra.cipher_suite' : 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA', - 'extra.content_length' : 131, - 'extra.content_type' : 'text/html', - 'extra.freak_vulnerable' : False, - 'extra.handshake' : 'TLSv1.2', - 'extra.http_code' : 200, - 'extra.http_date' : '2022-01-10T00:01:44+00:00', - 'extra.http_reason' : 'OK', - 'extra.http_response_type' : 'HTTP/1.1', - 'extra.issuer_common_name' : 'Entrust Certification Authority - L1K', - 'extra.issuer_country' : 'US', - 'extra.issuer_organization_name' : 'Entrust, Inc.', - 'extra.issuer_organization_unit_name' : '(c) 2012 Entrust, Inc. - for authorized use only', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'E7:34:BC:92:84:FA:39:DE:E1:46:6C:27:DA:5A:01:F4', - 'extra.self_signed' : False, - 'extra.server_type' : 'xxxxxxxx-xxxxx', - 'extra.sha1_fingerprint' : 'AD:19:B2:1C:CB:88:70:9B:DB:8E:7E:F5:65:50:13:D6:43:6C:BE:6E', - 'extra.sha256_fingerprint' : '9A:64:73:0B:8A:FA:DE:22:D4:6D:5A:C6:C4:6F:D4:A4:2A:28:FA:41:1E:FF:81:DC:D4:D9:00:FD:78:DF:C4:DD', - 'extra.sha512_fingerprint' : '9A:B7:BD:68:7D:F3:E7:C1:B7:D3:F4:2F:01:B6:C4:77:90:A3:2B:1E:C0:89:F5:08:EC:43:87:35:60:36:D4:87:61:AA:B8:A8:B3:8A:E9:F1:04:AA:5B:67:12:FF:63:D5:14:80:77:6E:8F:7D:C3:E2:3A:F3:13:DF:08:43:6C:B0', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.source.naics' : 454110, - 'extra.source.sector' : 'Retail Trade', - 'extra.ssl_poodle' : False, - 'extra.ssl_version' : 2, - 'extra.subject_country' : 'US', - 'extra.subject_locality_name' : 'Hanover', - 'extra.subject_organization_name' : 'Ciena Corporation', - 'extra.subject_state_or_province_name' : 'Maryland', - 'extra.tag' : 'ssl,vpn', - 'extra.validation_level' : 'OV', - 'feed.name' : 'Accessible SSL', - 'protocol.application': 'https', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 14618, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'ASHBURN', - 'source.geolocation.region' : 'VIRGINIA', - 'source.ip' : '34.224.0.0', - 'source.port' : 10443, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:42+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_freak.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_freak.py deleted file mode 100644 index 42221bda2b..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_freak.py +++ /dev/null @@ -1,136 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ssl_freak.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'SSL FREAK Vulnerable Servers', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ssl_freak-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'SSL FREAK Vulnerable Servers', - 'protocol.application': 'https', - "classification.identifier": "ssl-freak", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.browser_error": "x509: unknown error", - "extra.browser_trusted": False, - "extra.cert_expiration_date": "2032-05-05 00:01:19", - "extra.cert_expired": False, - "extra.cert_issue_date": "2012-05-10 00:01:19", - "extra.cert_length": 1024, - "extra.cert_serial_number": "4FAB054F", - "extra.cert_valid": True, - "extra.cipher_suite": "TLS_RSA_WITH_RC4_128_SHA", - "extra.content_type": "text/html", - "extra.freak_cipher_suite": "TLS_RSA_EXPORT_WITH_RC4_40_MD5", - "extra.freak_vulnerable": True, - "extra.handshake": "TLSv1.0", - "extra.http_code": 200, - "extra.http_date": "2018-04-23T13:25:26+00:00", - "extra.http_reason": "OK", - "extra.http_response_type": "HTTP/1.1", - "extra.issuer_common_name": "usg50_B0B2DC2FA69D", - "extra.key_algorithm": "rsaEncryption", - "extra.md5_fingerprint": "1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE", - "extra.self_signed": True, - "extra.sha1_fingerprint": "14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2", - "extra.sha256_fingerprint": "57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1", - "extra.sha512_fingerprint": "E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87", - "extra.signature_algorithm": "sha1WithRSAEncryption", - "extra.subject_common_name": "usg50_B0B2DC2FA69D", - "extra.tag": "ssl-freak", - "extra.transfer_encoding": "chunked", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 8447, - "source.geolocation.cc": "AT", - "source.geolocation.city": "VIENNA", - "source.geolocation.region": "WIEN", - "source.ip": "198.51.100.232", - "source.port": 443, - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2018-04-23T13:25:21+00:00" - }, - {'__type': 'Event', - 'feed.name': 'SSL FREAK Vulnerable Servers', - 'protocol.application': 'https', - "classification.identifier": "ssl-freak", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.browser_error": "x509: unknown error", - "extra.browser_trusted": False, - "extra.cert_expiration_date": "2029-12-27 00:00:53", - "extra.cert_expired": False, - "extra.cert_issue_date": "2010-01-01 00:00:53", - "extra.cert_length": 1024, - "extra.cert_serial_number": "4B3D3B35", - "extra.cert_valid": True, - "extra.cipher_suite": "TLS_RSA_WITH_RC4_128_SHA", - "extra.content_type": "text/html", - "extra.freak_cipher_suite": "TLS_RSA_EXPORT_WITH_RC4_40_MD5", - "extra.freak_vulnerable": True, - "extra.handshake": "TLSv1.0", - "extra.http_code": 200, - "extra.http_date": "2018-04-23T13:25:29+00:00", - "extra.http_reason": "OK", - "extra.http_response_type": "HTTP/1.1", - "extra.issuer_common_name": "usg20w_C86C870287EC", - "extra.key_algorithm": "rsaEncryption", - "extra.md5_fingerprint": "1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE", - "extra.self_signed": True, - "extra.sha1_fingerprint": "14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2", - "extra.sha256_fingerprint": "57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1", - "extra.sha512_fingerprint": "E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87", - "extra.signature_algorithm": "sha1WithRSAEncryption", - "extra.subject_common_name": "usg20w_C86C870287EC", - "extra.tag": "ssl-freak", - "extra.transfer_encoding": "chunked", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 12577, - "source.geolocation.cc": "AT", - "source.geolocation.city": "BADEN", - "source.geolocation.region": "NIEDEROSTERREICH", - "source.ip": "198.51.100.224", - "source.port": 443, - "source.reverse_dns": "198-51-100-224.example.net", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2018-04-23T13:25:26+00:00" - }] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_poodle.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_poodle.py deleted file mode 100644 index 41535e67a4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_poodle.py +++ /dev/null @@ -1,91 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_ssl_poodle.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'SSL POODLE Vulnerable Servers', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ssl_poodle-test-geo.csv", - } -EVENTS = [{'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'ssl-poodle', - 'extra.browser_error': 'x509: unknown error', - 'extra.browser_trusted': False, - 'extra.cert_expiration_date': '2034-06-20 00:00:42', - 'extra.cert_expired': False, - 'extra.cert_issue_date': '2014-06-25 00:00:42', - 'extra.cert_length': 1024, - 'extra.cert_serial_number': '53AA112A', - 'extra.cert_valid': True, - 'extra.cipher_suite': 'TLS_RSA_WITH_RC4_128_SHA', - 'extra.content_type': 'text/html', - 'extra.handshake': 'TLSv1.0', - 'extra.http_code': 200, - 'extra.http_date': '2018-08-08T00:51:44+00:00', - 'extra.http_reason': 'OK', - 'extra.http_response_type': 'HTTP/1.1', - 'extra.issuer_common_name': 'usg20_107BEF394BA5', - 'extra.key_algorithm': 'rsaEncryption', - 'extra.md5_fingerprint': '33:E3:61:3F:5D:AA:96:99:38:A5:D6:F1:11:C7:ED:FC', - 'extra.self_signed': True, - 'extra.sha1_fingerprint': '04:FA:DE:1D:BD:4A:05:25:61:FB:F3:D6:64:74:66:44:01:22:D7:C3', - 'extra.sha256_fingerprint': '16:25:9F:C7:A1:8D:64:1F:D9:25:42:BF:87:5C:4F:F3:63:14:97:21:EC:B6:67:10:F2:CA:52:37:C9:FE:49:2E', - 'extra.sha512_fingerprint': '0B:2D:48:8C:4B:55:8B:F3:AB:F8:45:ED:E0:A0:63:F4:84:2F:4C:19:DC:A8:6F:7D:6A:AF:61:D7:98:AA:58:0F:CB:CA:87:D2:C3:0B:C5:DF:49:A7:84:7C:47:58:89:7D:92:B6:7B:98:7D:B1:64:4B:DC:DD:BE:9D:11:2A:D1:AE', - 'extra.signature_algorithm': 'sha1WithRSAEncryption', - 'extra.ssl_poodle': True, - 'extra.ssl_version': 2, - 'extra.subject_common_name': 'usg20_107BEF394BA5', - 'extra.tag': 'ssl-poodle', - 'extra.transfer_encoding': 'chunked', - 'feed.name': 'SSL POODLE Vulnerable Servers', - 'protocol.application': 'https', - 'source.asn': 65540, - 'source.geolocation.cc': 'AT', - 'source.geolocation.city': 'VIENNA', - 'source.geolocation.region': 'WIEN', - 'source.ip': '203.0.113.85', - 'source.port': 8443, - 'source.reverse_dns': 'example.com', - 'time.source': '2018-08-08T00:51:42+00:00', - "time.observation": "2015-01-01T00:00:00+00:00", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - '__type': 'Event', - }, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_stun.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_stun.py deleted file mode 100644 index 7fd5f6ec21..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_stun.py +++ /dev/null @@ -1,146 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_stun.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible-Session-Traversal-Utilities-for-NAT', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_stun-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-stun', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'extra.amplification': 5.4, - 'extra.fingerprint': '0xfaedd06e', - 'extra.magic_cookie': '2112a442', - 'extra.mapped_address': '192.168.0.1', - 'extra.mapped_family': '01', - 'extra.mapped_port': 3243, - 'extra.message_length': 88, - 'extra.message_type': '0101', - 'extra.response_size': 108, - 'extra.software': "Coturn-4.5.1.1 'dan Eider'", - 'extra.tag': 'stun', - 'extra.transaction_id': '000000000000000000000000', - 'extra.xor_mapped_address': '192.168.0.1', - 'extra.xor_mapped_family': '01', - 'extra.xor_mapped_port': 3243, - 'feed.name': 'Accessible-Session-Traversal-Utilities-for-NAT', - 'protocol.application': 'session traversal utilities for nat', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 3478, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-stun', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'extra.amplification': 5.4, - 'extra.fingerprint': '0x21128641', - 'extra.magic_cookie': '2112a442', - 'extra.mapped_address': '51.77.39.195', - 'extra.mapped_family': '01', - 'extra.mapped_port': 45877, - 'extra.message_length': 88, - 'extra.message_type': '0101', - 'extra.response_size': 108, - 'extra.software': "Coturn-4.5.1.1 'dan Eider'", - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'stun', - 'extra.transaction_id': '000000000000000000000000', - 'extra.xor_mapped_address': '192.168.0.2', - 'extra.xor_mapped_family': '01', - 'extra.xor_mapped_port': 45877, - 'feed.name': 'Accessible-Session-Traversal-Utilities-for-NAT', - 'protocol.application': 'session traversal utilities for nat', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 3478, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-stun', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'extra.amplification': 4.8, - 'extra.magic_cookie': '2112a442', - 'extra.mapped_address': '192.168.0.3', - 'extra.mapped_family': '01', - 'extra.mapped_port': 16321, - 'extra.message_length': 76, - 'extra.message_type': '0101', - 'extra.response_size': 96, - 'extra.software': "ApolloProxy-1.20.1.28 'sunflower'", - 'extra.tag': 'stun', - 'extra.transaction_id': '000000000000000000000000', - 'extra.xor_mapped_address': '188.68.240.32', - 'extra.xor_mapped_family': '01', - 'extra.xor_mapped_port': 16321, - 'feed.name': 'Accessible-Session-Traversal-Utilities-for-NAT', - 'protocol.application': 'session traversal utilities for nat', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 3478, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_synfulknock.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_synfulknock.py deleted file mode 100644 index 9b7e1fd3d9..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_synfulknock.py +++ /dev/null @@ -1,117 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_synfulknock.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'SYNful Knock', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_synfulknock-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-synfulknock', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.ack_number' : 791102, - 'extra.raw_packet' : '3cfdfec601e4700f6a9a2000080045000034c3780000f706789442099555b869f7ee0050b20800000000000c123e8012200002aa0000020405b40101040201030305', - 'extra.tag' : 'synfulknock', - 'extra.tcp_flags' : '4608', - 'extra.window_size' : 8192, - 'feed.name' : 'SYNful Knock', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 18885, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'JERSEY CITY', - 'source.geolocation.region' : 'NEW JERSEY', - 'source.ip' : '66.9.0.0', - 'source.port' : 80, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T09:18:23+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-synfulknock', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.ack_number' : 791102, - 'extra.raw_packet' : '90e2baaf0b84700f6a9a200008004500003434100000f2064382d58337d2b8698b720050916200000000000c123e8012200059d50000020405b40101040201030305', - 'extra.tag' : 'synfulknock', - 'extra.tcp_flags' : '4608', - 'extra.window_size' : 8192, - 'feed.name' : 'SYNful Knock', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 35805, - 'source.geolocation.cc' : 'GE', - 'source.geolocation.city' : 'TBILISI', - 'source.geolocation.region' : 'TBILISI', - 'source.ip' : '213.131.0.0', - 'source.port' : 80, - 'source.reverse_dns' : 'host-213-131-55-210-customer.wanex.net', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T09:19:17+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-synfulknock', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.ack_number' : 791102, - 'extra.raw_packet' : '90e2bab9cfd4700f6a9a20000800450000340f1d0000ea068bdad5b2e6914a522f360050eb5200000000000c123e801220001b4a0000020405b40101040201030305', - 'extra.tag' : 'synfulknock', - 'extra.tcp_flags' : '4608', - 'extra.window_size' : 8192, - 'feed.name' : 'SYNful Knock', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 29256, - 'source.geolocation.cc' : 'SY', - 'source.geolocation.city' : 'DAMASCUS', - 'source.geolocation.region' : 'DIMASHQ', - 'source.ip' : '213.178.0.0', - 'source.port' : 80, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T09:27:39+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_telnet.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_telnet.py deleted file mode 100644 index 66408db4c5..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_telnet.py +++ /dev/null @@ -1,87 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_telnet.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible Telnet', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_telnet-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Accessible Telnet', - "classification.identifier": "open-telnet", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.banner": "|MikroTik v6.5|Login:", - "extra.tag": "telnet-alt", - "protocol.application": "telnet", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 20255, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.145", - "source.port": 5678, - "source.reverse_dns": "example.local", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T12:27:34+00:00" - }, - {'__type': 'Event', - 'feed.name': 'Accessible Telnet', - "classification.identifier": "open-telnet", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.banner": "|MikroTik v6.45.3 (stable)|Login:", - "extra.tag": "telnet-alt", - "protocol.application": "telnet", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 20255, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.145", - "source.port": 5678, - "source.reverse_dns": "example.local", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T12:27:40+00:00" - }] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_tftp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_tftp.py deleted file mode 100644 index 3cf3688f97..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_tftp.py +++ /dev/null @@ -1,121 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_tftp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open TFTP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2019-03-25T00:00:00+00:00", - "extra.file_name": "2019-03-25-scan_tftp-test-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-tftp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 1.57, - 'extra.error': 'Not defined', - 'extra.errormessage': 'Get not supported', - 'extra.opcode': '5', - 'extra.size': 22, - 'extra.tag': 'tftp', - 'feed.name': 'Open TFTP', - 'protocol.application': 'tftp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 35067, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-tftp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 1.36, - 'extra.error': 'File not found', - 'extra.errorcode': '1', - 'extra.errormessage': 'File not found', - 'extra.opcode': '5', - 'extra.size': 19, - 'extra.tag': 'tftp', - 'feed.name': 'Open TFTP', - 'protocol.application': 'tftp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 56709, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-tftp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 1.5, - 'extra.error': 'Access violation', - 'extra.errorcode': '2', - 'extra.errormessage': 'Access violation', - 'extra.opcode': '5', - 'extra.size': 21, - 'extra.tag': 'tftp', - 'feed.name': 'Open TFTP', - 'protocol.application': 'tftp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 32785, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ubiquiti.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ubiquiti.py deleted file mode 100644 index 396bff1e33..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ubiquiti.py +++ /dev/null @@ -1,124 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ubiquiti.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open Ubiquiti', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2018-03-04T00:00:00+00:00", - "extra.file_name": "2019-03-25-scan_ubiquiti-test-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'accessible-ubiquiti-discovery-service', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 37.0, - 'extra.essid': 'Kachine-Meta-Lidia-Tereixa', - 'extra.firmwarerev': 'XS5.ar2313.v3.5.4494.091109.1459', - 'extra.mac_address': '00156db98c3a', - 'extra.model': 'NS5', - 'extra.radio_name': 'kachine.meta.lidia.tereixa', - 'extra.response_size': 148, - 'extra.tag': 'ubiquiti,iot', - 'feed.name': 'Open Ubiquiti', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 10001, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-ubiquiti-discovery-service', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 39.0, - 'extra.essid': 'Adana-Mason-Lanikai-Ozaner', - 'extra.firmwarerev': 'XM.ar7240.v5.6.3.28591.151130.1749', - 'extra.mac_address': '00156d7c9188', - 'extra.model': 'LM5', - 'extra.model_full': 'NanoStation Loco M5', - 'extra.radio_name': 'adana.mason.lanikai.ozaner', - 'extra.response_size': 156, - 'extra.tag': 'ubiquiti,iot', - 'feed.name': 'Open Ubiquiti', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 10001, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-ubiquiti-discovery-service', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 36.25, - 'extra.essid': 'Tailynn-Kadija-Noreen-Dinkar', - 'extra.firmwarerev': 'XW.ar934x.v5.6.5.29033.160515.2108', - 'extra.mac_address': '0418d6000fd5', - 'extra.model': 'P2B-400', - 'extra.model_full': 'PowerBeam M2 400', - 'extra.radio_name': 'tailynn.kadija.noreen.dinkar', - 'extra.response_size': 145, - 'extra.tag': 'ubiquiti,iot', - 'feed.name': 'Open Ubiquiti', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 10001, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_vnc.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_vnc.py deleted file mode 100644 index 457ec4425a..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_vnc.py +++ /dev/null @@ -1,86 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_vnc.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible VNC', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_vnc-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Accessible VNC', - "classification.identifier": "open-vnc", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.banner": "RFB 003.889", - "extra.product": "Apple remote desktop vnc", - "protocol.application": "vnc", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 5678, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.53", - "source.port": 5678, - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T14:51:44+00:00" - }, - {'__type': 'Event', - 'feed.name': 'Accessible VNC', - "classification.identifier": "open-vnc", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.banner": "RFB 005.000", - "extra.naics": 517311, - "extra.product": "RealVNC Enterprise v5.3 or later", - "protocol.application": "vnc", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 5678, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.112", - "source.port": 5678, - "source.reverse_dns": "localhost.localdomain", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T14:51:44+00:00"}] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ws_discovery.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ws_discovery.py deleted file mode 100644 index 41ab55e584..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ws_discovery.py +++ /dev/null @@ -1,119 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ws_discovery.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible-WS-Discovery-Service', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_ws_discovery-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-ws-discovery', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 164.83, - 'extra.error': 'Validation constraint violation: SOAP message expected', - 'extra.raw_response': 'c2FtcGxlIHJlc3BvbnNlIGRhdGEK', - 'extra.response_size': 989, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'ws-discovery', - 'feed.name': 'Accessible-WS-Discovery-Service', - 'protocol.application': 'ws-discovery', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 3702, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-ws-discovery', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 183.6, - 'extra.error': 'Validation constraint violation: missing root element', - 'extra.raw_response': 'c2FtcGxlIHJlc3BvbnNlIGRhdGEK', - 'extra.response_size': 918, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'ws-discovery', - 'feed.name': 'Accessible-WS-Discovery-Service', - 'protocol.application': 'ws-discovery', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 3702, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-ws-discovery', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 197.8, - 'extra.error': 'Validation constraint violation: SOAP message expected', - 'extra.raw_response': 'c2FtcGxlIHJlc3BvbnNlIGRhdGEK', - 'extra.response_size': 989, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'ws-discovery', - 'feed.name': 'Accessible-WS-Discovery-Service', - 'protocol.application': 'ws-discovery', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 3702, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_xdmcp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_xdmcp.py deleted file mode 100644 index d17482e715..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_xdmcp.py +++ /dev/null @@ -1,117 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Sebastian Wagner -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_xdmcp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "ShadowServer XDMCP", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_xdmcp-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-xdmcp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 6.29, - 'extra.opcode': 'Willing', - 'extra.reported_hostname': 'node01.example.com', - 'extra.size': 44, - 'extra.status': 'Linux 3.0.101-100-default', - 'extra.tag': 'xdmcp', - 'feed.name': 'ShadowServer XDMCP', - 'protocol.application': 'xdmcp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 177, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-xdmcp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 6.86, - 'extra.opcode': 'Willing', - 'extra.reported_hostname': 'node02.example.com', - 'extra.size': 48, - 'extra.status': 'Linux 2.6.9-103.ELsmp', - 'extra.tag': 'xdmcp', - 'feed.name': 'ShadowServer XDMCP', - 'protocol.application': 'xdmcp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 47074, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-xdmcp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 6.57, - 'extra.opcode': 'Willing', - 'extra.reported_hostname': 'node03.example.com', - 'extra.size': 46, - 'extra.status': '1 user, load: 6,5, 6,6, 6,6', - 'extra.tag': 'xdmcp', - 'feed.name': 'ShadowServer XDMCP', - 'protocol.application': 'xdmcp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 177, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_special.py b/intelmq/tests/bots/parsers/shadowserver/test_special.py deleted file mode 100644 index abad86cacc..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_special.py +++ /dev/null @@ -1,106 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/special.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Special', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-special-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'special', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.status' : 'likely compromised', - 'feed.name' : 'Special', - 'malware.name' : 'cyclops-blink', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'special', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.status' : 'likely compromised', - 'feed.name' : 'Special', - 'malware.name' : 'cyclops-blink', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'special', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.source.sector' : 'Professional, Scientific, and Technical Services', - 'extra.status' : 'likely compromised', - 'feed.name' : 'Special', - 'malware.name' : 'cyclops-blink', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_testdata.py b/intelmq/tests/bots/parsers/shadowserver/test_testdata.py deleted file mode 100644 index 19cbdd7d77..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_testdata.py +++ /dev/null @@ -1,81 +0,0 @@ -# SPDX-FileCopyrightText: 2017 Sebastian Wagner -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import csv -import json -import os -import os.path -import unittest -import pathlib - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot -from intelmq.bots.parsers.shadowserver.parser_json import ShadowserverJSONParserBot - -def csvtojson(csvfile): - datalist = [] - - with open(csvfile) as fop: - reader = csv.DictReader(fop, restval="") - - for row in reader: - datalist.append(row) - - return json.dumps(datalist, indent=4) - -CSVREPORTS = {} -JSONREPORTS = {} -testdata = pathlib.Path(__file__).parent / 'testdata' -for filename in testdata.glob('*.csv'): - EXAMPLE_FILE = filename.read_text() - shortname = filename.stem - CSVREPORTS[shortname] = {"raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": f"2019-01-01-{shortname}-test-test.csv", - } - JSONREPORTS[shortname] = {"raw": utils.base64_encode(csvtojson(filename)), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": f"2019-01-01-{shortname}-test-test.json", - } - - -def generate_feed_function(feedname, reports): - def test_feed(self): - """ Test if no errors happen for feed %s. """ % feedname - self.input_message = reports[feedname] - self.run_bot() - return test_feed - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - -class TestShadowserverJSONParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverJSONParserBot - -for key in CSVREPORTS: - setattr(TestShadowserverParserBot, 'test_feed_%s' % key, generate_feed_function(key, CSVREPORTS)) -for key in JSONREPORTS: - setattr(TestShadowserverJSONParserBot, 'test_feed_%s' % key, generate_feed_function(key, JSONREPORTS)) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv deleted file mode 100644 index cfadcbb2d2..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","hostname","source","reason","asn","geo","region","city","naics","sic","sector","tag" -"2019-09-04 07:00:19","198.123.245.134",host.local,"Alien Vault","Malicious Host AA",5678,"XX","LOCATION","LOCATION",517311,0,0, -"2019-09-04 07:00:19","198.123.245.171",,"Alien Vault","Malicious Host AA",5678,"XX","LOCATION","LOCATION",517311,0,, -"2019-09-04 07:00:19","198.123.245.0/24",,"Alien Vault","Malicious Host AA",5678,"XX","LOCATION","LOCATION",517311,0,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv.license deleted file mode 100644 index 476908eebe..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2020 Thomas Hungenberg -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/botnet_drone.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/botnet_drone.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/botnet_drone.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/caida_ip_spoofer.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/caida_ip_spoofer.csv.license deleted file mode 100644 index 456b03316c..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/caida_ip_spoofer.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2020 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv deleted file mode 100644 index 117dd65607..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","port","hostname","tag","application","asn","geo","region","city","url","http_host","category","system","detected_since","server","redirect_target","naics","sic","sector","cc_url","family" -"2017-01-16 00:43:48","203.0.113.1",80,"example.com","hacked-webserver-stealrat-t1","http",64496,"AT","WIEN","VIENNA","/header.php","example.com","spam","WINNT","2015-05-09 05:51:12","Microsoft-IIS/7.5",,0,0,,, -"2018-04-09 15:43:41","203.0.113.1","80","","phishing","http","64496","AT","STEIERMARK","GRAZ","/","example.com","phishing","","","","","0","0","",, -"2022-02-07 21:52:29","66.249.0.0",,"66-249-0-0.example.com","magecart",,1234,"US","CALIFORNIA","MOUNTAIN VIEW",,,"stealer",,,,,519130,,"Communications, Service Provider, and Hosting Service","https://lolfree.pw/ads.txt", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv.license deleted file mode 100644 index f8f131c2ce..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/darknet.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/darknet.csv.license deleted file mode 100644 index f8f131c2ce..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/darknet.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/ddos_amplification.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/ddos_amplification.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/ddos_amplification.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv deleted file mode 100644 index 22cfdd69e6..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","device_vendor","device_type","device_model" -"2022-01-10 00:01:42","88.84.0.0","tcp",10443,,"ssl,vpn",2116,"NO","TROMS OG FINNMARK","TROMVIK",517311,,,"Fortinet","firewall","FortiGate" -"2022-01-10 00:01:42","170.231.0.0","tcp",10443,,"ssl,vpn",27843,"PE","METROPOLITANA DE LIMA","LIMA",,,,"Fortinet","firewall","FortiGate" -"2022-01-10 00:01:42","96.60.0.0","tcp",10443,"96-60-66-218.example.com","ssl,vpn",4181,"US","WISCONSIN","MILWAUKEE",517311,,,"Fortinet","firewall","FortiGate" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/drone_brute_force.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/drone_brute_force.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/drone_brute_force.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv deleted file mode 100644 index 3114c26b15..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","domain_source","public_source","infection","family","tag","application","version","event_id","dst_network","dst_netmask","attack","duration","attack_src_ip","attack_src_port","domain","domain_transaction_id","gcip","http_method","http_path","http_postdata","http_usessl","ip_header_ack","ip_header_acknum","ip_header_dont_fragment","ip_header_fin","ip_header_identity","ip_header_psh","ip_header_rst","ip_header_seqnum","ip_header_syn","ip_header_tos","ip_header_ttl","ip_header_urg","number_of_connections","packet_length","packet_randomized","http_agent" -"2010-02-10 00:00:00",tcp,192.168.0.1,38055,64512,ZZ,Region,City,node01.example.com,0,,,,,172.16.0.1,443,65534,ZZ,Region,City,node01.example.net,0,"",,,ddos-participant,,,https,,,,,,,,,www.example.com,,,GET,/??=GovpfOoaWYlk,,,,,,,,,,,,,,,,,, -"2010-02-10 00:00:01",udp,192.168.0.2,53,64512,ZZ,Region,City,node02.example.com,0,,,,,172.16.0.2,53,65534,ZZ,Region,City,node02.example.net,0,"",,,ddos-participant,,,dns,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -"2010-02-10 00:00:02",udp,192.168.0.3,53,64512,ZZ,Region,City,node03.example.com,0,,Microsoft,email,Exchange,172.16.0.3,53,65534,ZZ,Region,City,node03.example.net,0,"",,,ddos-participant,,,dns,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv.license deleted file mode 100644 index 9f58c89ef0..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2023 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv deleted file mode 100644 index 17ff15ee6c..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv +++ /dev/null @@ -1,7 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","service","start_time","end_time","client_version","username","password","payload_url","payload_md5" -"2021-03-27 00:00:00","tcp","141.98.1.2",30123,209588,"NL","NOORD-HOLLAND","AMSTERDAM",,,,,,,"162.250.1.2",22,26832,"CA","QUEBEC","MONTREAL",,,,"CAPRICA-EU","ssh-brute-force",,,"ssh",,,,"2021-03-27T00:00:00.521730Z","2021-03-27T00:00:01.710968Z","b'SSH-2.0-Go'",,,, -"2021-03-27 00:00:00","tcp","5.188.3.4",55690,57172,"NL","NOORD-HOLLAND","AMSTERDAM",,518210,,,,,"162.250.3.4",22,26832,"CA","QUEBEC","MONTREAL",,,,"CAPRICA-EU","ssh-brute-force",,,"ssh",,,,"2021-03-27T00:00:00.520927Z","2021-03-27T00:00:01.670993Z","b'SSH-2.0-Go'",,,, -"2021-03-27 00:00:00","tcp","45.14.5.6",38636,44220,"RO","BIHOR","ORADEA",,,,,,,"82.118.5.6",23,204957,"PL","MAZOWIECKIE","WARSAW",,,,"CAPRICA-EU","telnet-brute-force",,,"telnet",,,,"2021-03-27T00:00:00.781774Z","2021-03-27T00:00:00.857244Z",,,,, -"2021-03-27 00:00:00","tcp","5.188.6.7",56385,49453,"NL","NOORD-HOLLAND","AMSTERDAM",,518210,,,,,"102.16.6.7",22,37054,"MG","ANTANANARIVO","ANTANANARIVO",,,"Communications, Service Provider, and Hosting Service","CAPRICA-EU","ssh-brute-force",,,"ssh",,,,"2021-03-27T00:00:00.163870Z","2021-03-27T00:00:02.896640Z","b'SSH-2.0-Go'",,,, -"2021-03-27 00:00:00","tcp","45.14.7.8",35802,44220,"RO","BIHOR","ORADEA",,,,,,,"82.118.7.8",23,204957,"PL","MAZOWIECKIE","WARSAW",,,,"CAPRICA-EU","telnet-brute-force",,,"telnet",,,,"2021-03-27T00:00:00.781272Z","2021-03-27T00:00:00.856606Z",,,,, -"2021-03-27 00:00:00","tcp","5.188.9.10",33289,49453,"NL","NOORD-HOLLAND","AMSTERDAM",,518210,,,,,"60.234.9.10",22,9790,"NZ","WELLINGTON","LOWER HUTT",,,"Communications, Service Provider, and Hosting Service","CAPRICA-EU","ssh-brute-force",,,"ssh",,,,"2021-03-27T00:00:00.044871Z","2021-03-27T00:00:00.077322Z","b'SSH-2.0-Go'",,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv.license deleted file mode 100644 index 8b9580cf15..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv deleted file mode 100644 index dc78c1c1aa..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv +++ /dev/null @@ -1,9 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","count" -"2021-03-07 00:00:00","tcp","61.3.1.2",4717,9829,"IN","KERALA","CHENGANNUR",,518210,,,,,,23,,,,,,,,,"mirai",,"mirai",,,, -"2021-03-07 00:00:00","tcp","211.218.3.4",4405,4766,"KR","GANGWON-DO","PYEONGCHANG-EUP",,517311,,,,,,23,,,,,,,,,"mirai",,"mirai",,,, -"2021-03-07 00:00:00","tcp","45.225.5.6",59777,266915,"BR","BAHIA","VITORIA DA CONQUISTA","static-45-225-x-x.example.net",,,,,,,23,,,,,,,,,"mirai",,"mirai",,,, -"2021-03-07 00:00:00","tcp","125.122.7.8",8460,4134,"CN","ZHEJIANG SHENG","HANGZHOU",,517311,,,,,,23,,,,,,,,,"mirai",,"mirai",,,, -"2021-03-07 00:00:00","tcp","219.77.9.10",21867,4760,"HK","HONG KONG","HONG KONG","n219077092196.example.com",517311,,,,,,5555,,,,,,,,,"mirai",,"mirai",,,, -"2021-03-07 00:00:00","tcp","24.137.11.12",4680,14638,"PR","PUERTO RICO","SAN JUAN","dynamic.libertypr.net",,,,,,,5555,,,,,,,,,"mirai",,"mirai",,,, -"2021-03-07 00:00:00","tcp","119.182.13.14",13175,4837,"CN","SHANDONG SHENG","JINING",,517311,,,,,,23,,,,,,,,,"mirai",,"mirai",,,, -"2021-03-07 00:00:00","tcp","27.198.15.16",56133,4837,"CN","SHANDONG SHENG","JINAN",,517311,,,,,,23,,,,,,,,,"mirai",,"mirai",,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv.license deleted file mode 100644 index f4e16ec677..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Birger Schacht -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv deleted file mode 100644 index f41cb508f7..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","domain_source","public_source","infection","family","tag","application","version","event_id","dst_network","dst_netmask","attack","duration","attack_src_ip","attack_src_port","domain","domain_transaction_id","gcip","http_method","http_path","http_postdata","http_usessl","ip_header_ack","ip_header_acknum","ip_header_dont_fragment","ip_header_fin","ip_header_identity","ip_header_psh","ip_header_rst","ip_header_seqnum","ip_header_syn","ip_header_tos","ip_header_ttl","ip_header_urg","number_of_connections","packet_length","packet_randomized","http_agent" -"2010-02-10 00:00:00",,192.168.0.1,61234,64512,ZZ,Region,City,node01.example.com,0,"Communications, Service Provider, and Hosting Service",,,,172.16.0.1,88,65534,ZZ,Region,City,node01.example.net,0,,,,ddos,mirai,mirai,mirai,,,121.12.110.28/32,32,atk10,30,,,,,,,,,,,,,,,,,,,,,,,1440, -"2010-02-10 00:00:01",,192.168.0.2,61234,64512,ZZ,Region,City,node02.example.com,0,"Communications, Service Provider, and Hosting Service",,,,172.16.0.2,80,65534,ZZ,Region,City,node02.example.net,0,,,,ddos,mirai,mirai,mirai,,,180.97.183.94/32,32,atk10,30,,,,,,,,,,,,,,,,,,,,,,,1440, -"2010-02-10 00:00:02",,192.168.0.3,6379,64512,ZZ,Region,City,node03.example.com,0,"Communications, Service Provider, and Hosting Service",,,,172.16.0.3,,65534,ZZ,Region,City,node03.example.net,0,"Communications, Service Provider, and Hosting Service",,,ddos,mirai,mirai,mirai,,,104.237.138.135/32,32,atk7,10,,,,,,,,,,,,,,,,,,,,,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv deleted file mode 100644 index a7d0bc4f1d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv +++ /dev/null @@ -1,6 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","request","count","bytes","end_time","duration","avg_pps","max_pps" -"2021-03-28 00:00:02",,"107.141.1.2",,7018,"US","CALIFORNIA","VISALIA","107-141-x-x.lightspeed.frsnca.sbcglobal.net",517311,"Communications, Service Provider, and Hosting Service",,,,,389,,,,,,,,"CISPA","ddos-amplification",,,,,,,,,"2021-03-28 00:20:22",,, -"2021-03-28 00:00:02",,"74.59.3.4",,5769,"CA","QUEBEC","CHICOUTIMI","modemcablex-x-59-74.mc.videotron.ca",517311,"Communications, Service Provider, and Hosting Service",,,,,389,,,,,,,,"CISPA","ddos-amplification",,,,,,,,,"2021-03-28 00:13:50",,, -"2021-03-28 00:00:02",,"65.131.5.6",,209,"US","WYOMING","CASPER","65-131-x-x.chyn.qwest.net",517311,"Communications, Service Provider, and Hosting Service",,,,,389,,,,,,,,"CISPA","ddos-amplification",,,,,,,,,,,, -"2021-03-28 00:00:02",,"104.162.7.8",,12271,"US","NEW YORK","KINGSTON","cpe-104-162-x-x.hvc.res.rr.com",517311,"Communications, Service Provider, and Hosting Service",,,,,389,,,,,,,,"CISPA","ddos-amplification",,,,,,,,,,,, -"2021-03-28 00:00:02",,"37.120.178.9.10",,197540,"DE","NIEDERSACHSEN","GIFHORN","v22020111328131649.ultrasrv.de",,,,,,,389,,,,,,,,"CISPA","ddos-amplification",,,,,,,,,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv.license deleted file mode 100644 index 8b9580cf15..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv deleted file mode 100644 index 0e5b1e5e9c..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","protocol","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","device_vendor","device_type","device_model","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","domain_source","public_source","infection","family","tag","application","version","event_id","dst_network","dst_netmask","attack","duration","attack_src_ip","attack_src_port","domain","domain_transaction_id","gcip","http_method","http_path","http_postdata","http_usessl","ip_header_ack","ip_header_acknum","ip_header_dont_fragment","ip_header_fin","ip_header_identity","ip_header_psh","ip_header_rst","ip_header_seqnum","ip_header_syn","ip_header_tos","ip_header_ttl","ip_header_urg","number_of_connections","packet_length","packet_randomized" -"2010-02-10 00:00:00",,172.16.0.1,80,65534,ZZ,Region,City,node01.example.net,0,,,,,192.168.0.1,61234,64512,ZZ,Region,City,node01.example.com,0,"Communications, Service Provider, and Hosting Service",,,ddos,mirai,mirai,mirai,,,115.238.198.85/32,32,atk0,30,,,,,,,,,,,,,,,,,,,,,,,1440, -"2010-02-10 00:00:01",,172.16.0.2,43437,65534,ZZ,Region,City,node02.example.net,0,Information,,,,192.168.0.2,61234,64512,ZZ,Region,City,node02.example.com,0,"Communications, Service Provider, and Hosting Service",,,ddos,mirai,mirai,mirai,,,52.184.50.250/32,32,atk0,30,,,,,,,,,,,,,,,,,,,,,,,1440, -"2010-02-10 00:00:02",,172.16.0.3,80,65534,ZZ,Region,City,node03.example.net,0,,,,,192.168.0.3,61234,64512,ZZ,Region,City,node03.example.com,0,"Communications, Service Provider, and Hosting Service",,,ddos,mirai,mirai,mirai,,,211.99.102.216/32,32,atk10,30,,,,,,,,,,,,,,,,,,,,,,,1440, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv deleted file mode 100644 index d9448bd83d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","pattern","http_url","http_agent","http_request_method","url_scheme","session_tags","vulnerability_enum","vulnerability_id","vulnerability_class","vulnerability_score","vulnerability_severity","vulnerability_version","threat_framework","threat_tactic_id","threat_technique_id","target_vendor","target_product","target_class","file_md5","file_sha256","request_raw","body_raw" -"2021-08-01 00:24:08","tcp","191.23.45.67",36455,1234,"EE","HARJUMAA","TALLINN","191-23-45-67-host.example.com",518210,"Communications, Service Provider, and Hosting Service",,,,"109.87.65.43",80,5678,"UK","WINDSOR AND MAIDENHEAD","MAIDENHEAD",,518210,,"CAPRICA-EU","http-scan",,,,"3.1.3-dev",,"unknown","/js/ueditor/wwwroot/way-board.cgi",,,,,,,,,,,,,,,,,,,"GET /js/ueditor/wwwroot/way-board.cgi HTTP/1.0rnAccept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8rnAccept-Encoding: gzip, deflaternAccept-Language: en-US,en;q=0.5rnConnection: closernDnt: 1rnHost: 109.87.65.43rnOrigin: http://109.87.65.43rnReferer: http://109.87.65.43/rnUser-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.104 Safari/537.36 Core/1.53.3084.400 QQBrowser/9.6.11346.400", -"2021-08-01 05:21:59","tcp","45.67.89.123",58610,12345,"EE","HARJUMAA","TALLINN",,,,,,,"82.41.20.10",8080,23456,"UA","KHARKIVS'KA OBLAST'","KHARKIV",,,,"CAPRICA-EU","http-scan",,,,,,,"/","Mozilla/5.0 (Windows NT 5.1; rv:9.0.1) Gecko/20100101 Firefox/9.0.1","GET","http",,,,,,,,,,,,,,,,"R0VUIC8gSFRUUC8xLjENCkhvc3Q6IDgyLjQxLjIwLjEwOjgwODANCkFjY2VwdDogdGV4dC9odG1sLGFwcGxpY2F0aW9uL3hodG1sK3htbCxhcHBsaWNhdGlvbi94bWw7cT0wLjksKi8qO3E9MC44DQpBY2NlcHQtRW5jb2Rpbmc6IGRlZmxhdGUsIGd6aXAsIGlkZW50aXR5DQpBY2NlcHQtTGFuZ3VhZ2U6IGVuLVVTO3E9MC42LGVuO3E9MC40DQpVc2VyLUFnZW50OiBNb3ppbGxhLzUuMCAoV2luZG93cyBOVCA1LjE7IHJ2OjkuMC4xKSBHZWNrby8yMDEwMDEwMSBGaXJlZm94LzkuMC4xDQoNCg==", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv.license deleted file mode 100644 index c1900637ff..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Mikk Margus Möll -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv deleted file mode 100644 index 174360bbdc..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv +++ /dev/null @@ -1,7 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","network","routedspoof","session","nat" -"2021-03-28 00:42:59","tcp","98.191.250.0",,22898,"US","OKLAHOMA","OKLAHOMA CITY","ip-98.191.250.0.atlinkservices.com",517311,,,,,,,,,,,,,,"caida","ip-spoofer",,,,"ipv4",,"98.191.250.0/24","received",1112907,"True" -"2021-03-28 01:36:22","tcp","191.7.16.0",,262485,"BR","RIO DE JANEIRO","NOVA IGUACU",,,,,,,,,,,,,,,,"caida","ip-spoofer",,,,"ipv4",,"191.7.16.0/24","received",1112914,"False" -"2021-03-28 02:10:58","tcp","202.53.160.0",,23923,"BD","DHAKA","DHAKA",,,,,,,,,,,,,,,,"caida","ip-spoofer",,,,"ipv4",,"202.53.160.0/24","received",1112931,"True" -"2021-03-28 03:41:51","tcp","87.121.75.0",,134697,"AU","QUEENSLAND","BRISBANE",,,,,,,,,,,,,,,,"caida","ip-spoofer",,,,"ipv4",,"87.121.75.0/24","received",1112953,"True" -"2021-03-28 06:07:17","tcp","189.201.194.0",,262944,"MX","COAHUILA","SALTILLO","ip-189-201-194-0.slw.spectro.mx",,,,,,,,,,,,,,,"caida","ip-spoofer",,,,"ipv4",,"189.201.194.0/24","received",1113015,"True" -"2021-03-28 06:59:53","tcp","197.15.48.0",,37671,"TN","TUNIS","TUNIS",,517311,,,,,,,,,,,,,,"caida","ip-spoofer",,,,"ipv4",,"197.15.48.0/24","received",1113035,"True" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv.license deleted file mode 100644 index f4e16ec677..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Birger Schacht -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv deleted file mode 100644 index eb0cbbab95..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv +++ /dev/null @@ -1,7 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id" -"2021-06-07 00:00:00","tcp","190.229.1.2",52955,7303,"AR","BUENOS AIRES","CASEROS",,517311,,,,,"168.63.134.179",16464,8075,"HK","HONG KONG","HONG KONG",,334111,"Information","MSDCU","b68-zeroaccess-2-32bit","zeroaccess","b68-zeroaccess-2-32bit",,, -"2021-06-07 00:00:00","tcp","96.20.3.4",16464,5769,"CA","QUEBEC","LAVAL",,517311,"Communications, Service Provider, and Hosting Service",,,,"52.169.3.4",16464,8075,"IE","DUBLIN","DUBLIN",,334111,"Information","MSDCU","b68-zeroaccess-2-32bit","zeroaccess","b68-zeroaccess-2-32bit",,, -"2021-06-07 00:00:00","tcp","187.222.5.6",55049,8151,"MX","CIUDAD DE MEXICO","MEXICO CITY",,517311,,,,,"168.63.134.179",16464,8075,"HK","HONG KONG","HONG KONG",,334111,"Information","MSDCU","b68-zeroaccess-2-32bit","zeroaccess","b68-zeroaccess-2-32bit",,, -"2021-06-07 00:00:00","tcp","75.84.7.8",64190,20001,"US","CALIFORNIA","NORTH HOLLYWOOD",,517311,"Communications, Service Provider, and Hosting Service",,,,"52.169.7.8",16464,8075,"IE","DUBLIN","DUBLIN",,334111,"Information","MSDCU","b68-zeroaccess-2-32bit","zeroaccess","b68-zeroaccess-2-32bit",,, -"2021-06-07 00:00:00","tcp","24.15.9.10",60373,7922,"US","ILLINOIS","HOMER GLEN",,517311,"Communications, Service Provider, and Hosting Service",,,,"104.40.6.5",16464,8075,"US","CALIFORNIA","SAN FRANCISCO",,334111,"Information","MSDCU","b68-zeroaccess-2-32bit","zeroaccess","b68-zeroaccess-2-32bit",,, -"2021-06-07 00:00:00","tcp","124.101.11.12",50386,4713,"JP","FUKUOKA","FUKUOKA",,517311,"Communications, Service Provider, and Hosting Service",,,,"23.99.101.165",16465,8075,"HK","HONG KONG","HONG KONG",,334111,"Information","MSDCU","b68-zeroaccess-2-64bit","zeroaccess","b68-zeroaccess-2-64bit",, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv.license deleted file mode 100644 index f4e16ec677..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Birger Schacht -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv deleted file mode 100644 index c56d1f218b..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv +++ /dev/null @@ -1,6 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","http_url","http_host","http_agent","forwarded_by","ssl_cipher","http_referer" -"2021-06-07 00:00:00","tcp","31.206.1.2",49245,8386,"TR","ANTALYA","KEPEZ",,,,,,,"40.121.206.97",80,8075,"US","VIRGINIA","ASHBURN",,334111,"Information","MSDCU","necurs","necurs","necurs",,,,"/locator.php","40.121.206.97",,,, -"2021-06-07 00:00:00","tcp","177.140.3.4",35919,28573,"BR","SAO PAULO","SAO PAULO",,517312,,,,,"204.95.99.204",443,8075,"US","WASHINGTON","REDMOND",,334111,"Information","MSDCU","caphaw","caphaw","caphaw",,,,"/index.php","3fo8jrthz3y.rgk.cc","Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.0.6103)",,,"null" -"2021-06-07 00:00:01","tcp","180.190.5.6",49264,132199,"PH","CEBU","MANDAUE",,517311,,,,,"40.121.206.97",80,8075,"US","VIRGINIA","ASHBURN",,334111,"Information","MSDCU","necurs","necurs","necurs",,,,"/locator.php","40.121.206.97",,,, -"2021-06-07 00:00:01","tcp","197.157.7.8",55307,37129,"KE","NAIROBI CITY","NAIROBI",,,,,,,"40.121.206.97",80,8075,"US","VIRGINIA","ASHBURN",,334111,"Information","MSDCU","necurs","necurs","necurs",,,,"/news/stream.php","40.121.206.97",,,, -"2021-06-07 00:00:01","tcp","174.114.9.10",59000,812,"CA","ONTARIO","OTTAWA",,517311,"Communications, Service Provider, and Hosting Service",,,,"40.121.206.97",80,8075,"US","VIRGINIA","ASHBURN",,334111,"Information","MSDCU","necurs","necurs","necurs",,,,"/locator.php","40.121.206.97",,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv.license deleted file mode 100644 index f4e16ec677..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Birger Schacht -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv deleted file mode 100644 index c5126c843a..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id" -"2021-03-04 00:00:00","tcp","190.113.1.2",17409,12252,"PE","METROPOLITANA DE LIMA","LIMA",,,,,,,"178.162.1.2",4455,28753,"DE","HESSEN","FRANKFURT AM MAIN",,518210,"Communications, Service Provider, and Hosting Service","eset","victorygate.b","victorygate.b",,,, -"2021-03-04 00:00:00","tcp","35.205.9.10",44696,15169,"BE","BRUXELLES-CAPITALE","BRUSSELS","x.x.205.35.bc.googleusercontent.com",519130,"Communications, Service Provider, and Hosting Service",,,,"148.81.9.10",80,1887,"PL","MAZOWIECKIE","WARSAW",,,,,"virut","virut",,,, -"2021-03-04 00:00:00","tcp","35.197.11.12",36968,15169,"US","OREGON","THE DALLES","x.x.197.35.bc.googleusercontent.com",519130,"Communications, Service Provider, and Hosting Service",,,,"148.81.111.11.12",80,1887,"PL","MAZOWIECKIE","WARSAW",,,,,"virut","virut",,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv.license deleted file mode 100644 index f4e16ec677..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Birger Schacht -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv deleted file mode 100644 index 3e85690d85..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","infection","family","tag","query_type","query","count" -"2022-01-06 00:00:02","udp","217.110.0.0",29614,8220,"DE","HESSEN","FRANKFURT AM MAIN",,518210,"Communications, Service Provider, and Hosting Service",,,,"calypso","calypso","msexchange","A","YolkIsh.COM",1 -"2022-01-06 00:00:02","udp","209.66.0.0",46189,40934,"US","VIRGINIA","ASHBURN",,518210,,,,,"orcus","orcus","rat","A","verble.rocks",1 -"2022-01-06 00:00:02","udp","217.110.0.0",3590,8220,"DE","HESSEN","FRANKFURT AM MAIN",,518210,"Communications, Service Provider, and Hosting Service",,,,"calypso","calypso","msexchange","A","RAwFuNS.COM",1 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv.license deleted file mode 100644 index 662bb20b71..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv deleted file mode 100644 index 4514f248ed..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv +++ /dev/null @@ -1,6 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","http_url","http_host","http_agent","forwarded_by","ssl_cipher","http_referer" -"2021-03-04 00:00:00","tcp","103.196.1.2",60902,134707,"PH","NUEVA ECIJA","DEL PILAR",,,,,,,"184.105.1.2",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,,"andromeda","avalanche-andromeda",,,,,"differentia.ru",,,, -"2021-03-04 00:00:00","tcp","5.14.3.4",55002,8708,"RO","CONSTANTA","CONSTANTA",,517311,"Communications, Service Provider, and Hosting Service",,,,"184.105.3.4",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,,"andromeda","avalanche-andromeda",,,,,"differentia.ru",,,, -"2021-03-04 00:00:00","tcp","49.145.5.6",31350,9299,"PH","CEBU","CEBU",,517311,,,,,"184.105.5.6",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,,"andromeda","avalanche-andromeda",,,,,"disorderstatus.ru",,,, -"2021-03-04 00:00:00","tcp","200.44.7.8",28063,8048,"VE","CARABOBO","VALENCIA",,517311,,,,,"184.105.7.8",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,,"andromeda","avalanche-andromeda",,,,,"differentia.ru",,,, -"2021-03-04 00:00:00","tcp","187.189.9.10",45335,17072,"MX","CHIHUAHUA","JUAREZ",,,,,,,"184.105.9.10",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,,"andromeda","avalanche-andromeda",,,,,"differentia.ru",,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv.license deleted file mode 100644 index f4e16ec677..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Birger Schacht -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv deleted file mode 100644 index 23a3cb2b68..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv +++ /dev/null @@ -1,6 +0,0 @@ -"timestamp","protocol","http_referer_ip","http_referer_port","http_referer_asn","http_referer_geo","http_referer_region","http_referer_city","http_referer_hostname","http_referer_naics","http_referer_sector","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","http_url","http_host","http_referer" -"2021-03-04 00:00:02","tcp","178.162.203.211",80,28753,"DE","HESSEN","FRANKFURT AM MAIN","12106.mobapptrack.com",518210,"Communications, Service Provider, and Hosting Service","85.17.31.82",80,60781,"NL","NOORD-HOLLAND","AMSTERDAM",,518210,"Communications, Service Provider, and Hosting Service",,,"kovter","kovter",,,1614816002,"GET /favicon.ico HTTP/1.1","12106.mobapptrack.com","http://12106.mobapptrack.com/click/redirect?feed_id=12106&sub_id=7&q=8A5491983C8FBE7743E2D2C36E45EBC4-18307118D2626C9BD756B3F09D14BB910E381EE4" -"2021-03-04 00:00:11","tcp","59.106.1.2",80,9370,"JP","OSAKA","OSAKA","x.noizm.com",518210,"Communications, Service Provider, and Hosting Service","178.162.1.2",80,28753,"DE","HESSEN","FRANKFURT AM MAIN",,518210,"Communications, Service Provider, and Hosting Service",,,"sunburst","sunburst",,,1614816011,"GET /animalally.com HTTP/1.1","freescanonline.com","http://x.noizm.com/jump.php?u=http://freescanonline.com/animalally.com" -"2021-03-04 00:00:12","tcp","142.250.3.4",80,15169,"US","CALIFORNIA","MOUNTAIN VIEW","x.blogspot.com",519130,"Communications, Service Provider, and Hosting Service","178.162.1.2",80,28753,"DE","HESSEN","FRANKFURT AM MAIN",,518210,"Communications, Service Provider, and Hosting Service",,,"kovter","kovter",,,1614816012,"GET /getjs?r=0.6393021999392658 HTTP/1.1","rxrtb.bid","http://x.blogspot.com/" -"2021-03-04 00:00:13","tcp","34.232.5.6",80,14618,"US","VIRGINIA","ASHBURN","www.example.com",454110,"Retail Trade","5.79.71.225",80,60781,"NL","NOORD-HOLLAND","AMSTERDAM",,518210,"Communications, Service Provider, and Hosting Service",,,"sunburst","sunburst",,,1614816013,"GET /personalationmall.com HTTP/1.1","freescanonline.com","http://www.example.com/teams/default.asp?u=EKL&t=c&s=lacrosse&p=remote&url=http://freescanonline.com/personalationmall.com" -"2021-03-04 00:01:26","tcp","210.172.7.8",80,2516,"JP","HOKKAIDO","SAPPORO","x.communes.jp",517312,"Communications, Service Provider, and Hosting Service","5.79.1.2",80,60781,"NL","NOORD-HOLLAND","AMSTERDAM",,518210,"Communications, Service Provider, and Hosting Service",,,"sunburst","sunburst",,,1614816086,"GET /raftcomply.com HTTP/1.1","freescanonline.com","http://x.communes.jp/?url=http://freescanonline.com/raftcomply.com" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv.license deleted file mode 100644 index c1900637ff..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Mikk Margus Möll -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv deleted file mode 100644 index 016d2f912b..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","http_url","http_host","http_agent","forwarded_by","ssl_cipher","http_referer" -"2022-03-02 09:14:19","tcp","2001:448a:1082:4d9b:7491:bf9e:3d5f:a634",49431,7713,"ID","JAKARTA RAYA","JAKARTA",,,,,,,"2001:470:1:332::fe",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,"boaxxe","boaxxe","3ve",,,,"GET /QKMSvF2hl11j%2fbMkyPbF5EpHYhd6VWTG4u19K3Rt7JGU3lMYRqpq8wPYEuOGKKeidKW3pefVfKSjBnL0cXizZbmuWWu8AQNRqw5g9Ny5vZtiv638XKoWwCLuUOTISTV%2fLcpcS1%2f22NjWqgXkHGISAuyVtafqyCC%2f5cA0eYg9Me8VzAIFDdTArogQOdYhElf2xluhEFPsstGQ%2bwrM4VmKHJpzyjD7Y%2fN%2bQV3wnZNdVkEVk1k2iKBJkotYv3ajgYWr56xxCbY5vE1IpZBRNhhaUDNZo0kJgi%2b6knXZ4m7JHt%2fGtJeP%2bNTxHSUL2ELlTIiT3ENlPYD6FdH6ZBxT1OneW%2f0ih%2fcN7vctb5B5Qwa1ez7ZjN2QxgBYkFDDHHTs42ej5eF2BysWAQDSUr%2fcySyGxcfPveIpfQEdrynGKR6z3OYqkFnP%2bYRDQp2rt1qt0FwCB4L9cg05TQlSSTJVGfPDrtcqjvKY4c9hWwSHtE8jMRpeCYO4Es%2bWgwr5DjzMicmuZo%2f4Ycr16jpN7xlDJdJ8iCFZxbSGgVC7ksVlGE8wlfWPI4KTuX5U5s61eNWPTlAC%2fOGb8grtw%2ffzizoIX9D6ZUMvslGLQIp%2fvNmNQkZy8HhNoV6Lns%2figITP%2fpN0H8h9HjUTl9qn65xFOEVpc0motSy8alcTPtTRKq5Jvc4Ao0x3N%2fvCB1v4Epx7XC0UpFbw8TrYEvAczEfGsGM HTTP/1.1","devps.net","Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET4.0C; .NET4.0E)",,, -"2022-03-02 09:15:10","tcp","2001:448a:1082:4d9b:7491:bf9e:3d5f:a634",49460,7713,"ID","JAKARTA RAYA","JAKARTA",,,,,,,"2001:470:1:332::ef",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,"boaxxe","boaxxe","3ve",,,,"GET /QKMSvF2hl11j%2fbMkyPbF5EpHYhd6VWTG4u19K3Rt7JGU3lMYRqpq8wPYEuOGKKeidKW3pefVfKSjBnL0cXizZbmuWWu8AQNRqw5g9Ny5vZtiv638XKoWwCLuUOTISTV%2fLcpcS1%2f22NjWqgXkHGISAuyVtafqyCC%2f5cA0eYg9Me8VzAIFDdTArogQOdYhElf2xluhEFPsstGQ%2bwrM4VmKHJpzyjD7Y%2fN%2bQV3wnZNdVkEVk1k2iKBJkotYv3ajgYWr56xxCbY5vE1IpZBRNhhaUDNZo0kJgi%2b6knXZ4m7JHt%2fGtJeP%2bNTxHSUL2ELlTIiT3ENlPYD6FdH6ZBxT1OneW%2f0ih%2fcN7vctb5B5Qwa1ez7ZjN2QxgBYkFDDHHTs42ej5eF2BysWAQDSUr%2fcySyGxcfPveIpfQEdrynGKR6z3OYqkFnP%2bYRDQp2rt1qt0FwCB4L9cg05TQlSSTJVGfPDrtcqjvKY4c9hWwSHtE8jMRpeCYO4Es%2bWgwr5DjzMicmuZo%2f4Ycr16jpN7xlDJdJ8iCFZxbSGgVC7ksVlGE8wlfWPI4KTuX5U5s61eNWPTlAC%2fOGb8grtw%2ffzizoIX9D6ZUMvslGLQIp%2fvNmNQkZy8HhNoV6Lns%2figITP%2fpN0H8h9HjUTl9qn65xFOEVpc0motSy8alcTPtTRKq5Jvc4Ao0x3N%2fvCB1v4Epx7XC0UpFbw8TrYEvAczEfGsGM HTTP/1.1","devps.net","Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET4.0C; .NET4.0E)",,, -"2022-03-02 14:15:10","tcp","2603:8080:b20a:dc00:f06e:8304:71f6:27e2",62932,11427,"US","TEXAS","GARLAND",,517311,"Communications, Service Provider, and Hosting Service",,,,"2001:470:1:332::fe",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,"boaxxe","boaxxe","3ve",,,,"GET /WMoUNCvuKGzdqSCeQcadP1%2f0B%2f3bzpOmyKBU85Z25HVOhvDQUPFl%2fk8uOcLewS%2b1BsuHXalRAOIgGOYYs2igj6UX8FkdCAmDewWPvfDhPD45nwd2tx1lLf2IoIfuOtIpGR6bN5Q6hGpSBgfERqCa0ImHcwfcZ2EdO%2fWvg7R8H6SLcTiuUC0I4pzvlWt1CRLgLdIEU1hZ0nnFHIHhchb6D7ITEgBQ2chQDxy5TJMrGjm4Dac6dKl%2ft5uYhRhSjAHkLLtgrJjsqVtVbelTAkt5kdcqLlO09m1SH%2fvtAb%2fOvR2DbhBss7%2f64DG7g6cAnghNA6JrFn1uW7sw%2bnKH8koKQwzUjdSsbrQAvmg4r0KDDW8Diq64gfDzxFWkzCLOYifc%2fwlinXPCl7aJiNCoieDC1U98RNQg%2f5td4SZmJnDQ2%2f96CPbFeSpCez5WD1rCjrxLj1h2cqzIgkydEWACceWP9ztxc4QaObzEcgOGxbRckWC7H2aaLeT8jaYEYdKi1pwEKChSL3YdEt4ZIb2IFrWwzNaXEpQzFXf07f902OEdI9vVA1ZdEOBPG6rAIkzMdebfprfVyhKEWtrCd3Skg3COUFtRQks5jzG1nv4sVGijTfSgyn6xE9Taka668Nycik6nmHy8Huj3oC01j3tee%2f1Z3eI6tV7lgM5d3uFJ84slRGHUCwMfVozOGmZRwNo%2fz%2bA HTTP/1.1","devps.net","Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko",,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv.license deleted file mode 100644 index 662bb20b71..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/hp_http_scan.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/hp_http_scan.csv.license deleted file mode 100644 index f8f131c2ce..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/hp_http_scan.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/hp_ics_scan.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/hp_ics_scan.csv.license deleted file mode 100644 index f8f131c2ce..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/hp_ics_scan.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv deleted file mode 100644 index ccafbab3f1..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","url","host","ip","asn","geo","region","city","naics","sector","tag","source","sha256","application" -"2022-01-07 00:02:07","http://41.86.0.0:50008/Mozi.m","41.86.0.0","41.86.0.0",37203,"LR","MONTSERRADO","MONROVIA",,,"CVE-2016-10372",,"12013662c71da69de977c04cd7021f13a70cf7bed4ca6c82acbc100464d4b0ef","http" -"2022-01-07 00:03:14","http://42.225.0.0:38173/Mozi.m","42.225.0.0","42.225.0.0",4837,"CN","HENAN SHENG","ZHUMADIAN",517311,,"CVE-2018-10562",,,"http" -"2022-01-07 00:10:26","http://211.52.0.0:53029/Mozi.m","211.52.0.0","211.52.0.0",4766,"KR","CHUNGCHEONGNAM-DO","SAGOK-MYEON",517311,,"CVE-2018-10562",,,"http" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/outdated_dnssec_key.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/outdated_dnssec_key.csv.license deleted file mode 100644 index f8f131c2ce..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/outdated_dnssec_key.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv deleted file mode 100644 index 965d763a3c..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","url","host","ip","asn","geo","region","city","naics","sector","source" -"2022-02-01 08:00:07","https://priceless-pare.example.net/Postal-/acec6/","priceless-pare.example.net","172.245.0.0",64512,"US","NEW YORK","BUFFALO",518210,"Communications, Service Provider, and Hosting Service","openphish.com" -"2022-02-01 08:00:07","https://mailyahooattt.example.net/","mailyahooattt.example.net","199.34.0.0",64512,"US","CALIFORNIA","SAN FRANCISCO",,"Professional, Scientific, and Technical Services","openphish.com" -"2022-02-01 08:00:07","https://www.example.net/viewer/vbid-730ec2b1-omsttuer","www.example.net","216.58.0.0",64512,"US","UTAH","DRAPER",519130,"Communications, Service Provider, and Hosting Service","openphish.com" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv deleted file mode 100644 index d5baa730fe..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","http","http_code","http_reason","content_type","connection","proxy_authenticate","via","server","content_length","transfer_encoding","http_date" -"2010-02-10 00:00:00",192.168.0.1,tcp,3128,node01.example.com,http-connect-proxy-closed,64512,ZZ,Region,City,0,0,HTTP/1.1,407,"Proxy Authentication Required",text/html;charset=utf-8,keep-alive,"Basic realm=\"\"Squid proxy-caching web server\"\"",,squid/4.10,3741,,"Wed, 10 Feb 2010 00:00:00 GMT" -"2010-02-10 00:00:01",192.168.0.2,tcp,3128,node02.example.com,http-connect-proxy-closed,64512,ZZ,Region,City,0,0,HTTP/1.1,407,"Proxy Authentication Required",text/html;charset=utf-8,keep-alive,"Basic realm=\"\"00:23:24:43:1c:34\"\"",,,3833,,"Wed, 10 Feb 2010 00:00:01 GMT" -"2010-02-10 00:00:02",192.168.0.3,tcp,3128,node03.example.com,http-connect-proxy-closed,64512,ZZ,Region,City,0,0,HTTP/1.1,407,"Proxy Authentication Required",text/html;charset=utf-8,keep-alive,"Basic realm=\"\"Proxy\"\"",,,179,,"Wed, 10 Feb 2010 00:00:02 GMT" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv deleted file mode 100644 index 4710af9742..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","asn","geo","md5","protocol","port","host","bytes_in","bytes_out" -"2022-01-10 00:00:03","40.119.6.228",8075,"US","b575ce6dcce6502a8431db5610135c25","udp",123,"time.windows.com",0,0 -"2022-01-10 00:00:03","8.252.70.126",3356,"US","c0d947f9a8685b0d9f3efdba966389c2","tcp",80,,0,0 -"2022-01-10 00:00:03","52.109.8.22",8075,"US","c0d947f9a8685b0d9f3efdba966389c2","tcp",443,,0,0 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv deleted file mode 100644 index 697cb6209a..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","md5hash","request","type","response","family","tag","source" -"2022-01-10 00:00:02","b575ce6dcce6502a8431db5610135c25","time.windows.com","A","40.119.6.228",,, -"2022-01-10 00:00:08","807679198a39c80d3ca07e60fd51b581","time.windows.com","A","40.119.6.228",,, -"2022-01-10 00:00:20","d97e973b9bf073bd3a217425259cea26","client-office365-tas.msedge.net","A","13.107.5.88",,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv deleted file mode 100644 index bbfe596a24..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","asn","geo","md5","url","user_agent","host","method" -"2022-01-10 00:01:13","23.196.47.89",20940,"US","37514b54e679a5313334e830ad780ec7","http://www.msftncsi.com/ncsi.txt","Microsoft NCSI","www.msftncsi.com","GET" -"2022-01-10 00:01:28","72.21.81.240",15133,"US","37514b54e679a5313334e830ad780ec7","http://www.download.windowsupdate.com/msdownload/update/v3/static/trustedr/en/authrootstl.cab","Microsoft-CryptoAPI/6.1","www.download.windowsupdate.com","GET" -"2022-01-10 00:08:24","23.56.4.57",20940,"US","e97ea2820c0d79f3f3ca241d4dcd1060","http://crl.microsoft.com/pki/crl/products/MicTimStaPCA_2010-07-01.crl","Microsoft-CryptoAPI/6.1","crl.microsoft.com","GET" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv deleted file mode 100644 index c0ff0bdf1e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","name","model","device","features","device_vendor","device_type","device_model","device_version","device_sector" -"2018-07-26 02:07:16","36.239.124.210","tcp",5555,"36-239-124-210.dynamic-ip.hinet.net","adb",3462,"TW","TAOYUAN COUNTY","TAOYUAN CITY",518210,737415,"hlteuc","SAMSUNG-SM-N900A","hlteatt",,,,,, -"2018-07-26 02:07:16","36.236.108.107","tcp",5555,"36-236-108-107.dynamic-ip.hinet.net","adb",3462,"TW","TAIPEI CITY","TAIPEI",518210,737415,"marlin","Pixel XL","marlin","cmd,shell_v2",,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv deleted file mode 100644 index c5494d4582..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","machine_type","afp_versions","uams","flags","server_name","signature","directory_service","utf8_servername","network_address" -"2019-09-04 05:05:53","198.13.34.22","tcp",548,"host.local","afp",6057,"AA","LOCATION","LOCATION",517311,0,"TimeCapsule8,119","AFP3.3,AFP3.2,AFP3.1","DHCAST128,DHX2,SRP,Recon1","SupportsCopyFile,SupportsChgPwd,SupportsServerMessages,SupportsServerSignature,SupportsTCP/IP,SupportsSrvrNotifications,SupportsReconnect,SupportsOpenDirectory,SupportsUTF8Servername,SupportsUUIDs,SupportsSuperClient","airport-time-capsule-de-jack","4338364e37364442463948350069672d",,"AirPort Time Capsule de jack","198.33.24.165:548,10.0.1.1:548,fe80:0008:0000:0000:6e70:9fff:fed4::548,fe80:0009:0000:0000:6e70:9fff:fed4::548,179.24.24.165 (DNS address)," -"2019-09-04 05:05:56","198.40.27.212","tcp",548,"host.local","afp",6057,"AA","LOCATION","LOCATION",517311,0,"TimeCapsule8,119","AFP3.3,AFP3.2,AFP3.1","DHCAST128,DHX2,SRP,Recon1","SupportsCopyFile,SupportsChgPwd,SupportsServerMessages,SupportsServerSignature,SupportsTCP/IP,SupportsSrvrNotifications,SupportsReconnect,SupportsOpenDirectory,SupportsUTF8Servername,SupportsUUIDs,SupportsSuperClient","time-capsule-del-jack","433836544b303147463948360069672d",,"Time Capsule del Jack","0.0.0.1:548,10.0.1.1:548,198.33.42.1:548,fe80:000b:0000:0000:dea4:caff:feba::548,fe80:000c:0000:0000:dea4:caff:feba::548,fe80:000d:0000:0000:4c7d:ffff:fec7::548,0.0.0.1 (DNS address)," diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv deleted file mode 100644 index 92f078af7b..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","channel","message_length","class","method","version_major","version_minor","capabilities","cluster_name","platform","product","product_version","mechanisms","locales","sector" -"2022-01-10 04:32:13","47.103.0.0","tcp",5672,,"amqp",37963,"CN","SHANGHAI SHI","SHANGHAI",518210,,0,509,10,10,0,9,"publisher_confirms,exchange_exchange_bindings,basic.nack,consumer_cancel_notify,connection.blocked,consumer_priorities,authentication_failure_close,per_consumer_qos","rabbit@iZuf63m0nnq9bwf7lhjxrkZ","Erlang/OTP","RabbitMQ","3.3.5","PLAIN AMQPLAIN","en_US", -"2022-01-10 04:32:13","141.95.0.0","tcp",5672,,"amqp",16276,"DE","SAARLAND","SAARBRUCKEN",518210,,0,509,10,10,0,9,"publisher_confirms,exchange_exchange_bindings,basic.nack,consumer_cancel_notify,connection.blocked,consumer_priorities,authentication_failure_close,per_consumer_qos,direct_reply_to","rabbit@mtk-breizh","Erlang/OTP 24.0.3","RabbitMQ","3.8.19","AMQPLAIN PLAIN","en_US", -"2022-01-10 04:32:13","54.234.0.0","tcp",5672,"ec2-54.234.0.0.compute-1.amazonaws.com","amqp",14618,"US","VIRGINIA","ASHBURN",454110,,0,509,10,10,0,9,"publisher_confirms,exchange_exchange_bindings,basic.nack,consumer_cancel_notify,connection.blocked,consumer_priorities,authentication_failure_close,per_consumer_qos,direct_reply_to","rabbit@1397a0e9629b","Erlang/OTP 24.2","RabbitMQ","3.9.11","PLAIN AMQPLAIN","en_US", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv deleted file mode 100644 index 9c43f8598b..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","machine_name","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,3283,node01.example.com,ard,64512,ZZ,Region,City,0,0,"Macmini (radio)",1006,201.20 -"2010-02-10 00:00:01",192.168.0.2,udp,3283,node02.example.com,ard,64512,ZZ,Region,City,0,0,biuro-rip-org-pl,1006,201.20 -"2010-02-10 00:00:02",192.168.0.3,udp,3283,node03.example.com,ard,64512,ZZ,Region,City,0,0,127.0.0.1,1006,201.20 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv deleted file mode 100644 index 7bd2b20e03..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","size","asn","geo","region","city","naics","sic","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,19,node01.example.com,chargen,,64512,ZZ,Region,City,0,0,,74,74.00 -"2010-02-10 00:00:01",192.168.0.2,udp,19,node02.example.com,chargen,,64512,ZZ,Region,City,0,0,,74,74.00 -"2010-02-10 00:00:02",192.168.0.3,udp,19,node03.example.com,chargen,,64512,ZZ,Region,City,0,0,Government,74,74.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv deleted file mode 100644 index 5182817c11..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic" -"2017-11-18 08:42:45","198.51.100.103","tcp",4786,"198-51-100-103.example.net","cisco-smart-install",8559,"AT","WIEN","VIENNA",0,0 -"2017-11-18 08:47:54","198.51.100.218","tcp",4786,,"cisco-smart-install",35609,"AT","WIEN","VIENNA",0,0 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv deleted file mode 100644 index 6d72dac539..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","response","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,5683,node01.example.com,coap,2,64512,ZZ,Region,City,0,0,",,",43,2.05 -"2010-02-10 00:00:01",192.168.0.2,udp,5683,node02.example.com,coap,2,64512,ZZ,Region,City,0,0,",,,,,,,,,",113,5.38 -"2010-02-10 00:00:02",192.168.0.3,udp,5683,node03.example.com,coap,1,64512,ZZ,Region,City,0,0,"`EsjAy************************************************************|CoAP RFC 7252 |************************************************************|This server is using the Eclipse Californium (Cf) CoAP framework|published under EPL+EDL: http://www.eclipse.org/californium/||(c) 2014, 2015, 2016 Institute for Pervasive Computing, ETH Zurich and others|************************************************************",454,113.50 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv deleted file mode 100644 index f4074f3ed9..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","server_version","couchdb_message","couchdb_version","git_sha","features","vendor","visible_databases","error","error_reason" -"2010-02-10 00:00:00",192.168.0.1,tcp,5984,node01.example.com,couchdb,64512,ZZ,Region,City,0,0,,"CouchDB/1.6.1 (Erlang OTP/18)",Welcome,1.6.1,,,"Ubuntu 16.04",_replicator;_users;test;shops;god,, -"2010-02-10 00:00:01",192.168.0.2,tcp,5984,node02.example.com,couchdb,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service","CouchDB/3.2.1 (Erlang OTP/23)",Welcome,3.2.1,244d428af,"access-ready,partitioned,pluggable-storage-engines,reshard,scheduler","The Apache Software Foundation",,, -"2010-02-10 00:00:02",192.168.0.3,tcp,5984,node03.example.com,couchdb,64512,ZZ,Region,City,0,0,"Retail Trade","CouchDB/3.2.1 (Erlang OTP/20)",Welcome,3.2.1,244d428af,"access-ready,partitioned,pluggable-storage-engines,reshard,scheduler","The Apache Software Foundation",,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv deleted file mode 100644 index 5aebed0500..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","http","http_code","http_reason","content_type","connection","www_authenticate","set_cookie","server","content_length","transfer_encoding","date","sector" -"2019-09-04 10:44:55","198.123.245.142","tcp",30005,,"cwmp",5678,"AA","LOCATION","LOCATION",517311,0,"HTTP/1.1",200,"OK","text/html","keep-alive",,,"DNVRS-Webs",5678,,"Wed, 04 Sep 2019 07:42:37 GMT", -"2019-09-04 11:06:50","198.123.245.162","tcp",5678,"localhost.localdomain","cwmp",5678,"AA","LOCATION","LOCATION",517311,0,"HTTP/1.1",404,"Not Found","text/html",,,,"RomPager/4.07 UPnP/1.0",,"chunked",, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv deleted file mode 100644 index c4bb32e573..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","db2_hostname","servername","size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,523,node01.example.com,db2,64512,ZZ,Region,City,0,0,NOWAK_SERWER,node01.example.com,298,14.90 -"2010-02-10 00:00:01",192.168.0.2,udp,523,node02.example.com,db2,64512,ZZ,Region,City,0,0,SPZOZ-DZIEWIN,node02.example.com,298,14.90 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv deleted file mode 100644 index 25e6f11d0e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","source_port","bytes","amplification","method" -"2010-02-10 00:00:00",192.168.0.1,tcp,80,node01.example.com,ddos-middlebox,64512,ZZ,Region,City,0,0,,49002,99,2,SYN+ACK:PSH -"2010-02-10 00:00:01",192.168.0.2,tcp,80,node02.example.com,ddos-middlebox,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",41200,99,2,SYN+ACK:PSH -"2010-02-10 00:00:02",192.168.0.3,tcp,80,node03.example.com,ddos-middlebox,64512,ZZ,Region,City,0,0,,47492,99,2,SYN+ACK:PSH diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv deleted file mode 100644 index 05b8078835..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv +++ /dev/null @@ -1,101 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","dns_version","asn","geo","region","city","min_amplification","p0f_genre","p0f_detail","naics","sic","sector" -"2018-04-14 00:14:34","198.51.100.179","udp",53,"198-51-100-189.example.net","openresolver","dnsmasq-2.66",25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:14:36","198.51.100.8","udp",53,"198-51-100-111.example.net","openresolver","dnsmasq-2.51",25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:14:34","198.51.100.179","udp",53,"198-51-100-189.example.net","openresolver","dnsmasq-2.66",25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:14:36","198.51.100.8","udp",53,"198-51-100-111.example.net","openresolver","dnsmasq-2.51",25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:14:36","198.51.100.158","udp",53,,"openresolver",,8447,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:14:37","198.51.100.7","udp",53,"198-51-100-7.example.net","openresolver","9.9.4-rpz2.13269.14-P2",13292,"AT","STEIERMARK","EISENERZ","4.6190",,,0,0, -"2018-04-14 00:14:38","198.51.100.167","udp",53,"198-51-100-167.example.net","openresolver","ZyWALL DNS",6830,"AT","KARNTEN","VILLACH","4.6667",,,0,0, -"2018-04-14 00:14:40","198.51.100.10","udp",53,"198-51-100-10.example.net","openresolver",,6830,"AT","WIEN","VIENNA","1.3810",,,518210,737415, -"2018-04-14 00:14:41","198.51.100.191","udp",53,"198-51-100-63.example.net","openresolver",,25255,"AT","TIROL","LIENZ","4.6190",,,0,0, -"2018-04-14 00:14:43","198.51.100.25","udp",53,"198-51-100-187.example.net","openresolver","p.4.0",25255,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:14:54","198.51.100.174","udp",53,"198-51-100-174.example.net","openresolver",,8445,"AT","SALZBURG","SALZBURG","6.4048",,,0,0, -"2018-04-14 00:14:54","198.51.100.181","udp",53,"198-51-100-181.example.net","openresolver","PowerDNS Recursor tele2/sil",8437,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:14:54","198.51.100.80","udp",53,"198-51-100-80.example.net","openresolver",,1901,"AT","WIEN","VIENNA","1.3810",,,518210,737415, -"2018-04-14 00:14:57","198.51.100.43","udp",53,"198-51-100-43.example.net","openresolver","vi2zcnsat10, Customer DNS",6830,"AT","STEIERMARK","GRAZ","1.3810",,,0,0, -"2018-04-14 00:14:58","198.51.100.124","udp",53,"198-51-100-124.example.net","openresolver","dnsmasq-2.47",28919,"AT","TIROL","EIBERG","3.8095",,,0,0, -"2018-04-14 00:15:00","198.51.100.60","udp",53,"198-51-100-60.example.net","openresolver",,24992,"AT","VORARLBERG","DORNBIRN","3.4762",,,0,0, -"2018-04-14 00:15:00","198.51.100.201","udp",53,"198-51-100-201.example.net","openresolver",,1853,"AT","STEIERMARK","GRAZ","4.6190",,,0,0, -"2018-04-14 00:15:01","198.51.100.82","udp",53,"198-51-100-82.example.net","openresolver","9.6-ESV-R7-P2",20811,"AT","TIROL","INNSBRUCK","4.6190",,,0,0, -"2018-04-14 00:15:01","198.51.100.105","udp",53,"198-51-100-105.example.net","openresolver",,6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:15:02","198.51.100.173","udp",53,"198-51-100-173.example.net","openresolver",,8445,"AT","NIEDEROSTERREICH","WALD","1.3810",,,0,0, -"2018-04-14 00:15:03","198.51.100.82","udp",53,"198-51-100-82.example.net","openresolver","PowerDNS Recursor tele2/sil",1257,"AT","NIEDEROSTERREICH","MODLING","1.3810",,,518111,737401, -"2018-04-14 00:15:05","198.51.100.39","udp",53,,"openresolver",,8437,"AT","VORARLBERG","LUSTENAU","1.3810",,,0,0, -"2018-04-14 00:15:09","198.51.100.33","udp",53,,"openresolver","dnsmasq-2.55",8447,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:15:09","198.51.100.248","udp",53,"198-51-100-248.example.net","openresolver",,39912,"AT","NIEDEROSTERREICH","HOLLABRUNN","3.8095",,,0,0, -"2018-04-14 00:15:10","198.51.100.119","udp",53,"198-51-100-172.example.net","openresolver","dnsmasq-2.62",25255,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:15:12","198.51.100.135","udp",53,"198-51-100-135.example.net","openresolver","no access.",43848,"AT","NIEDEROSTERREICH","WIESELBURG","3.8095",,,0,0, -"2018-04-14 00:15:15","198.51.100.64","udp",53,"198-51-100-64.example.net","openresolver",,6830,"AT","VORARLBERG","UBERSAXEN","1.3810",,,0,0, -"2018-04-14 00:15:17","198.51.100.80","udp",53,"198-51-100-80.example.net","openresolver",,42473,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:15:18","198.51.100.60","udp",53,"198-51-100-60.example.net","openresolver","198-51-100-60.example.net",35369,"AT","OBEROSTERREICH","LINZ","1.3810",,,0,0, -"2018-04-14 00:15:21","198.51.100.50","udp",53,,"openresolver","ZyWALL DNS",8447,"AT","STEIERMARK","TAUPLITZ","4.6667",,,0,0, -"2018-04-14 00:15:23","198.51.100.93","udp",53,,"openresolver","Microsoft DNS 6.1.7601 (1DB15D39)",8447,"AT","NIEDEROSTERREICH","SCHWADORF","1.3810",,,0,0, -"2018-04-14 00:15:24","198.51.100.33","udp",53,,"openresolver",,8447,"AT","STEIERMARK","FURSTENFELD","4.6190",,,0,0, -"2018-04-14 00:15:31","198.51.100.45","udp",53,,"openresolver","dnsmasq-2.52",8245,"AT","BURGENLAND","EISENSTADT","1.3810",,,0,0, -"2018-04-14 00:15:34","198.51.100.13","udp",53,"198-51-100-13.example.net","openresolver",,8447,"AT","WIEN","VIENNA","6.4048",,,518210,737415, -"2018-04-14 00:15:36","198.51.100.190","udp",53,,"openresolver",,8447,"AT","BURGENLAND","PINKAFELD","1.3810",,,0,0, -"2018-04-14 00:15:41","198.51.100.104","udp",53,,"openresolver",,21013,"AT","OBEROSTERREICH","LEONDING","2.6667",,,0,0, -"2018-04-14 00:15:42","198.51.100.101","udp",53,"198-51-100-101.example.net","openresolver",,8447,"AT","STEIERMARK","KAINACH BEI VOITSBERG","1.3810",,,0,0, -"2018-04-14 00:15:44","198.51.100.62","udp",53,"198-51-100-62.example.net","openresolver",,1901,"AT","OBEROSTERREICH","GMUNDEN","1.3810",,,518210,737415, -"2018-04-14 00:15:46","198.51.100.186","udp",53,"198-51-100-186.example.net","openresolver",,31239,"AT","WIEN","VIENNA","6.4048",,,0,0, -"2018-04-14 00:15:46","198.51.100.197","udp",53,,"openresolver",,8447,"AT","OBEROSTERREICH","KIRCHDORF AN DER KREMS","4.6190",,,0,0, -"2018-04-14 00:15:49","198.51.100.16","udp",53,,"openresolver",,8447,"AT","OBEROSTERREICH","LAAKIRCHEN","4.6190",,,0,0, -"2018-04-14 00:15:50","198.51.100.62","udp",53,"198-51-100-62.example.net","openresolver",,6830,"AT","NIEDEROSTERREICH","WIENER NEUSTADT","4.6190",,,0,0, -"2018-04-14 00:15:53","198.51.100.7","udp",53,"198-51-100-7.example.net","openresolver",,198950,"AT","TIROL","REUTTE","4.6190",,,518210,737415, -"2018-04-14 00:15:53","198.51.100.177","udp",53,"198-51-100-177.example.net","openresolver","Microsoft DNS 6.1.7601 (1DB1446A)",12605,"AT","OBEROSTERREICH","LINZ","1.3810",,,0,0, -"2018-04-14 00:15:57","198.51.100.47","udp",53,,"openresolver",,8447,"AT","NIEDEROSTERREICH","KOTTINGBRUNN","1.3810",,,0,0, -"2018-04-14 00:15:59","198.51.100.95","udp",53,"198-51-100-67.example.net","openresolver","GNS DNS Version 3",57169,"AT","WIEN","VIENNA","1.3810",,,0,0,"Information Technology" -"2018-04-14 00:16:02","198.51.100.104","udp",53,"198-51-100-104.example.net","openresolver",,6830,"AT","OBEROSTERREICH","BAD WIMSBACH-NEYDHARTING","1.3810",,,0,0, -"2018-04-14 00:16:04","198.51.100.106","udp",53,,"openresolver",,8447,"AT","STEIERMARK","GRAZ","1.3810",,,0,0, -"2018-04-14 00:16:05","198.51.100.204","udp",53,"198-51-100-204.example.net","openresolver",,12605,"AT","OBEROSTERREICH","LINZ","4.6190",,,0,0, -"2018-04-14 00:16:05","198.51.100.111","udp",53,"198-51-100-111.example.net","openresolver",,8447,"AT","OBEROSTERREICH","LINZ","1.3810",,,518210,737415, -"2018-04-14 00:16:06","198.51.100.131","udp",53,"198-51-100-139.example.net","openresolver","p.4.0",25255,"AT","OBEROSTERREICH","TRAUN","1.3810",,,0,0, -"2018-04-14 00:16:10","198.51.100.240","udp",53,"198-51-100-240.example.net","openresolver",,6830,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:16:13","198.51.100.9","udp",53,"198-51-100-42.example.net","openresolver",,13026,"AT","STEIERMARK","LEIBNITZ","6.4048",,,0,0, -"2018-04-14 00:16:15","198.51.100.231","udp",53,"198-51-100-74.example.net","openresolver",,25255,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:16:17","198.51.100.228","udp",53,"198-51-100-227.example.net","openresolver","u.1.0",25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:16:19","198.51.100.152","udp",53,"198-51-100-152.example.net","openresolver",,34694,"AT","TIROL","WORGL","4.6190",,,0,0, -"2018-04-14 00:16:21","198.51.100.88","udp",53,,"openresolver",,8447,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:16:22","198.51.100.97","udp",53,"198-51-100-97.example.net","openresolver",,8447,"AT","TIROL","INNSBRUCK","1.3810",,,518210,737415, -"2018-04-14 00:16:23","198.51.100.208","udp",53,"198-51-100-208.example.net","openresolver","dnsmasq-2.62",8447,"AT","TIROL","OTZTAL-BAHNHOF","1.3810",,,0,0, -"2018-04-14 00:16:33","198.51.100.113","udp",53,"198-51-100-121.example.net","openresolver","dnsmasq-2.62",25255,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:16:35","198.51.100.34","udp",53,"198-51-100-44.example.net","openresolver",,25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:16:37","198.51.100.236","udp",53,,"openresolver",,8447,"AT","NIEDEROSTERREICH","ST. ANDRAE-WOERDERN","4.6190",,,0,0, -"2018-04-14 00:16:40","198.51.100.46","udp",53,"198-51-100-46.example.net","openresolver",,21013,"AT","OBEROSTERREICH","LEONDING","2.6667",,,0,0, -"2018-04-14 00:16:45","198.51.100.72","udp",53,"198-51-100-5.example.net","openresolver",,25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:16:50","198.51.100.179","udp",53,"198-51-100-179.example.net","openresolver",,31125,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:16:50","198.51.100.107","udp",53,"198-51-100-107.example.net","openresolver","dnsmasq-2.66",18845,"AT","WIEN","VIENNA","1.3810",,,0,0,"Information Technology" -"2018-04-14 00:16:51","198.51.100.188","udp",53,,"openresolver","9.9.4-RedHat-9.9.4-51.el7_4.2",49322,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:16:54","198.51.100.232","udp",53,"198-51-100-232.example.net","openresolver",,6830,"AT","SALZBURG","SALZBURG","1.3810",,,0,0, -"2018-04-14 00:16:55","198.51.100.102","udp",53,"198-51-100-102.example.net","openresolver","ZyWALL DNS",6830,"AT","KARNTEN","WERNBERG","3.4762",,,0,0, -"2018-04-14 00:16:59","198.51.100.162","udp",53,"198-51-100-162.example.net","openresolver",,8445,"AT","SALZBURG","SALZBURG","1.3810",,,0,0, -"2018-04-14 00:17:00","198.51.100.110","udp",53,"198-51-100-110.example.net","openresolver",,31543,"AT","TIROL","SOLDEN","4.6190",,,0,0, -"2018-04-14 00:17:02","198.51.100.193","udp",53,"198-51-100-193.example.net","openresolver",,8447,"AT","STEIERMARK","FOHNSDORF","1.3810",,,0,0, -"2018-04-14 00:17:06","198.51.100.45","udp",53,"198-51-100-45.example.net","openresolver",,61201,"AT","KARNTEN","KLAGENFURT AM WORTHERSEE","1.3810",,,0,0, -"2018-04-14 00:17:06","198.51.100.219","udp",53,"198-51-100-219.example.net","openresolver",,8437,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:10","198.51.100.47","udp",53,"198-51-100-47.example.net","openresolver","unsupported query",8412,"AT","NIEDEROSTERREICH","WIENER NEUSTADT","3.8095",,,0,0, -"2018-04-14 00:17:13","198.51.100.87","udp",53,"198-51-100-87.example.net","openresolver","dnsmasq-2.40",6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:16","198.51.100.121","udp",53,"198-51-100-121.example.net","openresolver",,8447,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:20","198.51.100.115","udp",53,,"openresolver",,8447,"AT","TIROL","WAIDRING","1.3810",,,0,0, -"2018-04-14 00:17:22","198.51.100.235","udp",53,,"openresolver",,8447,"AT","OBEROSTERREICH","GRIESKIRCHEN","1.3810",,,0,0, -"2018-04-14 00:17:33","198.51.100.154","udp",53,,"openresolver",,8447,"AT","STEIERMARK","GRAZ","4.6190",,,0,0, -"2018-04-14 00:17:36","198.51.100.36","udp",53,"198-51-100-36.example.net","openresolver","BIND",12605,"AT","OBEROSTERREICH","LINZ","4.6190",,,0,0, -"2018-04-14 00:17:38","198.51.100.100","udp",53,"198-51-100-100.example.net","openresolver","dnsmasq-2.40",6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:41","198.51.100.181","udp",53,"198-51-100-181.example.net","openresolver","Microsoft DNS 6.0.6002 (17724D35)",6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:41","198.51.100.242","udp",53,"198-51-100-242.example.net","openresolver","Microsoft DNS 6.0.6002 (17724D35)",34767,"AT","NIEDEROSTERREICH","WIENER NEUSTADT","3.2857",,,0,0, -"2018-04-14 00:17:42","198.51.100.38","udp",53,,"openresolver","PowerDNS Recursor tele2/sil",8437,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:43","198.51.100.132","udp",53,"198-51-100-132.example.net","openresolver","PowerDNS Recursor tele2/sil",8437,"AT","STEIERMARK","GRAZ","1.3810",,,0,0, -"2018-04-14 00:17:49","198.51.100.166","udp",53,"198-51-100-166.example.net","openresolver","9.8.4-rpz2+rl005.12-P1",13292,"AT","STEIERMARK","KINDBERG","4.6190",,,0,0, -"2018-04-14 00:17:49","198.51.100.212","udp",53,"198-51-100-212.example.net","openresolver","dnsmasq-2.40",6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:51","198.51.100.225","udp",53,,"openresolver",,8220,"AT","WIEN","VIENNA","1.3810",,,518210,737415, -"2018-04-14 00:17:53","198.51.100.161","udp",53,"198-51-100-161.example.net","openresolver",,6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:54","198.51.100.12","udp",53,,"openresolver",,8447,"AT","NIEDEROSTERREICH","LANGENLOIS","1.3810",,,0,0, -"2018-04-14 00:17:55","198.51.100.113","udp",53,"198-51-100-113.example.net","openresolver",,6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:57","198.51.100.175","udp",53,"198-51-100-175.example.net","openresolver","PowerDNS Recursor tele2/sil",1257,"AT","NIEDEROSTERREICH","MODLING","1.3810",,,518111,737401, -"2018-04-14 00:17:59","198.51.100.107","udp",53,"198-51-100-107.example.net","openresolver",,50719,"AT","STEIERMARK","TIESCHEN","3.8095",,,0,0, -"2018-04-14 00:17:59","198.51.100.51","udp",53,"198-51-100-68.example.net","openresolver","dnsmasq-2.66",25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:18:04","198.51.100.131","udp",53,,"openresolver","ZyWALL DNS",8447,"AT","TIROL","OBERPERFUSS","3.4762",,,0,0, -"2018-04-14 00:18:05","198.51.100.138","udp",53,"198-51-100-138.example.net","openresolver","unsupported query",8412,"AT","NIEDEROSTERREICH","WIENER NEUSTADT","3.8095",,,0,0, -"2018-04-14 00:18:06","198.51.100.62","udp",53,"198-51-100-62.example.net","openresolver","viezcnsat13, Customer DNS",6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:18:07","198.51.100.109","udp",53,"198-51-100-109.example.net","openresolver",,1901,"AT","OBEROSTERREICH","LINZ","6.9524",,,518210,737415, -"2018-04-14 00:18:10","198.51.100.205","udp",53,"198-51-100-205.example.net","openresolver",,8437,"AT","WIEN","VIENNA","1.3810",,,0,0, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv deleted file mode 100644 index 535dc4ea8e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","sector","http","http_code","http_reason","content_type","server","date","experimental","api_version","arch","go_version","os","kernel_version","git_commit","min_api_version","build_time","pkg_version" -"2010-02-10 00:00:00",192.168.0.1,tcp,2375,node01.example.com,docker,18.05.0-ce,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",HTTP/1.1,200,OK,"application/json; charset=UTF-8","Docker/18.05.0-ce (linux)","Fri, 06 May 2022 14:06:30 GMT",false,1.37,amd64,go1.9.5,linux,3.10.0-514.26.2.el7.x86_64,f150324,1.12,2018-05-09T22:18:36.000000000+00:00, -"2010-02-10 00:00:01",192.168.0.2,tcp,2375,node02.example.com,docker,1.13.1,64512,ZZ,Region,City,0,0,,HTTP/1.1,200,OK,application/json,"Docker/1.13.1 (linux)","Fri, 06 May 2022 14:08:07 GMT",false,1.26,amd64,go1.10.3,linux,3.10.0-693.2.2.el7.x86_64,7d71120/1.13.1,1.12,2022-03-02T15:25:43.414574467+00:00,docker-1.13.1-209.git7d71120.el7.centos.x86_64 -"2010-02-10 00:00:02",192.168.0.3,tcp,2375,node03.example.com,docker,18.05.0-ce,64512,ZZ,Region,City,0,0,,HTTP/1.1,200,OK,"application/json; charset=UTF-8","Docker/18.05.0-ce (linux)","Fri, 06 May 2022 14:08:06 GMT",false,1.37,amd64,go1.9.5,linux,3.10.0-514.26.2.el7.x86_64,f150324,1.12,2018-05-09T22:18:36.000000000+00:00, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv deleted file mode 100644 index 60c7119733..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","device_vendor","device_type","device_model","device_version","device_id","device_serial","machine_name","manufacturer","method","http_port","internal_port","video_input_channels","alarm_input_channels","video_output_channels","alarm_output_channels","remote_video_input_channels","mac_address","ipv4_address","ipv4_gateway","ipv4_subnet_mask","ipv4_dhcp_enable","ipv6_address","ipv6_link_local","ipv6_gateway","ipv6_dhcp_enable","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,37810,node01.example.com,dvrdhcpdiscover,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",General,IPC,BCS-TIP3401IR-E-V,2.800.106F004.0.R,,6J0E022PAG35073,6J0E022PAG35073,General,client.notifyDevInfo,80,37777,1,0,0,0,0,38:c4:e8:03:b3:e2,192.168.0.1,192.168.0.240,255.255.255.0,0,fd09:4ab5:dae9:b078::1,fe80::3ac4:e8ff:fe03:b3e2/64,fd09:4ab5:dae9:b078::ff,0,794,794.00 -"2010-02-10 00:00:01",192.168.0.2,udp,37810,node02.example.com,dvrdhcpdiscover,64512,ZZ,Region,City,0,0,,Private,HCVR,HCVR,3.210.1.4,,2K0488CPAGS0ND6,HCVR,Private,client.notifyDevInfo,80,37777,3,0,0,0,9,3c:ef:8c:18:a5:07,192.168.0.2,192.168.0.240,255.255.255.0,0,fd09:4ab5:dae9:b078::2,fe80::3eef:8cff:fe18:a507/64,fd09:4ab5:dae9:b078::ff,,761,761.00 -"2010-02-10 00:00:02",192.168.0.3,udp,37810,node03.example.com,dvrdhcpdiscover,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",General,HCVR,BCS-XVR0401-IV,4.000.0000002.11,,5L034FAPAZA0E30,XVR,General,client.notifyDevInfo,80,37777,4,0,0,0,0,38:c4:e8:02:74:da,192.168.0.3,192.168.0.240,255.255.255.0,0,fd09:4ab5:dae9:b078::3,fe80::3ac4:e8ff:fe02:74da/64,fd09:4ab5:dae9:b078::ff,,711,711.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv deleted file mode 100644 index c681a8595d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","ok","name","cluster_name","http_code","build_hash","build_timestamp","build_snapshot","lucene_version","tagline","sector" -"2010-02-10 00:00:00",192.168.0.1,tcp,9200,node01.example.com,elasticsearch,2.3.5,64512,ZZ,Region,City,0,0,,"Red Skull",elasticsearch,,90f439ff60a3c0f497f91663701e64ccd01edbb4,2016-07-27T10:36:52Z,false,5.5.0,"You Know, for Search","Communications, Service Provider, and Hosting Service" -"2010-02-10 00:00:01",192.168.0.2,tcp,9200,node02.example.com,elasticsearch,7.17.0,64512,ZZ,Region,City,0,0,,allinonepod,docker-cluster,,bee86328705acaa9a6daede7140defd4d9ec56bd,,false,8.11.1,"You Know, for Search","Communications, Service Provider, and Hosting Service" -"2010-02-10 00:00:02",192.168.0.3,tcp,9200,node03.example.com,elasticsearch,7.15.0,64512,ZZ,Region,City,0,0,,f547c2952610,docker-cluster,,79d65f6e357953a5b3cbcc5e2c7c21073d89aa29,,false,8.9.0,"You Know, for Search","Communications, Service Provider, and Hosting Service" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv deleted file mode 100644 index 4e375a9b42..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv +++ /dev/null @@ -1,8 +0,0 @@ -"timestamp","ip","port","hostname","tag","asn","geo","region","city","naics","sic","sector","version","servername","url" -"2021-05-14 00:11:30","12.237.1.2",443,"afs-exch-cas2.xxx.com","exchange;cve-2021-26855",7018,"US","CALIFORNIA","TURLOCK",517311,,"Communications, Service Provider, and Hosting Service","15.2.721","AFS-EXCH2019", -"2021-05-14 00:11:37","98.153.3.4",443,"rrcs-98-153-x-x.west.biz.rr.com","exchange;webshell",20001,"US","CALIFORNIA","LOS ANGELES",517311,,"Communications, Service Provider, and Hosting Service","15.0.847","SSAMAIL", -"2021-05-14 00:11:38","206.210.5.6",443,"webmail.xxx.com","exchange;webshell",17054,"US","PENNSYLVANIA","PITTSBURGH",518210,,,"15.0.1178","OMNYXEXCH02", -"2021-05-14 00:11:38","12.33.7.8",443,"mail.xxx.org","exchange;cve-2021-26855",7018,"US","ARKANSAS","LITTLE ROCK",921120,,"Communications, Service Provider, and Hosting Service","15.1.2176","MHASVR02", -"2021-05-14 00:11:38","41.204.9.10",443,"mail.xxx.mg","exchange;cve-2021-26855",21042,"MG","ANTANANARIVO","ANTANANARIVO",,,,,"SABMHQE0232", -"2021-05-14 00:11:38","62.33.11.12",443,,"exchange;cve-2021-26855",20485,"RU","ALTAYSKIY KRAY","BARNAUL",,,,"15.2.659","PV-SRV04", -"2021-05-14 00:11:43","199.33.13.14",443,"mail.xxx.tv","exchange;cve-2021-26855",26481,"US","CALIFORNIA","LOS ANGELES",,,,"15.1.1779","MAIL", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv.license deleted file mode 100644 index f4e16ec677..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Birger Schacht -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv deleted file mode 100644 index 912e73d841..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","banner","handshake","cipher_suite","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serial_number","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","cert_valid","self_signed","cert_expired","validation_level","auth_tls_response","auth_ssl_response","tlsv13_support","tlsv13_cipher","jarm","device_vendor","device_type","device_model","device_version","device_sector" -"2019-03-06 06:37:00","61.126.3.70","tcp",21,"arcus-net.co.jp","ftp",4713,"JP","TOKYO","TOKYO",517311,737401,"220 FTP Server ready.|","TLSv1.2","TLS_RSA_WITH_AES_128_CBC_SHA",2048,"*.bizmw.com","GlobalSign Organization Validation CA - SHA256 - G2","Jan 14 08:04:50 2015 GMT","Jan 14 08:04:50 2020 GMT","D9:98:3F:2E:F9:D1:BE:9A:10:1E:DE:51:2C:C1:DF:01:18:0A:20:65","1121DC7421AB7924C3B1D396AEA3707E9E29",2,"sha256WithRSAEncryption","rsaEncryption","NTT Communications Corporation",,"JP","Tokyo","Minato-ku",,,,,,,,"GlobalSign nv-sa",,"BE",,,,,,,,,,"27:4A:8A:3A:A7:DF:82:D0:43:03:0E:6F:48:30:30:C9:24:77:11:1A:08:EF:F7:B9:74:0C:CE:40:87:03:D2:51","E5:93:8B:72:84:0F:35:52:8E:7A:6C:E3:EF:36:90:4C:F2:86:A7:4D:B2:DD:C0:C6:23:83:18:EF:DD:86:34:92:91:57:22:29:75:45:71:8B:3A:CD:F1:27:A9:CA:5F:70:5E:AC:15:A5:E6:63:FD:6F:BB:C5:E2:45:99:73:E9:E6","D1:A7:BC:96:78:1D:16:D0:24:A8:62:7C:3A:95:5A:4A","N","N","N","OV","234 AUTH TLS successful",,,,,,,,, -"2019-03-06 06:37:00","62.48.156.65","tcp",21,"dial-62-48-156-65.ptprime.net","ftp",15525,"PT","LISBOA","FRIELAS",0,0,"220-================================================================| PT Empresas| Acesso Reservado| Acesso nao autorizado punido por lei: 109/91; 67/98| ----------------------------------------------------------------| HENNES & MAURITZ LDA - 149093| SITE: PT303 - Cascais Shopping| MORADA: | NIR: EWS1822940| ================================================================|220 FTP server ready, 1 active clients of 4 simultaneous clients allowed.|",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"500 Syntax error, command unrecognized.","500 Syntax error, command unrecognized.",,,,,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv deleted file mode 100644 index 26f8ccbcf0..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","port","hostname","version","asn","geo","region","city","naics","sic","server_type","clusterid","total_disk","used_disk","free_disk","livenodes","namenodeaddress","volumeinfo" -"2017-09-13 02:06:05","199.116.235.200",50070,,"2.7.3, rbaa91f7c6bc9cb92be5982de4719c1c8af91ccff",15296,"CA","ALBERTA","CALGARY",0,0,"namenode","CID-64471a53-60cb-4302-9832-92f321f111fe",41567956992,53248,25160089600,"edmonton:50010",, -"2017-09-13 02:07:48","104.43.235.92",50075,,"2.7.1.2.4.0.0-169",8075,"US","IOWA","DES MOINES",334111,357101,"datanode","CID-771bae52-9e4f-4ec4-bc1a-c867585751f0",,,,,"sandbox.hortonworks.com","/hadoop/hdfs/data/current" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv.license deleted file mode 100644 index f8f131c2ce..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv deleted file mode 100644 index a7e3eb7074..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","http","http_code","http_reason","content_type","connection","www_authenticate","set_cookie","server","content_length","transfer_encoding","http_date" -"2018-04-19 00:02:26","75.74.78.113","tcp",8080,"c-75-74-78-113.hsd1.fl.comcast.net","http",7922,"US","FLORIDA","MIAMI",518111,737401,"HTTP/1.1",200,"OK","text/html",,,,"lighttpd",,"chunked","Thu, 19 Apr 2018 00:02:28 GMT" -"2018-04-19 00:02:26","88.162.174.130","tcp",8080,"sto95-3-88-162-174-130.fbx.proxad.net","http",12322,"FR",,"SAINT-OUEN-LAUMONE",518210,737415,"HTTP/1.1",200,"OK","text/html",,,,,17729,,"Thu, 19 Apr 2018 02:02:28 GMT" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv deleted file mode 100644 index b1f2330f1f..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","http","http_code","http_reason","content_type","connection","proxy_authenticate","via","server","content_length","transfer_encoding","http_date" -"2010-02-10 00:00:00",192.168.0.1,tcp,3128,node01.example.com,http-connect-proxy,64512,ZZ,Region,City,0,0,HTTP/1.1,200,"Connection established",,,,,,,,"Wed, 10 Feb 2010 00:00:00 GMT" -"2010-02-10 00:00:01",192.168.0.2,tcp,3128,node02.example.com,http-connect-proxy,64512,ZZ,Region,City,0,0,HTTP/1.1,200,"Connection established",,,,"HTTP/1.1 s_proxy_den1",,,,"Wed, 10 Feb 2010 00:00:01 GMT" -"2010-02-10 00:00:02",192.168.0.3,tcp,3128,node03.example.com,http-connect-proxy,64512,ZZ,Region,City,0,0,HTTP/1.1,200,"Connection established",,,,"HTTP/1.1 s_proxy_yvr",,,,"Wed, 10 Feb 2010 00:00:02 GMT" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv deleted file mode 100644 index 195342533e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","http","http_code","http_reason","content_type","connection","www_authenticate","set_cookie","server","content_length","transfer_encoding","http_date","version","build_date","detail" -"2010-02-10 00:00:00",192.168.0.1,tcp,8080,node01.example.com,"basic-auth,http",64512,ZZ,Region,City,0,0,HTTP/1.1,401,Unauthorized,"text/html; charset=utf-8",,"Basic realm=\"\"OpenWebif\"\"",TWISTED_SESSION=5473ad3faa3de66685fb3a53bffb390b4fcec2039893009a06caf38e1bec8aa8,TwistedWeb/19.7.0,149,,"Wed, 10 Feb 2010 00:00:00 GMT",,, -"2010-02-10 00:00:01",192.168.0.2,tcp,80,node02.example.com,"basic-auth,http",64512,ZZ,Region,City,0,0,HTTP/1.1,401,Unauthorized,"text/html; charset=utf-8",,"Basic realm=\"\"OpenWebif\"\"",TWISTED_SESSION=d2460d37b7fdbdd6c27dd74423ead5704e553d4f2c230672313edc5602059e33,TwistedWeb/19.7.0,149,,"Wed, 10 Feb 2010 00:00:01 GMT",,, -"2010-02-10 00:00:02",192.168.0.3,tcp,443,node03.example.com,git-config-file,64512,ZZ,Region,City,0,0,,,,,,,,,,,"Wed, 10 Feb 2010 00:00:02 GMT",,,"repositoryformatversion = 0;filemode = false;bare = false;logallrefupdates = true;symlinks = false;ignorecase = true" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv deleted file mode 100644 index d327f1f3ba..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","device_vendor","device_type","device_model","device_version","device_id","response_size","raw_response" -2022-03-02 00:34:22,192.168.0.1,tcp,502,host1.example.net,modbus,64512,ZZ,REGION,CITY,0,0,Sector,Vendor 1,device_type,device_model,device_version,0,5,dGVzdDE= -2022-03-02 00:34:22,192.168.0.2,tcp,502,host2.example.net,modbus,64513,ZZ,REGION,CITY,0,0,Sector,Vendor 2,device_type,device_model,device_version,0,5,dGVzdDI= -2022-03-02 00:34:22,192.168.0.3,tcp,502,host3.example.net,modbus,64514,ZZ,REGION,CITY,0,0,Sector,Vendor 3,device_type,device_model,device_version,0,5,dGVzdDM= diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv deleted file mode 100644 index 87a98157ff..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv +++ /dev/null @@ -1,96 +0,0 @@ -"timestamp","ip","port","hostname","tag","ipmi_version","asn","geo","region","city","none_auth","md2_auth","md5_auth","passkey_auth","oem_auth","defaultkg","permessage_auth","userlevel_auth","usernames","nulluser","anon_login","error","deviceid","devicerev","firmwarerev","version","manufacturerid","manufacturername","productid","productname","naics","sic","sector" -"2016-07-24 00:09:42","198.51.100.4",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:43","198.51.100.182",623,,"ipmi","2.0",28753,"DE","HESSEN","FRANKFURT AM MAIN","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:42","198.51.100.4",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:43","198.51.100.182",623,,"ipmi","2.0",28753,"DE","HESSEN","FRANKFURT AM MAIN","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:43","198.51.100.176",623,,"ipmi","1.5",2914,"DE","BAYERN","MUNICH","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:43","198.51.100.221",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:44","198.51.100.176",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:45","198.51.100.174",623,,"ipmi","1.5",2914,"DE","HESSEN","FRANKFURT AM MAIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:45","198.51.100.167",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:46","198.51.100.60",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:09:47","198.51.100.7",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:48","198.51.100.24",623,,"ipmi","2.0",24940,"DE","BAYERN","GUNZENHAUSEN","no","yes","yes","yes","yes","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:49","198.51.100.86",623,,"ipmi","1.5",2914,"DE","HESSEN","FRANKFURT AM MAIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:49","198.51.100.231",623,,"ipmi","1.5",20686,"DE","BAYERN","HAPPURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:49","198.51.100.197",623,,"ipmi","2.0",3320,"DE","BERLIN","BERLIN","no","no","yes","yes","yes","default","enabled","enabled","yes","no","yes",,,,,,,,,,541690,874899, -"2016-07-24 00:09:49","198.51.100.87",623,,"ipmi","1.5",3209,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:09:49","198.51.100.6",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:51","198.51.100.193",623,,"ipmi","2.0",15598,"DE","BAYERN","NUREMBERG","no","yes","yes","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:51","198.51.100.63",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:52","198.51.100.179",623,,"ipmi","2.0",3320,"DE","BAYERN","DENKLINGEN","no","no","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,518210,737415, -"2016-07-24 00:09:53","198.51.100.112",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:53","198.51.100.189",623,,"ipmi","2.0",30134,"DE","HESSEN","FRANKFURT AM MAIN","no","no","no","no","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0,"Communications" -"2016-07-24 00:09:54","198.51.100.44",623,"198-51-100-44.example.net","ipmi","2.0",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE","no","yes","yes","no","no","default","disabled","disabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:54","198.51.100.215",623,,"ipmi","1.5",2914,"DE","BAYERN","MUNICH","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:55","198.51.100.231",623,"198-51-100-231.example.net","ipmi","2.0",6805,"DE","HAMBURG","HAMBURG","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:55","198.51.100.234",623,,"ipmi","2.0",31103,"DE","THURINGEN","ERFURT","no","no","yes","no","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:55","198.51.100.165",623,,"ipmi","2.0",31400,"DE","RHEINLAND-PFALZ","FREINSHEIM","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:56","198.51.100.170",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:09:56","198.51.100.66",623,,"ipmi","2.0",41412,"DE","BAYERN","REGENSBURG","no","yes","yes","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:56","198.51.100.150",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:57","198.51.100.222",623,,"ipmi","2.0",34309,"DE","BERLIN","BERLIN","no","yes","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:57","198.51.100.19",623,,"ipmi","2.0",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:58","198.51.100.83",623,,"ipmi","1.5",3209,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:10:00","198.51.100.61",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:00","198.51.100.94",623,,"ipmi","1.5",2914,"DE","HESSEN","FRANKFURT AM MAIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:01","198.51.100.242",623,,"ipmi","2.0",28753,"DE","HESSEN","FRANKFURT AM MAIN","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:03","198.51.100.251",623,,"ipmi","2.0",553,"DE","BADEN-WURTTEMBERG","HEIDELBERG","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:10:03","198.51.100.41",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:04","198.51.100.160",623,"198-51-100-160.example.net","ipmi","1.5",2914,"DE","BAYERN","MUNICH","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:04","198.51.100.243",623,,"ipmi","1.5",2914,"DE","BAYERN","MUNICH","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:05","198.51.100.190",623,,"ipmi","1.5",2914,"DE","HESSEN","FRANKFURT AM MAIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:05","198.51.100.29",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:05","198.51.100.224",623,,"ipmi","2.0",13301,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:06","198.51.100.143",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","HEMER","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:07","198.51.100.120",623,,"ipmi","2.0",13003,"DE","SACHSEN","LEIPZIG","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:07","198.51.100.196",623,,"ipmi","1.5",20686,"DE","BAYERN","HAPPURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:07","198.51.100.123",623,,"ipmi","1.5",2914,"DE","HESSEN","FRANKFURT AM MAIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:08","198.51.100.122",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:08","198.51.100.192",623,,"ipmi","2.0",34171,"DE","BERLIN","BERLIN","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,518210,737415, -"2016-07-24 00:10:08","198.51.100.146",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:08","198.51.100.127",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:09","198.51.100.112",623,,"ipmi","2.0",24940,"DE","BAYERN","GUNZENHAUSEN","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:10:09","198.51.100.45",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:09","198.51.100.46",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","NEUSS","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:10","198.51.100.202",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:11","198.51.100.6",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:11","198.51.100.34",623,,"ipmi","2.0",3320,"DE","HESSEN","LEUN","no","yes","yes","no","yes","default","enabled","enabled","yes","no","no",,,,,,,,,,518210,737415, -"2016-07-24 00:10:12","198.51.100.210",623,,"ipmi","2.0",3320,"DE","BADEN-WURTTEMBERG","AALEN","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,541690,874899, -"2016-07-24 00:10:12","198.51.100.97",623,,"ipmi","2.0",42730,"DE","BERLIN","BERLIN","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:12","198.51.100.172",623,,"ipmi","1.5",2914,"DE","HESSEN","FRANKFURT AM MAIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:13","198.51.100.20",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:13","198.51.100.181",623,,"ipmi","2.0",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE","no","yes","yes","no","no","default","disabled","disabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:13","198.51.100.244",623,,"ipmi","2.0",28753,"DE","HESSEN","FRANKFURT AM MAIN","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:13","198.51.100.85",623,,"ipmi","2.0",34309,"DE","HESSEN","FRANKFURT AM MAIN","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:13","198.51.100.150",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:13","198.51.100.154",623,,"ipmi","2.0",196763,"DE","SAARLAND","ST. INGBERT","no","yes","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:13","198.51.100.83",623,,"ipmi","2.0",31342,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:14","198.51.100.6",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:14","198.51.100.228",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:14","198.51.100.150",623,,"ipmi","2.0",44066,"DE","BAYERN","MUNICH","no","yes","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:15","198.51.100.71",623,,"ipmi","2.0",44066,"DE","BAYERN","MUNICH","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:15","198.51.100.239",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:10:17","198.51.100.46",623,"198-51-100-53.example.net","ipmi","2.0",29083,"DE","BRANDENBURG","MAHLOW","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:17","198.51.100.78",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:18","198.51.100.164",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,812990,489999, -"2016-07-24 00:10:18","198.51.100.142",623,,"ipmi","2.0",34568,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:18","198.51.100.85",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:19","198.51.100.173",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:19","198.51.100.180",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:19","198.51.100.119",623,,"ipmi","2.0",12843,"DE","RHEINLAND-PFALZ","SPEYER","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:19","198.51.100.183",623,,"ipmi","1.5",12348,"DE","BAYERN","NUREMBERG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:20","198.51.100.108",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:20","198.51.100.221",623,"198-51-100-156.example.net","ipmi","2.0",24940,"DE","BAYERN","GUNZENHAUSEN","no","yes","yes","yes","yes","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:10:21","198.51.100.200",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:21","198.51.100.162",623,,"ipmi","1.5",30766,"DE","HESSEN","BENSHEIM","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:21","198.51.100.140",623,,"ipmi","2.0",31400,"DE","RHEINLAND-PFALZ","FREINSHEIM","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:21","198.51.100.121",623,,"ipmi","2.0",34549,"DE","HESSEN","FRANKFURT AM MAIN","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:22","198.51.100.33",623,,"ipmi","2.0",47215,"DE","NORDRHEIN-WESTFALEN","GUTERSLOH","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:22","198.51.100.203",623,,"ipmi","2.0",201011,"DE","BAYERN","NUREMBERG","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:23","198.51.100.16",623,,"ipmi","2.0",28753,"DE","HESSEN","FRANKFURT AM MAIN","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:24","198.51.100.166",623,,"ipmi","2.0",24940,"DE","BAYERN","GUNZENHAUSEN","no","no","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:25","198.51.100.135",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:25","198.51.100.154",623,"198-51-100-154.example.net","ipmi","1.5",2914,"DE","HESSEN","FRANKFURT AM MAIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:25","198.51.100.237",623,,"ipmi","2.0",12586,"DE","HESSEN","FRANKFURT AM MAIN","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:25","198.51.100.45",623,,"ipmi","2.0",13301,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv deleted file mode 100644 index a585db6eb6..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv +++ /dev/null @@ -1,2 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","ipp_version","cups_version","printer_uris","printer_name","printer_info","printer_more_info","printer_make_and_model","printer_firmware_name","printer_firmware_string_version","printer_firmware_version","printer_organization","printer_organization_unit","printer_uuid","printer_wifi_ssid","device_vendor","device_type","device_model","device_version","device_sector" -"2020-06-08 11:30:14","123.45.67.89","tcp",631,"some.host.com","ipp",12345,"AA","REGION","CITY",517311,0,"IPP/2.1","CUPS/2.0","ipp://123.45.67.89:631/ipp/print","NPI3F0D22","HP Color LaserJet MFP M277dw","http://123.45.67.89:631/hp/device/info_config_AirPrint.html?tab=Networking&menu=AirPrintStatus","HP Color LaserJet MFP M277dw",20191203,20191203,20191203,"org","unit","urn:uuid:456e4238-4a44-4643-4c42-10e1813f0a18","wifissid",,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv.license deleted file mode 100644 index 476908eebe..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2020 Thomas Hungenberg -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv deleted file mode 100644 index cef6b027c6..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","initiator_spi","responder_spi","next_payload","exchange_type","flags","message_id","next_payload2","domain_of_interpretation","protocol_id","spi_size","notify_message_type" -"2019-09-04 00:17:25","198.123.245.42","udp",500,"example.local","isakmp-vulnerable",5678,"AA","LOCATION","LOCATION",517311,0,"3e35c70729dfedef","253acab7cbfda607",11,05,00,00000000,00,00,,0,14 -"2019-09-04 00:17:28","198.123.245.67","udp",500,"example.local","isakmp-vulnerable",20255,"AA","LOCATION","LOCATION",0,0,"3e35c70729dfedef","b274460e7adc1bf0",11,05,00,00000000,00,00,,0,14 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv deleted file mode 100644 index ab71b9a15d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","sector","http","http_code","http_reason","content_type","server","date","major","minor","git_version","git_commit","git_tree_state","build_date","go_version","compiler","platform","handshake","cipher_suite","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serial_number","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","cert_valid","self_signed","cert_expired","validation_level","browser_trusted","browser_error","raw_cert","raw_cert_chain" -"2010-02-10 00:00:00",192.168.0.1,tcp,6443,node01.example.com,kubernetes,,64512,ZZ,Region,City,0,0,,HTTP/1.1,200,OK,application/json,,"Tue, 10 May 2022 14:24:13 GMT",1,20,v1.20.13,2444b3347a2c45eb965b182fb836e1f51dc61b70,clean,2021-11-17T13:00:29Z,go1.15.15,gc,linux/amd64,TLSv1.2,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,2,sha256WithRSAEncryption,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,N,Y,unknown,N,"x509: failed to load system roots and no roots provided",, -"2010-02-10 00:00:01",192.168.0.2,tcp,6443,node02.example.com,kubernetes,,64512,ZZ,Region,City,0,0,"Retail Trade",HTTP/1.1,200,OK,application/json,,"Tue, 10 May 2022 14:24:12 GMT",1,23,v1.23.3+e419edf,6f5a5295923a614a4202a7ad274b38b69f9ca8c0,clean,2022-02-25T06:26:46Z,go1.17.5,gc,linux/amd64,TLSv1.2,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,2,sha256WithRSAEncryption,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,N,Y,unknown,N,"x509: failed to load system roots and no roots provided",, -"2010-02-10 00:00:02",192.168.0.3,tcp,6443,node03.example.com,kubernetes,,64512,ZZ,Region,City,0,0,,HTTP/1.1,200,OK,application/json,,"Tue, 10 May 2022 14:24:12 GMT",1,16+,v1.16.9-aliyun.1,4f7ea78,,2020-05-08T07:29:59Z,go1.13.9,gc,linux/amd64,TLSv1.2,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,2,sha256WithRSAEncryption,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,N,Y,unknown,N,"x509: failed to load system roots and no roots provided",, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv deleted file mode 100644 index 54121fd3b7..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","size","configuration_naming_context","current_time","default_naming_context","dns_host_name","domain_controller_functionality","domain_functionality","ds_service_name","forest_functionality","highest_committed_usn","is_global_catalog_ready","is_synchronized","ldap_service_name","naming_contexts","root_domain_naming_context","schema_naming_context","server_name","subschema_subentry","supported_capabilities","supported_control","supported_ldap_policies","supported_ldap_version","supported_sasl_mechanisms","amplification" -"2010-02-10 00:00:00",192.168.0.1,tcp,389,node01.example.com,ldap-tcp,64512,ZZ,Region,City,0,0,0,"CN=Configuration,DC=ad,DC=example,DC=com",,"DC=ad,DC=example,DC=com",node01.example.com,7,,"CN=Configuration,DC=ad,DC=example,DC=com",2,,,,node01.example.com,"DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com","DC=example,DC=com","CN=Schema,CN=Configuration,DC=example,DC=com","CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com","CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com",,1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|,MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent,3|2,GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5, -"2010-02-10 00:00:01",192.168.0.2,tcp,389,node02.example.com,ldap-tcp,64512,ZZ,Region,City,0,0,0,"CN=Configuration,DC=ad,DC=example,DC=com",20220821124435.0Z,"DC=ad,DC=example,DC=com",node02.example.com,7,7,"CN=Configuration,DC=ad,DC=example,DC=com",7,25029662,TRUE,TRUE,node02.example.com,"DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com","DC=example,DC=com","CN=Schema,CN=Configuration,DC=example,DC=com","CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com","CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com",1.2.840.113556.1.4.800|1.2.840.113556.1.4.1670|1.2.840.113556.1.4.1791|1.2.840.113556.1.4.1935|1.2.840.113556.1.4.2080|1.2.840.113556.1.4.2237,1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|1.2.840.113556.1.4.2206|1.2.840.113556.1.4.2211|1.2.840.113556.1.4.2239|1.2.840.113556.1.4.2255|1.2.840.113556.1.4.2256|1.2.840.113556.1.4.2309,MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent,3|2,GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5, -"2010-02-10 00:00:02",192.168.0.3,tcp,389,node03.example.com,ldap-tcp,64512,ZZ,Region,City,0,0,0,"CN=Configuration,DC=ad,DC=example,DC=com",20220821124539.0Z,"DC=ad,DC=example,DC=com",node03.example.com,,,"CN=Configuration,DC=ad,DC=example,DC=com",,,,,node03.example.com,"DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com","DC=example,DC=com","CN=Schema,CN=Configuration,DC=example,DC=com","CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com","CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com",,1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv deleted file mode 100644 index 3cd5021c54..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","size","configuration_naming_context","current_time","default_naming_context","dns_host_name","domain_controller_functionality","domain_functionality","ds_service_name","forest_functionality","highest_committed_usn","is_global_catalog_ready","is_synchronized","ldap_service_name","naming_contexts","root_domain_naming_context","schema_naming_context","server_name","subschema_subentry","supported_capabilities","supported_control","supported_ldap_policies","supported_ldap_version","supported_sasl_mechanisms","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,389,node01.example.com,ldap-udp,64512,ZZ,Region,City,0,0,3038,"CN=Configuration,DC=ad,DC=example,DC=com",20220821044533.0Z,"DC=ad,DC=example,DC=com",node01.example.com,7,7,"CN=Configuration,DC=ad,DC=example,DC=com",7,222537,TRUE,TRUE,node01.example.com,"DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com","DC=example,DC=com","CN=Schema,CN=Configuration,DC=example,DC=com","CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com","CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com",1.2.840.113556.1.4.800|1.2.840.113556.1.4.1670|1.2.840.113556.1.4.1791|1.2.840.113556.1.4.1935|1.2.840.113556.1.4.2080|1.2.840.113556.1.4.2237,1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|1.2.840.113556.1.4.2206|1.2.840.113556.1.4.2211|1.2.840.113556.1.4.2239|1.2.840.113556.1.4.2255|1.2.840.113556.1.4.2256|1.2.840.113556.1.4.2309|1.2.840.113556.1.4.2330|1.2.840.113556.1.4.2354,MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent,3|2,GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5,58.42 -"2010-02-10 00:00:01",192.168.0.2,udp,389,node02.example.com,ldap-udp,64512,ZZ,Region,City,0,0,3062,"CN=Configuration,DC=ad,DC=example,DC=com",20220821044948.0Z,"DC=ad,DC=example,DC=com",node02.example.com,7,7,"CN=Configuration,DC=ad,DC=example,DC=com",7,1478714,TRUE,TRUE,node02.example.com,"DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com","DC=example,DC=com","CN=Schema,CN=Configuration,DC=example,DC=com","CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com","CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com",1.2.840.113556.1.4.800|1.2.840.113556.1.4.1670|1.2.840.113556.1.4.1791|1.2.840.113556.1.4.1935|1.2.840.113556.1.4.2080|1.2.840.113556.1.4.2237,1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|1.2.840.113556.1.4.2206|1.2.840.113556.1.4.2211|1.2.840.113556.1.4.2239|1.2.840.113556.1.4.2255|1.2.840.113556.1.4.2256|1.2.840.113556.1.4.2309|1.2.840.113556.1.4.2330|1.2.840.113556.1.4.2354,MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent,3|2,GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5,58.88 -"2010-02-10 00:00:02",192.168.0.3,udp,389,node03.example.com,ldap-udp,64512,ZZ,Region,City,0,0,36,"CN=Configuration,DC=ad,DC=example,DC=com",,"DC=ad,DC=example,DC=com",node03.example.com,,,"CN=Configuration,DC=ad,DC=example,DC=com",,,,,node03.example.com,"DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com","DC=example,DC=com","CN=Schema,CN=Configuration,DC=example,DC=com","CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com","CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com",,,,,,0.69 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv deleted file mode 100644 index 4a97121e75..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","mdns_name","mdns_ipv4","mdns_ipv6","services","workstation_name","workstation_ipv4","workstation_ipv6","workstation_info","http_name","http_ipv4","http_ipv6","http_ptr","http_info","http_target","http_port","spotify_name","spotify_ipv4","spotify_ipv6","opc_ua_discovery" -"2010-02-10 00:00:00",192.168.0.1,udp,5353,node01.example.com,mdns,64512,ZZ,Region,City,0,0,,,,"_smb._tcp.local.; _device-info._tcp.local.; _http._tcp.local.; _dacp._tcp.local.;",,192.168.0.1,fd09:4ab5:dae9:b078::1,,,192.168.0.1,fd09:4ab5:dae9:b078::1,,,,,,,, -"2010-02-10 00:00:01",192.168.0.2,udp,5353,node02.example.com,mdns,64512,ZZ,Region,City,0,0,,,,_home-assistant._tcp.local.;,,192.168.0.2,fd09:4ab5:dae9:b078::2,,,192.168.0.2,fd09:4ab5:dae9:b078::2,,,,,,,, -"2010-02-10 00:00:02",192.168.0.3,udp,5353,node03.example.com,"mdns,iot",64512,ZZ,Region,City,0,0,,,,"_webdav._tcp.local.; _adisk._tcp.local.; _smb._tcp.local.; _http._tcp.local.; _dacp._tcp.local.; _afpovertcp._tcp.local.; _device-info._tcp.local.;",,192.168.0.3,fd09:4ab5:dae9:b078::3,,snmeijer.local.,192.168.0.3,fd09:4ab5:dae9:b078::3,snmeijer._http._tcp.local.,"\"\"vendor=Synology\"\" \"\"model=DS218+\"\" \"\"serial=17A0PCN482002\"\" \"\"version_major=6\"\" \"\"version_minor=2\"\" \"\"version_build=25556\"\" \"\"admin_port=5000\"\" \"\"secure_admin_port=5001\"\" \"\"mac_address=00:11:32:80:fd:b5\"\"",snmeijer.local.,5000,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv deleted file mode 100644 index 6a1d445e7a..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","pid","pointer_size","uptime","time","curr_connections","total_connections","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,50260,node01.example.com,memcached,1.4.15,64512,ZZ,Region,City,0,0,1010,64,32908114,"2022-08-21 10:34:06",243,6106,"Communications, Service Provider, and Hosting Service",1144,81.71 -"2010-02-10 00:00:01",192.168.0.2,udp,11211,node02.example.com,memcached,1.4.13,64512,ZZ,Region,City,0,0,5316,64,9618498,"2022-08-21 10:39:21",9,2962,"Communications, Service Provider, and Hosting Service",1053,75.21 -"2010-02-10 00:00:02",192.168.0.3,udp,11211,node03.example.com,memcached,1.2.6,64512,ZZ,Region,City,0,0,1460,32,1375159,"2022-08-21 10:39:39",2,534,,442,31.57 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv deleted file mode 100644 index 1228dcfc60..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv +++ /dev/null @@ -1,11 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","gitversion","sysinfo","opensslversion","allocator","javascriptengine","bits","maxbsonobjectsize","ok","visible_databases","sector" -"2016-07-24 00:40:07","198.51.100.203","tcp",27017,"198-51-100-203.example.net","mongodb","2.4.5",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,"a2ddc68ba7c9cee17bfe69ed840383ec3506602b","Linux ip-198-51-100-100 198.51.100.103-2.ec2.v1.2.fc8xen #1 SMP Fri Nov 20 17:48:28 EST 2009 x86_64 BOOST_LIB_VERSION=1_49",,"tcmalloc","V8",64,16777216,1,"local | countly | admin", -"2016-07-24 00:40:07","198.51.100.42","tcp",27017,"198-51-100-208.example.net","mongodb","2.6.12",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,"d73c92b1c85703828b55c2916a5dd4ad46535f6a","Linux build5.ny.cbi.10gen.cc 2.6.32-431.3.1.el6.x86_64 #1 SMP Fri Jan 3 21:39:27 UTC 2014 x86_64 BOOST_LIB_VERSION=1_49",,"tcmalloc","V8",64,16777216,1,"none visible","Information Technology" -"2016-07-24 00:40:07","198.51.100.225","tcp",27017,"198-51-100-225.example.net","mongodb","3.0.6",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,"1ef45a23a4c5e3480ac919b28afcba3c615488f2","Linux ip-198-51-100-100 3.4.43-43.43.amzn1.x86_64 #1 SMP Mon May 6 18:04:41 UTC 2013 x86_64 BOOST_LIB_VERSION=1_49","OpenSSL 1.0.0-fips 29 Mar 2010","tcmalloc","V8",64,16777216,1,"bluu | local","Communications" -"2016-07-24 00:40:07","198.51.100.144","tcp",27017,"198-51-100-144.example.net","mongodb","2.2.2",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,"d1b43b61a5308c4ad0679d34b262c5af9d664267","Linux ip-198-51-100-100 198.51.100.252-2.ec2.v1.2.fc8xen #1 SMP Fri Nov 20 17:48:28 EST 2009 x86_64 BOOST_LIB_VERSION=1_49",,,,64,16777216,1,"errbit_production | DELETED_BECAUSE_YOU_DIDNT_PASSWORD_PROTECT_YOUR_MONGODB | admin | local", -"2016-07-24 00:40:07","198.51.100.68","tcp",27017,,"mongodb","3.2.6",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,"05552b562c7a0b3143a729aaa0838e558dc49b25","deprecated",,"tcmalloc","mozjs",64,16777216,1,"none visible", -"2016-07-24 00:40:07","198.51.100.101","tcp",27017,,"mongodb","3.0.9",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,"20d60d3491908f1ae252fe452300de3978a040c7","Linux ip-198-51-100-100 3.13.0-24-generic #46-Ubuntu SMP Thu Apr 10 19:11:08 UTC 2014 x86_64 BOOST_LIB_VERSION=1_49","OpenSSL 1.0.1f 6 Jan 2014","tcmalloc","V8",64,16777216,1,"none visible", -"2016-07-24 00:40:07","198.51.100.53","tcp",27017,"198-51-100-162.example.net","mongodb","3.2.6",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,"05552b562c7a0b3143a729aaa0838e558dc49b25","deprecated",,"tcmalloc","mozjs",64,16777216,1,"none visible", -"2016-07-24 00:40:07","198.51.100.206","tcp",27017,"198-51-100-206.example.net","mongodb","2.4.10",34011,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,"e3d78955d181e475345ebd60053a4738a4c5268a","Linux bs-linux32.10gen.cc 198.51.100.34-2.fc8xen #1 SMP Fri Feb 15 12:39:36 EST 2008 i686 BOOST_LIB_VERSION=1_49",,"system","V8",32,16777216,1,"sharelatex | test1 | local | tmp | lococms_production", -"2016-07-24 00:40:10","198.51.100.157","tcp",27017,"198-51-100-157.example.net","mongodb","2.0.6",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,"nogitversion","Linux biber 2.6.32-5-amd64 #1 SMP Mon Feb 25 00:26:11 UTC 2013 i686 BOOST_LIB_VERSION=1_49",,,,32,16777216,1,"none visible", -"2016-07-24 00:40:10","198.51.100.173","tcp",27017,"198-51-100-173.example.net","mongodb","2.6.12",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,"nogitversion","FreeBSD 101amd64-default-job-24 10.1-RELEASE-p33 FreeBSD 10.1-RELEASE-p33 amd64 BOOST_LIB_VERSION=1_49","OpenSSL 1.0.1l-freebsd 15 Jan 2015","system","V8",64,16777216,1,"none visible", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv deleted file mode 100644 index cfe4f00614..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv +++ /dev/null @@ -1,2 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","anonymous_access","raw_response","hex_code","code","cipher_suite","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serialNumber" -"2022-02-07 12:56:53","18.220.0.0","tcp",8883,"18-220-0-0.example.com","mqtt",12345,"US","OHIO","COLUMBUS",454110,,"N",20020005,05,"Connection Refused, not authorized","TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",2048,"*.tracesafe.io","Sectigo RSA Domain Validation Secure Server CA","2020-08-12 00:00:00","2022-11-14 00:00:00","70:84:F1:6D:28:DA:B6:E6:27:60:13:8B:2C:93:52:B6:7B:4B:13:7B","D2:D7:54:52:EB:86:4E:2D:34:4D:FC:CE:CD:CF:39:41:E1:06:5C:8B:B8:54:E6:0C:DF:FD:6E:E3:F1:B5:41:00","17:57:FB:88:9D:BE:A7:F0:29:A5:31:FC:79:DF:F7:8A:1C:D6:4A:DF:1B:4A:DC:BF:05:E7:E8:2F:79:9A:FA:FE:F7:E8:66:22:CB:B9:4C:72:F7:FB:6C:1D:59:8C:54:63:70:05:DE:7F:3C:2F:BA:B8:37:18:CE:29:6F:11:E8:AB","DE:2C:98:30:27:2E:7D:C9:ED:A3:9D:AF:9E:CE:14:CC","085699743A23114C9B6B8DC975A8AF42",2,"sha256WithRSAEncryption","rsaEncryption",,,,,,,,,,,,,"Sectigo Limited",,"GB","Greater Manchester","Salford",,,,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv.license deleted file mode 100644 index 476908eebe..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2020 Thomas Hungenberg -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv deleted file mode 100644 index e0ab4b9298..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","raw_response","hex_code","code","cipher_suite","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serialNumber" -"2022-01-10 00:59:34","47.106.0.0","tcp",8883,,"mqtt,mqtt-anon",37963,"CN","GUANGDONG SHENG","SHENZHEN",518210,,20020000,00,"Connection Accepted","TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",2048,"Server","RootCA","2020-05-08 08:07:05","2030-05-06 08:07:05","70:1A:1E:1F:EC:5F:7E:A9:12:32:B2:C9:8A:C9:EE:91:8E:0B:82:45","85:26:A2:F2:A2:50:CD:96:33:19:A6:2D:12:2E:97:6B:D3:06:3C:11:EA:01:B4:B7:25:2A:B7:4F:0A:8F:45:40","72:50:07:30:9A:6F:CB:FD:E2:80:69:02:65:62:77:16:C3:B4:0C:98:44:4E:D4:2C:AC:6B:AF:F8:9E:AB:51:C2:FA:A8:72:A3:45:DF:81:09:50:08:18:EB:03:34:FC:92:33:A7:12:46:FE:90:20:91:86:C5:4D:89:48:86:4C:CD","AB:A8:E0:2C:EF:AE:BF:9D:DD:FA:70:BA:2F:F2:CA:5C",02,2,"sha256WithRSAEncryption","rsaEncryption","EMQ",,"CN","hangzhou",,,,,,,,,"EMQ",,"CN","hangzhou",,,,,,,, -"2022-01-10 00:59:34","144.76.0.0","tcp",8883,,"mqtt,mqtt-anon",24940,"DE","SACHSEN-ANHALT","WERNIGERODE",518210,,20020000,00,"Connection Accepted","TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",2048,"example.com","R3","2021-12-06 13:48:04","2022-03-06 13:48:03","20:0E:AC:E7:AF:07:8D:D3:16:7C:63:D1:B9:12:AD:1D:2C:F0:46:86","DD:7A:4C:8A:1D:66:1D:7C:F5:17:04:5B:A0:B4:C4:E0:80:58:44:B4:DB:A7:5E:61:AE:43:9D:85:4C:9E:DC:83","55:B6:3D:56:A4:39:6E:99:B6:AF:72:AF:4D:3C:7C:C5:A8:C5:4F:A1:79:92:D0:46:8A:A2:9B:2A:48:0D:00:68:39:F0:B8:67:B4:E0:88:51:2A:D7:55:46:83:BD:ED:1E:09:6E:DB:3D:21:E2:AA:DB:42:6A:33:45:1A:2A:DB:4C","23:99:39:C6:77:D8:9F:55:90:FC:A5:FB:BA:72:8B:42","06B25BEAD1F43266ABCFCDDE408D3544D04B",2,"sha256WithRSAEncryption","rsaEncryption",,,,,,,,,,,,,"Lets Encrypt",,"US",,,,,,,,, -"2022-01-10 00:59:34","173.0.0.0","tcp",8883,"example.com","mqtt,mqtt-anon",5555,"US","CALIFORNIA","BURBANK",,,20020000,00,"Connection Accepted","TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA",2048,"foo.example.com","ClearView2Dev","2020-08-07 16:51:57","2030-08-05 16:51:57","32:4B:66:98:FA:5B:D2:D1:F2:53:83:21:19:11:5A:A9:BE:85:56:16","AE:0D:65:34:2F:51:F7:32:1E:DF:B1:DA:12:C7:6A:DE:42:B5:4B:FF:80:2C:E5:EF:99:F6:CC:01:4B:C9:77:68","44:C4:B8:19:FA:39:55:51:EC:E4:6D:C4:6D:0F:A5:46:BB:D5:F9:FD:A6:8D:DF:F3:2D:D2:92:6C:0B:D5:D3:25:CB:19:50:9D:A6:A4:D4:D3:2E:53:10:F5:8D:77:F7:90:F8:65:A7:79:AB:14:62:72:01:F3:EA:38:E2:68:C7:25","43:0D:A7:89:9E:76:8D:6E:D5:AD:95:CC:F2:91:87:56","A71541EFAE529B03",0,"sha256WithRSAEncryption","rsaEncryption","Sohonet",,,,"<",,,,,,,,"Sohonet","ClearView2Dev",,,,,,,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv deleted file mode 100644 index c12a6063eb..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","server_name","instance_name","tcp_port","named_pipe","response_size","amplification","sector" -"2010-02-10 00:00:00",192.168.0.1,udp,1434,node01.example.com,mssql,13.2.5026.0,64512,ZZ,Region,City,0,0,ERPOPTIMA,OPTIMA,49729,"\\\\ERPOPTIMA\\pipe\\MSSQL$OPTIMA\\sql\\query",310,310.00, -"2010-02-10 00:00:01",192.168.0.2,udp,1434,node02.example.com,mssql,13.0.1601.5,64512,ZZ,Region,City,0,0,SERWER,MSSQLSERVER,1433,,226,226.00,"Communications, Service Provider, and Hosting Service" -"2010-02-10 00:00:02",192.168.0.3,udp,1434,node03.example.com,mssql,10.50.2500.0,64512,ZZ,Region,City,0,0,ILONY,INSERTGT,49358,"\\\\ILONY\\pipe\\MSSQL$INSERTGT\\sql\\query",304,304.00, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv deleted file mode 100644 index 25fed2166b..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","sector","mysql_protocol_version","server_version","error_code","error_id","error_message","client_can_handle_expired_passwords","client_compress","client_connect_attrs","client_connect_with_db","client_deprecated_eof","client_found_rows","client_ignore_sigpipe","client_ignore_space","client_interactive","client_local_files","client_long_flag","client_long_password","client_multi_results","client_multi_statements","client_no_schema","client_odbc","client_plugin_auth","client_plugin_auth_len_enc_client_data","client_protocol_41","client_ps_multi_results","client_reserved","client_secure_connection","client_session_track","client_ssl","client_transactions","handshake","cipher_suite","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serial_number","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","cert_valid","self_signed","cert_expired","validation_level","browser_trusted","browser_error","raw_cert","raw_cert_chain" -"2010-02-10 00:00:00",192.168.0.1,tcp,3306,node01.example.com,mysql,,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",10,5.7.37-0ubuntu0.18.04.1,1,1,1,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,N,N,N,TLSv1.2,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,2,sha256WithRSAEncryption,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,,Y,unknown,,"x509: failed to load system roots and no roots provided",, -"2010-02-10 00:00:01",192.168.0.2,tcp,3306,node02.example.com,mysql,,64512,ZZ,Region,City,0,0,,10,5.7.30-0ubuntu0.18.04.1-log,1,1,1,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,N,N,N,TLSv1.2,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,2,sha256WithRSAEncryption,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,,Y,unknown,,"x509: failed to load system roots and no roots provided",, -"2010-02-10 00:00:02",192.168.0.3,tcp,3306,node03.example.com,mysql,,64512,ZZ,Region,City,0,0,"Retail Trade",10,8.0.23,1,1,1,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,N,N,N,TLSv1.2,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,2,sha256WithRSAEncryption,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,,Y,unknown,,"x509: failed to load system roots and no roots provided",, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv deleted file mode 100644 index e8a1108d5a..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","opcode","uptime","external_ip","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,5351,node01.example.com,nat-pmp,0,64512,ZZ,Region,City,0,0,128,291278940,192.168.0.1,,12,6.00 -"2010-02-10 00:00:01",192.168.0.2,udp,5351,node02.example.com,nat-pmp,0,64512,ZZ,Region,City,0,0,128,768416,192.168.0.2,,12,6.00 -"2010-02-10 00:00:02",192.168.0.3,udp,5351,node03.example.com,nat-pmp,0,64512,ZZ,Region,City,0,0,128,19629454,192.168.0.3,,12,6.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv deleted file mode 100644 index 932225b0b0..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","mac_address","asn","geo","region","city","workgroup","machine_name","username","naics","sic","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,137,node01.example.com,netbios,00-00-00-00-00-00,64512,ZZ,Region,City,,NBG6503,NBG6503,0,0,,229,4.58 -"2010-02-10 00:00:01",192.168.0.2,udp,137,node02.example.com,netbios,00-00-00-00-00-00,64512,ZZ,Region,City,PRACOWNIAELN.,NAS-OLD,NAS-OLD,0,0,,193,3.86 -"2010-02-10 00:00:02",192.168.0.3,udp,137,node03.example.com,netbios,00-25-90-F0-64-64,64512,ZZ,Region,City,HRSIGMA,HR-SRV01,,0,0,Government,157,3.14 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv deleted file mode 100644 index 4e91593565..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","port","hostname","tag","response","asn","geo","region","city","naics","sic","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,53413,node01.example.com,netis_vulnerability,Login:,64512,ZZ,Region,City,0,0,,18,18.00 -"2010-02-10 00:00:01",192.168.0.2,53413,node02.example.com,netis_vulnerability,Login:,64512,ZZ,Region,City,0,0,,18,18.00 -"2010-02-10 00:00:02",192.168.0.3,53413,node03.example.com,netis_vulnerability,Login:,64512,ZZ,Region,City,0,0,,18,18.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv deleted file mode 100644 index cc3cf6fc2f..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","version","clk_wander","clock","error","frequency","jitter","leap","mintc","noise","offset","peer","phase","poll","precision","processor","refid","reftime","rootdelay","rootdispersion","stability","state","stratum","system","tai","tc","naics","sic","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,123,node01.example.com,ntpversion,64512,ZZ,Region,City,4,,0xe6ac3809.363028e7,,2.018,0.977,0,,0.984,0.557,18986,,10,-10,unknown,81.15.252.130,0xe6ac35ba.2d2e8f2b,17.685,61.254,0.027,4,4,UNIX,,,0,0,,324,27.00 -"2010-02-10 00:00:01",192.168.0.2,udp,123,node02.example.com,ntpversion,64512,ZZ,Region,City,4,0.007,0xE6AC3806.7DF3B7A0,,-20.407,8.776,0,3,,-14.502,19244,,,-10,unknown,10.48.21.21,0xE6AC3431.B3B64790,32.25,105.778,,,8,UNIX,,10,0,0,"Transportation and Warehousing",328,27.33 -"2010-02-10 00:00:02",192.168.0.3,udp,123,node03.example.com,ntpversion,64512,ZZ,Region,City,4,0.001,0xE6AC380A.5A1CAD00,,-24.01,2.343,0,3,,0.49,51892,,,-10,unknown,172.28.0.1,0xE6AC3020.0C49BA80,7.749,81.612,,,4,UNIX,,10,0,0,,324,27.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv deleted file mode 100644 index dca5386d9e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","packets","size","asn","geo","region","city","naics","sic","sector","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,123,node01.example.com,2,664,64512,ZZ,Region,City,0,0,,55.33 -"2010-02-10 00:00:01",192.168.0.2,udp,123,node02.example.com,100,44000,64512,ZZ,Region,City,0,0,,3666.67 -"2010-02-10 00:00:02",192.168.0.3,udp,123,node03.example.com,100,44000,64512,ZZ,Region,City,0,0,,3666.67 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv deleted file mode 100644 index c32bc3d4d0..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","programs","mountd_port","exports","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,111,node01.example.com,portmapper,64512,ZZ,Region,City,0,0,"100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp; 100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp;",,"/mnt/export 192.168.0.0","Communications, Service Provider, and Hosting Service",148,3.70 -"2010-02-10 00:00:01",192.168.0.2,udp,111,node02.example.com,portmapper,64512,ZZ,Region,City,0,0,"100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp; 100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp;",,"/mnt/export 192.168.0.0","Communications, Service Provider, and Hosting Service",148,3.70 -"2010-02-10 00:00:02",192.168.0.3,udp,111,node03.example.com,portmapper,64512,ZZ,Region,City,0,0,"100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp; 100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp;",,"/mnt/export 192.168.0.0",Government,148,3.70 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv deleted file mode 100644 index 8c1d6f725a..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","sector","supported_protocols","protocol_error_code","protocol_error_file","protocol_error_line","protocol_error_message","protocol_error_routine","protocol_error_severity","protocol_error_severity_v","startup_error_code","startup_error_file","startup_error_line","startup_error_message","startup_error_routine","startup_error_severity","startup_error_severity_v","client_ssl","handshake","cipher_suite","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serial_number","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","cert_valid","self_signed","cert_expired","validation_level","browser_trusted","browser_error","raw_cert","raw_cert_chain" -"2010-02-10 00:00:00",192.168.0.1,tcp,5432,node01.example.com,postgres,,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",1.0-3.0,0A000,postmaster.c,1798,"unsupported frontend protocol 255.255: server supports 1.0 to 3.0",ProcessStartupPacket,FATAL,,28000,postmaster.c,1893,"no PostgreSQL user name specified in startup packet",ProcessStartupPacket,FATAL,,N,,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,,,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,,Y,,,,, -"2010-02-10 00:00:01",192.168.0.2,tcp,5432,node02.example.com,postgres,,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",1.0-3.0,0A000,postmaster.c,1798,"unsupported frontend protocol 255.255: server supports 1.0 to 3.0",ProcessStartupPacket,FATAL,,28000,postmaster.c,1893,"no PostgreSQL user name specified in startup packet",ProcessStartupPacket,FATAL,,N,,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,,,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,,Y,,,,, -"2010-02-10 00:00:02",192.168.0.3,tcp,5432,node03.example.com,postgres,,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",1.0-3.0,0A000,postmaster.c,1798,"unsupported frontend protocol 255.255: server supports 1.0 to 3.0",ProcessStartupPacket,FATAL,,28000,postmaster.c,1893,"no PostgreSQL user name specified in startup packet",ProcessStartupPacket,FATAL,,N,,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,,,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,,Y,,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv deleted file mode 100644 index 857699376e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","quote","asn","geo","region","city","naics","sic","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,17,node01.example.com,qotd,"_The secret of being miserable is to have leisure to bother about whether?? you are happy or not. The cure for it is occupation._?? George Bernard Shaw (1856-1950)?",64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",166,166.00 -"2010-02-10 00:00:01",192.168.0.2,udp,17,node02.example.com,qotd,"_Oh the nerves, the nerves; the mysteries of this machine called man!?? Oh the little that unhinges it, poor creatures that we are!_?? Charles Dickens (1812-70)?",64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",162,162.00 -"2010-02-10 00:00:02",192.168.0.3,udp,17,node03.example.com,qotd,"_Oh the nerves, the nerves; the mysteries of this machine called man!?? Oh the little that unhinges it, poor creatures that we are!_?? Charles Dickens (1812-70)?",64512,ZZ,Region,City,0,0,,162,162.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv deleted file mode 100644 index c9fb18896e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","version_field_1","version_field_2","version_field_3","version_field_4" -"2022-01-10 14:31:17","176.255.0.0","udp",443,"test1.example.com","quic",5607,"UK","LONDON","LONDON",517311,,"Q050",,"Q046","Q043" -"2022-01-10 14:31:17","24.244.0.0","udp",443,,"quic",6327,"CA","SASKATCHEWAN","MEACHAM",517311,,"Q050","Q046",,"Q043" -"2022-01-10 14:31:17","23.60.0.0","udp",443,"test3.example.com","quic",20940,"JP","OSAKA","OSAKA",517919,,,"Q050","Q046","Q043" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv deleted file mode 100644 index 76b388acaa..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv +++ /dev/null @@ -1,10 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic" -"2020-07-06 13:55:26","74.101.218.75","tcp",4899,"static-74-101-218-75.nycmny.fios.verizon.net","radmin","Radmin (Details Unknown)",701,"US","NEW YORK","BROOKLYN",517312, -"2020-07-06 13:55:27","192.162.189.171","tcp",4899,"rubin.an.ru","radmin","Radmin v3.X Radmin Authentication",56618,"RU","MURMANSKAYA OBLAST","MURMANSK",0, -"2020-07-06 13:55:27","111.197.143.69","tcp",4899,,"radmin","Radmin (Details Unknown)",4808,"CN","BEIJING SHI","BEIJING",517311, -"2020-07-06 13:55:27","121.147.215.220","tcp",4899,,"radmin","Radmin v3.X Radmin Authentication",4766,"KR","GWANGJU-GWANGYEOKSI","DAEIN-DONG",517311, -"2020-07-06 13:55:27","121.147.215.178","tcp",4899,,"radmin","Radmin v3.X Radmin Authentication",4766,"KR","GWANGJU-GWANGYEOKSI","DAEIN-DONG",517311, -"2020-07-06 13:55:27","183.230.5.219","tcp",4899,,"radmin","Radmin v3.X Radmin Authentication",9808,"CN","CHONGQING SHI","CHONGQING",517312, -"2020-07-06 13:55:27","85.93.154.74","tcp",4899,,"radmin","Radmin v3.X Radmin Authentication",34300,"RU","MOSKVA","MOSCOW",0, -"2020-07-06 13:55:27","81.246.135.247","tcp",4899,"247.135-246-81.adsl-dyn.isp.belgacom.be","radmin","Radmin v3.X Radmin Authentication",5432,"BE","ANTWERPEN","BRASSCHAAT",517311, -"2020-07-06 13:55:27","46.27.146.22","tcp",4899,"static-22-146-27-46.ipcom.comunitel.net","radmin","Radmin v3.X Radmin Authentication",12430,"ES","LAS PALMAS","LAS PALMAS DE GRAN CANARIA",517312, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv.license deleted file mode 100644 index 833024a759..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2020 sinus-x -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv deleted file mode 100644 index 4bac90f199..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","port","hostname","tag","handshake","asn","geo","region","city","rdp_protocol","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","naics","sic","sector","tlsv13_support","tlsv13_cipher","cve20190708_vulnerable","bluekeep_vulnerable","jarm" -"2019-09-04 15:45:51","198.123.245.178",5678,,"rdp",,5678,"AA","LOCATION","LOCATION","RDP",5678,"KABESRV.KABE.local","KABESRV.KABE.local","2019-04-29 02:22:06","2019-10-29 02:22:06","EC:BB:4D:DB:9F:0C:D3:FF:5B:49:EA:B1:56:62:B6:A7:5D:60:54:42","1EF2B37AF850C9BF4E88F18177001D6B",2,"sha256WithRSAEncryption","rsaEncryption","B7:C9:F4:07:D5:C0:75:1D:EA:0C:40:E7:26:39:C2:30:C6:13:83:7E:18:46:D8:E9:4C:45:3F:88:1B:0B:70:76","08:AC:75:FA:EB:A3:2B:44:15:DE:6D:A7:0B:C0:AE:17:94:F3:55:D9:EC:70:AC:5B:B7:94:79:F0:D7:84:83:89:CB:A9:11:E0:08:D7:54:4D:33:85:89:D2:A8:DD:9D:15:F4:CC:95:DE:6A:E3:DF:6B:FA:8B:27:E3:DA:16:AF:0A","BC:6E:C3:E2:98:22:EC:BA:5B:30:E2:53:FD:4A:9D:FF",517311,0,,,,"N","N" -"2019-09-04 15:45:51","198.123.245.233",5678,,"rdp",,5678,"AA","LOCATION","LOCATION","RDP",5678,"RAMBLA01.rambla.local","RAMBLA01.rambla.local","2019-04-16 06:15:20","2019-10-16 06:15:20","7A:67:1F:F8:87:C6:B0:AC:A9:84:15:B7:40:EC:CB:19:AA:E3:19:52","3FF3EBC5CF154BA54D128A8548C8AAF5",2,"sha1WithRSAEncryption","rsaEncryption","8F:CD:7D:C4:80:2D:8D:9B:06:A0:40:18:9F:ED:73:7A:BA:83:55:BE:1B:56:83:A2:97:DF:BB:B4:06:57:CB:F1","E8:9B:9A:93:69:B4:58:01:D8:46:C2:DC:01:20:1E:DD:93:E1:EB:E3:9D:6B:65:A0:C5:00:6C:A4:44:08:FE:A4:A6:19:FF:55:79:F2:AA:61:68:C8:1C:B0:CE:78:EB:84:DD:29:9D:64:2F:4E:25:31:3A:6C:B8:02:C9:AF:F5:1F","38:73:6A:B3:AA:41:69:C9:BA:E7:3D:D7:40:16:F8:AA",517311,0,"Information Technology",,,"N","N" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv deleted file mode 100644 index 73d0d55efd..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sessionid","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,3389,node01.example.com,rdpeudp,64512,ZZ,Region,City,0,0,05b28c0c,1232,77.00 -"2010-02-10 00:00:01",192.168.0.2,udp,3389,node02.example.com,rdpeudp,64512,ZZ,Region,City,0,0,053d355f,1232,77.00 -"2010-02-10 00:00:02",192.168.0.3,udp,3389,node03.example.com,rdpeudp,64512,ZZ,Region,City,0,0,0567a8cb,1232,77.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv deleted file mode 100644 index dc9760cf2d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv +++ /dev/null @@ -1,94 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","git_sha1","git_dirty_flag","build_id","mode","os","architecture","multiplexing_api","gcc_version","process_id","run_id","uptime","connected_clients","sector" -"2016-07-24 00:42:33","198.51.100.152","tcp",6379,,"redis","2.8.19",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"26069fb482f6334b","standalone","Linux 3.2.0-4-amd64 x86_64",,"epoll","4.7.2",2127,"d440b0b2fb3d1db655ad607e11e6f38011a0f599",27946314,50, -"2016-07-24 00:42:43","198.51.100.67","tcp",6379,"198-51-100-67.example.net","redis","3.0.6",12586,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",310556,25376, -"2016-07-24 00:42:43","198.51.100.125","tcp",6379,"198-51-100-125.example.net","redis","2.8.17",8972,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"5937320cdd80c1e4","standalone","Linux 2.6.32-5-amd64 x86_64",,"epoll","4.9.2",11573,"0d58143df099738a7ce9330ee5ec2367d11b1187",25888041,4, -"2016-07-24 00:42:43","198.51.100.203","tcp",6379,"198-51-100-203.example.net","redis","2.8.4",31103,"DE","THURINGEN","ERFURT",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.13.0-83-generic x86_64",,"epoll","4.8.2",3847,"4f7765dee91d8c4b1b24604cc5f0c29fca1a4f32",3068554,38, -"2016-07-24 00:42:43","198.51.100.240","tcp",6379,"198-51-100-30.example.net","redis","3.0.7",20473,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"869e89100d5ea8c2","standalone","Linux 3.13.0-87-generic x86_64",,"epoll","4.8.4",1011,"864c8d7df1e72c662a4edd77b6df6cd30161af6e",2476542,2,"Information Technology" -"2016-07-24 00:42:49","198.51.100.69","tcp",6379,"198-51-100-69.example.net","redis","3.0.6",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"315c8c74805fca88","standalone","Linux 3.2.0-98-generic x86_64",,"epoll","4.6.3",28961,"bc705102c854ea1818213e4740a3c6fd9b9f1716",4633191,1, -"2016-07-24 00:42:53","198.51.100.50","tcp",6379,"198-51-100-50.example.net","redis","3.0.7",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"6afb1e1f0d80abd0","standalone","Linux 2.6.32-5-amd64 x86_64",,"epoll","4.4.5",1717,"f729595b3642b48f3ac9e098bcccab1d6ef82e3e",6345372,3, -"2016-07-24 00:43:49","198.51.100.113","tcp",6379,,"redis","3.0.6",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",310623,24628, -"2016-07-24 00:43:49","198.51.100.228","tcp",6379,"198-51-100-131.example.net","redis","2.8.210",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"83ad777ec89a946b","standalone","Windows",,"winsock_IOCP",,1948,"f5d6ad26e423039636afaf3918ee7e6a7e0b5b68",2214134,4,"Information Technology" -"2016-07-24 00:43:59","198.51.100.155","tcp",6379,,"redis","3.0.7",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"f09a0843cc9876c3","standalone","Linux 3.13.0-79-generic x86_64",,"epoll","4.9.2",1,"5f4f5b7158f928cc96e3ae6af6092a163ace15eb",2897902,24, -"2016-07-24 00:43:59","198.51.100.171","tcp",6379,,"redis","3.0.6",12586,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",310633,25031, -"2016-07-24 00:44:09","198.51.100.230","tcp",6379,"198-51-100-230.example.net","redis","2.8.11",8560,"DE","BADEN-WURTTEMBERG","KARLSRUHE",0,0,00000000,0,"f26bfdf4b8265fc","standalone","Linux 3.8.0-29-generic x86_64",,"epoll","4.6.3",14551,"0d001175cf26cee88486d814b4f0c972a5aa89b9",21038337,9, -"2016-07-24 00:44:09","198.51.100.182","tcp",6379,"198-51-100-182.example.net","redis","3.0.7",197540,"DE","BADEN-WURTTEMBERG","KARLSRUHE",0,0,00000000,0,"fd24f54fec00684b","standalone","Linux 3.13.0-85-generic x86_64",,"epoll","4.8.4",949,"b11fdf2b95251b8e6c3e9e782409ef82fc8b89aa",8643389,11, -"2016-07-24 00:44:10","198.51.100.23","tcp",6379,"198-51-100-116.example.net","redis","2.8.4",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 4.2.0-27-generic x86_64",,"epoll","4.8.2",335,"90079d58e970a1ae94aa91bc0ea0236a0e55269c",4930922,2,"Information Technology" -"2016-07-24 00:44:19","198.51.100.51","tcp",6379,"198-51-100-51.example.net","redis","3.0.6",12586,"DE","BERLIN","BERLIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",310652,26257, -"2016-07-24 00:44:22","198.51.100.88","tcp",6379,,"redis","3.0.6",13301,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",310656,26371, -"2016-07-24 00:44:22","198.51.100.107","tcp",6379,"octopus-dev","redis","2.8.14",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"78be6d5e32e34139","standalone","Linux 2.6.32-042stab108.2 x86_64",,"epoll","4.8.2",21205,"b98a41b6ea690c207527587f60bff1f1d24236b4",9364864,4, -"2016-07-24 00:44:22","198.51.100.75","tcp",6379,,"redis","3.0.0",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"2b5201a6bfd5f75e","standalone","Linux 3.11.0-19-generic x86_64",,"epoll","4.8.2",832,"2bdcda8b3b59cef244785b58935d68daf48645be",6745479,5, -"2016-07-24 00:44:25","198.51.100.12","tcp",6379,,"redis","3.0.6",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.13.0-79-generic x86_64",,"epoll","4.8.4",899,"94550e510bf770aa315cc3983ce9958853c77cfe",7816856,9, -"2016-07-24 00:44:27","198.51.100.13","tcp",6379,"198-51-100-13.example.net","redis","3.0.7",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"6f8b503a2787e3a6","standalone","Linux 4.4.5-15.26.amzn1.x86_64 x86_64",,"epoll","4.9.2",1,"e050f40e755a739ffecdb2468e1333f371e2abca",7124048,6,"Communications" -"2016-07-24 00:44:29","198.51.100.12","tcp",6379,"198-51-100-12.example.net","redis","2.8.3",8972,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"992c97be25a6b6d2","standalone","Linux 2.6.32-042stab111.12 x86_64",,"epoll","4.4.5",12340,"d7cda18212cf4bcdfd7c42fff33e506a4e9a2614",16874891,8, -"2016-07-24 00:44:38","198.51.100.66","tcp",6379,"198-51-100-66.example.net","redis","3.2.1",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"ed627d97d5dc311e","standalone","Linux 4.2.0-19-generic x86_64",,"epoll","4.9.2",1,"4a6beb721ddbaa411f53e5268e6112127903cae3",2029470,3,"Chemical" -"2016-07-24 00:44:38","198.51.100.170","tcp",6379,,"redis","3.0.6",8881,"DE","SACHSEN","RADEBEUL",0,0,00000000,0,"1b14d17ce6fea422","standalone","Linux 4.2.6-1-pve x86_64",,"epoll","4.9.2",728,"c423ba856285690a2fae350b03514cec80db9d5e",1679635,1, -"2016-07-24 00:44:38","198.51.100.67","tcp",6379,"198-51-100-67.example.net","redis","2.8.23",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"8e819a849ea2d7f8","standalone","Linux 4.2.0-23-generic x86_64",,"epoll","4.9.2",1,"7ee1dc403540ff4d1fc0a80d9f0b2910857b6c1b",9451832,68,"Information Technology" -"2016-07-24 00:44:44","198.51.100.238","tcp",6379,,"redis","2.8.4",44066,"DE","BAYERN","MUNICH",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 2.6.32-19-pve x86_64",,"epoll","4.8.2",2207,"6a079396cc44c1aca745edab13f4014c394da3ab",10338949,3, -"2016-07-24 00:44:44","198.51.100.84","tcp",6379,"198-51-100-84.example.net","redis","3.0.2",51862,"DE","BADEN-WURTTEMBERG","KARLSRUHE",0,0,00000000,0,"4795df119e2d77fe","standalone","Linux 3.13.0-91-generic x86_64",,"epoll","4.7.2",1,"c120481a551c232b8e1a9cff20d9e0968a402dd9",1040551,7, -"2016-07-24 00:44:44","198.51.100.23","tcp",6379,"198-51-100-23.example.net","redis","3.0.6",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"98c227055d7fa7b6","standalone","Linux 3.10.0-327.10.1.el7.x86_64 x86_64",,"epoll","4.8.5",35198,"424b15e04ce09f26299ff19b252a920916d4e4be",8875355,2, -"2016-07-24 00:44:47","198.51.100.160","tcp",6379,"198-51-100-160.example.net","redis","2.8.210",6724,"DE","BERLIN","BERLIN",0,0,00000000,0,"83ad777ec89a946b","standalone","Windows",,"winsock_IOCP",,2284,"9bde76afda6f81acfb241ea5ee3a9e878ad53881",742778,2, -"2016-07-24 00:44:47","198.51.100.111","tcp",6379,"198-51-100-98.example.net","redis","3.2.1",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"e19bb8c3d1c28291","standalone","Linux 3.10.0-327.22.2.el7.x86_64 x86_64",,"epoll","5.3.0",1,"c951371f430c1d94299bfc93759f6940d8bfce78",208557,2, -"2016-07-24 00:44:48","198.51.100.227","tcp",6379,"198-51-100-227.example.net","redis","3.0.6",12586,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",310681,26496, -"2016-07-24 00:44:54","198.51.100.18","tcp",6379,"198-51-100-18.example.net","redis","2.8.9",8972,"DE","NORDRHEIN-WESTFALEN","COLOGNE",0,0,00000000,0,"52c7b9284559eb20","standalone","Linux 2.6.32-5-amd64 x86_64",,"epoll","4.4.5",31887,"e5b1da35862482c4df8d4fce635ec89a36476a4d",14393072,6, -"2016-07-24 00:44:54","198.51.100.248","tcp",6379,"198-51-100-248.example.net","redis","3.0.6",12586,"DE","BERLIN","BERLIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",310687,26112, -"2016-07-24 00:44:57","198.51.100.228","tcp",6379,"198-51-100-228.example.net","redis","3.0.7",8972,"DE","NORDRHEIN-WESTFALEN","COLOGNE",0,0,00000000,0,"5e03212a543f54f8","standalone","Linux 3.13.0-042stab116.1 x86_64",,"epoll","4.8.4",719,"537e3e824a45414c3199ef20201b4362b752eeb5",1263367,2, -"2016-07-24 00:45:04","198.51.100.227","tcp",6379,"198-51-100-227.example.net","redis","2.8.12",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"ff040dde4a39b4ff","standalone","Windows",,"winsock_IOCP","0.0.0",1872,"c78751c65793a9a72f6fb0318efa532eb4fc87de",277953,18,"Chemical" -"2016-07-24 00:45:07","198.51.100.132","tcp",6379,,"redis","3.0.5",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"30405cba8f6c2d55","standalone","Linux 3.13.0-57-generic x86_64",,"epoll","4.8.4",2500,"10b4084b930d5a77e5f09e89cf0b21702027bd60",10028956,695, -"2016-07-24 00:46:10","198.51.100.47","tcp",6379,"198-51-100-185.example.net","redis","3.0.7",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"6a943c0b5bf37fa1","standalone","Linux 4.4.0-24-generic x86_64",,"epoll","5.3.1",1023,"de9c9c0da3d971f689bd7366c1edc93a00fd1506",2791106,1, -"2016-07-24 01:23:27","198.51.100.246","tcp",6379,"198-51-100-190.example.net","redis","2.8.19",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"665519ce00ddac9b","standalone","Linux 2.6.32-43-pve x86_64",,"epoll","4.9.2",2310,"94595838457eddb30a60184a9db66212268e6f82",9481199,4, -"2016-07-24 01:23:29","198.51.100.187","tcp",6379,"198-51-100-63.example.net","redis","2.8.19",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"c0359e7aa3798aa2","standalone","Linux 3.10.0-229.7.2.el7.x86_64 x86_64",,"epoll","4.8.3",14050,"e67a19de4bd2dc485b98ca353eb6fdc65e8fed4a",14051444,10, -"2016-07-24 01:23:29","198.51.100.228","tcp",6379,"198-51-100-228.example.net","redis","2.8.4",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.13.0-32-generic x86_64",,"epoll","4.8.2",22837,"daf5dba760d3db12716c6dc1d0bfe6d5e7b33749",10916038,8, -"2016-07-24 01:23:43","198.51.100.180","tcp",6379,"198-51-100-180.example.net","redis","3.2.1",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"ed627d97d5dc311e","standalone","Linux 4.2.0-19-generic x86_64",,"epoll","4.9.2",1,"569881874d8d5e1508d584a3fd9dff0ac3515839",1677711,1,"Chemical" -"2016-07-24 01:23:56","198.51.100.5","tcp",6379,"198-51-100-207.example.net","redis","3.0.7",20473,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"869e89100d5ea8c2","standalone","Linux 3.13.0-87-generic x86_64",,"epoll","4.8.4",1011,"864c8d7df1e72c662a4edd77b6df6cd30161af6e",2479015,2,"Information Technology" -"2016-07-24 01:24:03","198.51.100.226","tcp",6379,"198-51-100-226.example.net","redis","3.0.5",8972,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"b33bc3e2f8ad13f6","standalone","Linux 2.6.32-573.12.1.el6.x86_64 x86_64",,"epoll","4.4.7",1801,"7f4bb7ed008cdbd665672e88d57fc55616b6dbf2",13189200,9, -"2016-07-24 01:24:14","198.51.100.253","tcp",6379,"198-51-100-136.example.net","redis","2.8.4",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.19.0-39-generic x86_64",,"epoll","4.8.2",28272,"13a889aa846c6302dc8f5453e35e051a6f359e9a",14046610,185, -"2016-07-24 01:24:28","198.51.100.206","tcp",6379,,"redis","3.0.6",13301,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",313061,26695, -"2016-07-24 01:24:35","198.51.100.73","tcp",6379,,"redis","3.0.2",28753,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"6be7fc9e6b88f79","standalone","Linux 2.6.32-71.29.1.el6.x86_64 x86_64",,"epoll","4.4.7",811,"bbd4dce247ab51d029a64243810aeb900e00d1d6",10082205,15, -"2016-07-24 01:24:35","198.51.100.83","tcp",6379,"198-51-100-174.example.net","redis","3.2.1",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"7e7b61a55b95e8e7","standalone","Linux 4.2.0-41-generic x86_64",,"epoll","4.8.4",1076,"48f5f780ca53553fc4c0bbdbb32a5cb06a0551cd",814255,88,"Information Technology" -"2016-07-24 01:25:30","198.51.100.182","tcp",6379,,"redis","3.0.7",31400,"DE","RHEINLAND-PFALZ","FREINSHEIM",0,0,00000000,0,"d9ceac045f7983a9","standalone","FreeBSD 10.1-RELEASE-p26 amd64",,"kqueue","4.2.1",957,"48f37d15b3f5169f11aa5d7194fdfccc7f8df20b",6364747,1, -"2016-07-24 01:25:30","198.51.100.211","tcp",6379,"198-51-100-118.example.net","redis","2.8.17",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"e4968abcd4b78b2e","standalone","Linux 3.13.0-36-generic x86_64",,"epoll","4.8.2",1643,"665565b1b1fb6e773039707a0f680bbc417186be",20180649,4,"Information Technology" -"2016-07-24 01:25:35","198.51.100.249","tcp",6379,,"redis","3.0.2",28753,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"6be7fc9e6b88f79","standalone","Linux 2.6.32-71.29.1.el6.x86_64 x86_64",,"epoll","4.4.7",811,"bbd4dce247ab51d029a64243810aeb900e00d1d6",10082265,15, -"2016-07-24 01:25:40","198.51.100.55","tcp",6379,,"redis","3.2.1",3320,"DE","NORDRHEIN-WESTFALEN","SOLINGEN",518210,737415,00000000,0,"e19bb8c3d1c28291","standalone","Linux 4.4.0-24-generic x86_64",,"epoll","5.3.0",1,"49687ba2a5be7f7b6cdf0c837e06307442f6a369",494739,1, -"2016-07-24 01:25:42","198.51.100.62","tcp",6379,"198-51-100-62.example.net","redis","3.0.7",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"2b87841ee28adfc3","standalone","Linux 3.13.0-042stab113.11 x86_64",,"epoll","4.8.4",525,"4045d68fd2e59a1135bb303206d7cd0439ba7ffd",6971251,4, -"2016-07-24 01:25:55","198.51.100.127","tcp",6379,"198-51-100-25.example.net","redis","2.8.4",20473,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.13.0-87-generic x86_64",,"epoll","4.8.2",11492,"3de3e977405eef9392a77db4a50d99a5caa2f2d9",2194103,3,"Information Technology" -"2016-07-24 01:26:08","198.51.100.92","tcp",6379,"198-51-100-92.example.net","redis","2.8.10",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"5fce0c4aab65e01","standalone","Linux 2.6.32-042stab113.11 x86_64",,"epoll","4.6.3",490,"15abe68a10b011972f50d0abb3bb18f1735994a5",7505621,4, -"2016-07-24 01:26:17","198.51.100.218","tcp",6379,,"redis","3.0.7",34011,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"dc142e699f115c40","standalone","Linux 3.2.60-grsec-x86_64 x86_64",,"epoll","4.7.3",8006,"53a093bd4d0a7b72b2d084ec3767d23b18b8b947",4024979,7, -"2016-07-24 01:26:29","198.51.100.168","tcp",6379,"198-51-100-168.example.net","redis","3.0.6",51167,"DE","BAYERN","MUNICH",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.13.0-37-generic x86_64",,"epoll","4.8.4",1279,"8218bd77a0dcb0e00bd77dbb9478115757c70ba5",2405965,1, -"2016-07-24 01:26:29","198.51.100.155","tcp",6379,"198-51-100-155.example.net","redis","3.0.7",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"d9155128f7b25ea0","standalone","Linux 3.19.0-25-generic x86_64",,"epoll","4.8.4",27030,"0ede623cb268643672abc04d0267f684a5ee7a0d",6880190,5,"Information Technology" -"2016-07-24 01:26:34","198.51.100.185","tcp",6379,,"redis","2.8.4",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.13.0-43-generic x86_64",,"epoll","4.8.2",1196,"ae80fcbb54017f521212caf257418885cd6836a0",5412584,5, -"2016-07-24 01:26:34","198.51.100.1","tcp",6379,"198-51-100-1.example.net","redis","3.2.0",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"5382f69a4e75566b","standalone","Linux 4.2.0-19-generic x86_64",,"epoll","4.9.2",1,"ff8990f109ff5b2d4e0eee47e5ebc66acc43f9e3",4615889,4,"Chemical" -"2016-07-24 01:26:39","198.51.100.51","tcp",6379,"198-51-100-164.example.net","redis","3.0.0",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"9526f4809583faaa","standalone","Linux 2.6.32-042stab113.21 x86_64",,"epoll","4.4.5",14528,"d7271feff55175f434ace92d199f332ad35776a9",7440370,16, -"2016-07-24 01:26:44","198.51.100.138","tcp",6379,,"redis","3.0.6",12586,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",313197,26452, -"2016-07-24 01:26:47","198.51.100.16","tcp",6379,,"redis","2.8.17",25074,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"5937320cdd80c1e4","standalone","Linux 2.6.32-43-pve x86_64",,"epoll","4.9.2",266,"e1d403f2daff849a64b178f74c672db6712f217a",351253,1, -"2016-07-24 01:26:54","198.51.100.171","tcp",6379,"198-51-100-171.example.net","redis","3.0.6",12586,"DE","BERLIN","BERLIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",313207,26601, -"2016-07-24 01:27:14","198.51.100.89","tcp",6379,"198-51-100-89.example.net","redis","3.0.6",12586,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",313227,26358, -"2016-07-24 01:27:24","198.51.100.65","tcp",6379,,"redis","3.0.7",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"869e89100d5ea8c2","standalone","Linux 3.13.0-85-generic x86_64",,"epoll","4.8.4",21575,"3ec40168300e14f5776d82a48ba873a3999caec1",1897530,1, -"2016-07-24 01:27:24","198.51.100.248","tcp",6379,"198-51-100-248.example.net","redis","3.0.6",12586,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",313237,25902, -"2016-07-24 01:27:33","198.51.100.17","tcp",6379,,"redis","2.8.17",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"43dd9e14444e6aea","standalone","Linux 3.16.0-4-amd64 x86_64",,"epoll","4.9.2",556,"3e8fc2878511cc72f79b765fca86cefe21346912",2607965,72, -"2016-07-24 01:27:33","198.51.100.134","tcp",6379,"198-51-100-134.example.net","redis","3.0.7",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"6f8b503a2787e3a6","standalone","Linux 3.16.0-4-amd64 x86_64",,"epoll","4.9.2",1,"b85b2419cf35dd81ff5b9ba6e8bf802cf1d439f6",128621,33, -"2016-07-24 01:27:42","198.51.100.186","tcp",6379,"198-51-100-186.example.net","redis","2.8.13",34011,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"d588bf57ea0dfa69","standalone","Linux 4.4.8-jb1 i686",,"epoll","4.6.3",2460,"97b8d49e62d340d94a38c96c5104abfcacbfa4cb",181557,1, -"2016-07-24 01:27:42","198.51.100.21","tcp",6379,"198-51-100-21.example.net","redis","2.8.19",34011,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"920d7eda78149e99","standalone","Linux 4.4.8-x86_64-jb1 x86_64",,"epoll","4.7.2",3722,"74dfd8a7d87cbb9ecc590ceafd438c85d5073903",183984,1, -"2016-07-24 01:27:43","198.51.100.128","tcp",6379,"198-51-100-203.example.net","redis","3.0.5",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"f3bd5bc2b8b4c486","standalone","Linux 2.6.32-573.8.1.el6.x86_64 x86_64",,"epoll","4.4.7",1968,"0d92b1323fea791ba4b0a43435a156b6ec0aac1c",2967611,2,"Information Technology" -"2016-07-24 01:27:44","198.51.100.216","tcp",6379,"198-51-100-229.example.net","redis","2.8.4",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.16.0-30-generic x86_64",,"epoll","4.8.2",1470,"e76cd0cf25eec5d254c880965189ae011a119220",302420,1, -"2016-07-24 01:27:53","198.51.100.242","tcp",6379,"198-51-100-242.example.net","redis","3.0.2",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"6a04b5ede30cd4cd","standalone","Linux 3.13.0-32-generic x86_64",,"epoll","4.8.4",29725,"1b7e8dc53dec8fb29a8a2d76f516fd3dcb8df652",5815739,7, -"2016-07-24 01:27:53","198.51.100.54","tcp",6379,"198-51-100-54.example.net","redis","2.8.4",8560,"DE","BADEN-WURTTEMBERG","KARLSRUHE",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.13.0-91-generic x86_64",,"epoll","4.8.2",2903,"0e02514dec6031018eb148b13a4a9639cab3e8aa",905886,1, -"2016-07-24 01:27:54","198.51.100.225","tcp",6379,"198-51-100-225.example.net","redis","3.0.6",12586,"DE","BERLIN","BERLIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",313267,25281, -"2016-07-24 01:27:57","198.51.100.38","tcp",6379,"198-51-100-38.example.net","redis","3.0.5",6724,"DE","BERLIN","BERLIN",0,0,00000000,0,"3b863f97501297e9","standalone","Linux 3.13.0-042stab111.12 x86_64",,"epoll","4.8.4",2088,"31a8cececad2e4a33310a741143d85cdef3479b4",11906868,10, -"2016-07-24 01:27:58","198.51.100.22","tcp",6379,"198-51-100-22.example.net","redis","2.8.9",51167,"DE","BAYERN","MUNICH",0,0,00000000,0,"2ac6afaedfd3ea15","standalone","Linux 3.13.0-86-generic x86_64",,"epoll","4.8.4",9082,"8e5d9d74c86a9f148a7012733eb52a21938c3c04",5833880,5, -"2016-07-24 01:28:05","198.51.100.106","tcp",6379,"198-51-100-106.example.net","redis","2.8.19",36351,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"9968db13395be4aa","standalone","Windows",,"winsock_IOCP","0.0.0",4372,"89716352a10cd53b5c10e6d5e6cd1d46f5f53a30",485031,4,"Information Technology" -"2016-07-24 01:28:06","198.51.100.130","tcp",6379,"198-51-100-130.example.net","redis","2.8.3",51167,"DE","BAYERN","MUNICH",0,0,00000000,0,"542faa6f897d2236","standalone","Linux 2.6.32-573.3.1.el6.x86_64 x86_64",,"epoll","4.4.7",25531,"9d7606a883f764e744d766b7bf0036ba61f7fb6e",496133,5, -"2016-07-24 01:28:08","198.51.100.37","tcp",6379,"198-51-100-37.example.net","redis","2.8.23",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"50630e46be5feb4f","standalone","Linux 3.13.0-74-generic x86_64",,"epoll","4.9.2",1,"62d16be721c3c62d6c4d080a9bdbe9502c57ca86",3481683,9,"Communications" -"2016-07-24 01:28:32","198.51.100.148","tcp",6379,"198-51-100-148.example.net","redis","3.0.5",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"83dc15dcf8ee3eb8","standalone","Linux 4.1.7-15.23.amzn1.x86_64 x86_64",,"epoll","4.8.3",2304,"883accf76dc364c60902b4eab7861dd1a7eac71d",10981957,10,"Communications" -"2016-07-24 01:28:49","198.51.100.247","tcp",6379,"198-51-100-247.example.net","redis","3.0.7",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"3e971e94fbe2eaa6","standalone","Linux 3.2.0-4-amd64 x86_64",,"epoll","4.7.2",2535,"d223aab0621cdd2e4ab752978ad3009ad3814d8b",7715188,57, -"2016-07-24 02:08:46","198.51.100.220","tcp",6379,"198-51-100-220.example.net","redis","3.0.6",51167,"DE","BAYERN","MUNICH",0,0,00000000,0,"1f8e4c92f1ca309","standalone","Linux 3.13.0-74-generic x86_64",,"epoll","4.8.4",3355,"dd517756bb6ee81e1929fa605972318b2baebb93",5211978,10, -"2016-07-24 02:08:46","198.51.100.239","tcp",6379,"198-51-100-239.example.net","redis","2.8.23",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"83a5616190c5a1aa","standalone","Linux 3.16.0-4-amd64 x86_64",,"epoll","4.9.2",711,"4117960b13fa313b823c79b0e9f188d8ec6aa3ac",10156283,6, -"2016-07-24 02:08:50","198.51.100.233","tcp",6379,,"redis","2.8.11",8560,"DE","BADEN-WURTTEMBERG","KARLSRUHE",0,0,00000000,0,"f26bfdf4b8265fc","standalone","Linux 3.8.0-29-generic x86_64",,"epoll","4.6.3",14551,"0d001175cf26cee88486d814b4f0c972a5aa89b9",21043417,9, -"2016-07-24 02:08:51","198.51.100.208","tcp",6379,"198-51-100-181.example.net","redis","3.0.6",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 4.2.0-38-generic x86_64",,"epoll","4.8.4",809,"14c5ec7f9669e42ea45a40ff26a6501d593695c0",2405839,19, -"2016-07-24 02:08:51","198.51.100.60","tcp",6379,"198-51-100-60.example.net","redis","3.0.7",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"4ed99bd9c45dfc14","standalone","Linux 3.13.0-57-generic x86_64",,"epoll","4.8.4",1144,"9e28c29ff40017e2fbe32fb97755caf801f95793",843538,2, -"2016-07-24 02:08:51","198.51.100.107","tcp",6379,"198-51-100-39.example.net","redis","3.2.0",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"82b2619163aabc80","standalone","Linux 4.2.0-25-generic x86_64",,"epoll","4.9.2",1,"98f6640bbde04b1214730937212e1fd4e58d03a8",2195657,12, -"2016-07-24 02:08:54","198.51.100.31","tcp",6379,,"redis","2.8.4",6724,"DE","BERLIN","BERLIN",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.13.0-042stab111.12 x86_64",,"epoll","4.8.2",1112,"9c4e55b5ebd06045c5d89d43fa202e219ec8b42c",8839783,7, -"2016-07-24 02:08:56","198.51.100.221","tcp",6379,,"redis","3.0.7",44066,"DE","BAYERN","MUNICH",0,0,00000000,0,"49f951dce0725d71","standalone","FreeBSD 10.0-RELEASE-p7 amd64",,"kqueue","4.2.1",932,"28c6af3c4dedcd9b71cf51a7ebc4e84899196aee",8000949,1, -"2016-07-24 02:09:01","198.51.100.155","tcp",6379,"198-51-100-155.example.net","redis","2.8.22",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"fcdf45e47686c89b","standalone","Linux 3.13.0-57-generic x86_64",,"epoll","4.8.4",7,"946ec6b96fe9925d2b677ce02b6c56097c5e69a8",8449694,6, -"2016-07-24 02:09:02","198.51.100.219","tcp",6379,"198-51-100-219.example.net","redis","2.8.4",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.13.0-74-generic x86_64",,"epoll","4.8.2",1047,"9b83d6a6e7a6ffe50e75dac88cdc5e06f6203c9c",966148,1,"Chemical" -"2016-07-24 02:09:02","198.51.100.193","tcp",6379,"198-51-100-193.example.net","redis","3.0.7",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"fd640d8ef55a22dd","standalone","Linux 4.2.0-42-generic x86_64",,"epoll","4.8.4",1397,"ed5ec17d78d089af53afd4abc339f7decf4641d4",651175,2,"Information Technology" -"2016-07-24 02:09:20","198.51.100.120","tcp",6379,"198-51-100-120.example.net","redis","3.2.1",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"ed627d97d5dc311e","standalone","Linux 3.16.0-4-amd64 x86_64",,"epoll","4.9.2",1,"f524508ad29334eee2fcf7bdda5c80b9f99d3dfe",987580,167, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv deleted file mode 100644 index a61e4573ec..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","module","motd","has_password" -"2010-02-10 00:00:00",192.168.0.1,tcp,873,node01.example.com,rsync,64512,ZZ,Region,City,0,0,"system|Backup system;system_full|Backup full system;mysql|Backup virtual mysql;netadmin|Backup virtual netadmin;",,N -"2010-02-10 00:00:01",192.168.0.2,tcp,873,node02.example.com,rsync,64512,ZZ,Region,City,0,0,"system|Backup system;system_full|Backup full system;mysql|Backup virtual mysql;netadmin|Backup virtual netadmin;",,N -"2010-02-10 00:00:02",192.168.0.3,tcp,873,node03.example.com,rsync,64512,ZZ,Region,City,0,0,"system|Backup system;system_full|Backup full system;mysql|Backup virtual mysql;netadmin|Backup virtual netadmin;",,N diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv deleted file mode 100644 index ee0a625e55..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","sip","sip_code","sip_reason","user_agent","sip_via","sip_to","sip_from","content_length","content_type","server","contact","cseq","call_id","allow","amplification","response_size" -"2010-02-10 00:00:00",192.168.0.1,udp,5060,node01.example.com,sip,64512,ZZ,Region,City,SIP/2.0,489,"Event Package Not Supported",,,,,0,,,,,,"INVITE,ACK,BYE,CANCEL,REGISTER",15.57,109 -"2010-02-10 00:00:01",192.168.0.2,udp,5060,node02.example.com,sip,64512,ZZ,Region,City,SIP/2.0,400,"Bad Request",,,,,364,text/plain,,,,,,62.57,438 -"2010-02-10 00:00:02",192.168.0.3,udp,5060,node03.example.com,sip,64512,ZZ,Region,City,SIP/2.0,400,"Bad Request",,,,,0,,,,,,,6.57,46 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv.license deleted file mode 100644 index 9f58c89ef0..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2023 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv deleted file mode 100644 index 256dd78f60..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","version","function","function_text","flags","next_extension_offset","xid","language_tag_length","language_tag","error_code","error_code_text","response_size","raw_response" -"2010-02-10 00:00:00",192.168.0.1,tcp,427,node01.example.com,slp,64512,ZZ,Region,City,0,0,,2,2,"Service reply",0x0000,0,5,2,en,5,"Unsupported SLP SPI",40,MDIwMjAwMDAxNDAwMDAwMDAwMDAwMDA1MDAwMjY1NmUwMDA1MDAwMA== -"2010-02-10 00:00:01",192.168.0.2,tcp,427,node02.example.com,slp,64512,ZZ,Region,City,0,0,,2,2,"Service reply",0x0000,0,5,2,en,5,"Unsupported SLP SPI",40,MDIwMjAwMDAxNDAwMDAwMDAwMDAwMDA1MDAwMjY1NmUwMDA1MDAwMA== -"2010-02-10 00:00:02",192.168.0.3,tcp,427,node03.example.com,slp,64512,ZZ,Region,City,0,0,,2,2,"Service reply",0x0000,0,5,2,en,5,"Unsupported SLP SPI",40,MDIwMjAwMDAxNDAwMDAwMDAwMDAwMDA1MDAwMjY1NmUwMDA1MDAwMA== diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv.license deleted file mode 100644 index 9f58c89ef0..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2023 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv deleted file mode 100644 index fc7fe2fff6..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","port","hostname","tag","asn","geo","region","city","naics","sic","smb_implant","arch","key","smbv1_support","smb_major_number","smb_minor_number","smb_revision","smb_version_string" -"2010-02-10 00:00:00",192.168.0.1,445,node01.example.com,smb,64512,ZZ,Region,City,0,0,N,,,N,2,1,0,"SMB 2.1" -"2010-02-10 00:00:01",192.168.0.2,445,node02.example.com,smb,64512,ZZ,Region,City,0,0,N,,,N,2,1,0,"SMB 2.1" -"2010-02-10 00:00:02",192.168.0.3,445,node03.example.com,smb,64512,ZZ,Region,City,0,0,N,,,N,2,1,0,"SMB 2.1" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv deleted file mode 100644 index 19eb560538..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","banner" -"2021-07-08 11:58:42","1.2.3.4","tcp",25,"smtp-server.invalid","smtp;21nails",12345,"EE","HARJUMAA","TALLINN",,,"220 smtp-server.invalid ESMTP Exim 4.80 Wed, 11 Jun 2021 10:00:00 +0300|" -"2021-07-08 11:58:44","5.6.7.8","tcp",25,"smtp-out.invalid","smtp;21nails",23456,"EE","HARJUMAA","TALLINN",,,"220 smtp-out.invalid, ESMTP EXIM 4.86_2|" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv.license deleted file mode 100644 index c1900637ff..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Mikk Margus Möll -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv deleted file mode 100644 index f489261c42..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","sysdesc","sysname","asn","geo","region","city","version","naics","sic","sector","device_vendor","device_type","device_model","device_version","device_sector","tag","community","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,161,node01.example.com,"Linux localhost 3.18.20 #1 SMP Mon Jul 9 14:11:21 CST 2018 armv7l",,64512,ZZ,Region,City,2,0,0,,,,,,,snmp,public,165,1.94 -"2010-02-10 00:00:01",192.168.0.2,udp,161,node02.example.com,"RouterOS CCR1009-8G-1S-1S+",,64512,ZZ,Region,City,2,0,0,,MikroTik,router,,,consumer,"snmp,iot",public,115,1.35 -"2010-02-10 00:00:02",192.168.0.3,udp,161,node03.example.com,,,64512,ZZ,Region,City,2,0,0,,,,,,,snmp,public,85,1.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv deleted file mode 100644 index c591a5c099..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector" -"2010-02-10 00:00:00",192.168.0.1,tcp,1080,node01.example.com,socks4,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service" -"2010-02-10 00:00:01",192.168.0.2,tcp,1080,node02.example.com,socks5,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service" -"2010-02-10 00:00:02",192.168.0.3,tcp,1080,node03.example.com,socks4,64512,ZZ,Region,City,0,0,"Retail Trade" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv deleted file mode 100644 index 460be32c50..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","header","asn","geo","region","city","systime","cache_control","location","server","search_target","unique_service_name","host","nts","nt","content_type","naics","sic","sector","server_port","instance","version","updated_at","resource_identifier","amplification","response_size" -"2010-02-10 00:00:00",192.168.0.1,udp,60194,node01.example.com,ssdp,"HTTP/1.1 200 OK",64512,ZZ,Region,City,"Sun, 21 Aug 2022 09:51:13 GMT",max-age=100,http://192.168.200.254:49152/description.xml,"Linux/2.6.26, UPnP/1.0, Portable SDK for UPnP devices/1.3.1",upnp:rootdevice,uuid:28802880-2880-1880-a880-001bc502f600::upnp:rootdevice,node01.example.com,,,,0,0,Government,,,,,,3.35,325 -"2010-02-10 00:00:01",192.168.0.2,udp,38732,node02.example.com,ssdp,"HTTP/1.1 200 OK",64512,ZZ,Region,City,,"max-age = 1800",http://95.160.216.14:52235/dmr/SamsungMRDesc.xml,"Linux/9.0 UPnP/1.0 PROTOTYPE/1.0",upnp:rootdevice,uuid:f144ca92-6816-94b5-b95f-b58180834044::upnp:rootdevice,node02.example.com,,,,0,0,,,,,,,2.71,263 -"2010-02-10 00:00:02",192.168.0.3,udp,57626,node03.example.com,ssdp,"HTTP/1.1 200 OK",64512,ZZ,Region,City,"Sun, 03 Jan 2016 21:37:50 GMT",max-age=1800,http://192.168.1.3:8008/ssdp/device-desc.xml,"Linux/3.10.79, UPnP/1.0, Portable SDK for UPnP devices/1.6.18",upnp:rootdevice,uuid:62fa0fc8-079d-d00f-2e22-59b49fb488f9::upnp:rootdevice,node03.example.com,,,,0,0,Government,,,,,,4.79,465 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv deleted file mode 100644 index 837adbad10..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","serverid_raw","serverid_version","serverid_software","serverid_comment","server_cookie","available_kex","available_ciphers","available_mac","available_compression","selected_kex","algorithm","selected_cipher","selected_mac","selected_compression","server_signature_value","server_signature_raw","server_host_key","server_host_key_sha256","rsa_prime","rsa_prime_length","rsa_generator","rsa_generator_length","rsa_public_key","rsa_public_key_length","rsa_exponent","rsa_modulus","rsa_length","dss_prime","dss_prime_length","dss_generator","dss_generator_length","dss_public_key","dss_public_key_length","dss_dsa_public_g","dss_dsa_public_p","dss_dsa_public_q","dss_dsa_public_y","ecdsa_curve25519","ecdsa_curve","ecdsa_public_key_length","ecdsa_public_key_b","ecdsa_public_key_gx","ecdsa_public_key_gy","ecdsa_public_key_n","ecdsa_public_key_p","ecdsa_public_key_x","ecdsa_public_key_y","ed25519_curve25519","ed25519_cert_public_key_nonce","ed25519_cert_public_key_bytes","ed25519_cert_public_key_raw","ed25519_cert_public_key_sha256","ed25519_cert_public_key_serial","ed25519_cert_public_key_type_id","ed25519_cert_public_key_type_name","ed25519_cert_public_key_keyid","ed25519_cert_public_key_principles","ed25519_cert_public_key_valid_after","ed25519_cert_public_key_valid_before","ed25519_cert_public_key_duration","ed25519_cert_public_key_sigkey_bytes","ed25519_cert_public_key_sigkey_raw","ed25519_cert_public_key_sigkey_sha256","ed25519_cert_public_key_sigkey_value","ed25519_cert_public_key_sig_raw","banner","userauth_methods","device_vendor","device_type","device_model","device_version","device_sector" -"2022-01-10 02:20:37","18.179.0.0","tcp",22,"ec2-18-179-0-0.ap-northeast-1.compute.amazonaws.com","ssh",16509,"JP","TOKYO","TOKYO",454110,,"SSH-2.0-OpenSSH_7.4","2.0","OpenSSH_7.4",,"bGjsifbPIDWT7tAu8BMjyg==","curve25519-sha256, curve25519-sha256@libssh.org, ecdh-sha2-nistp256, ecdh-sha2-nistp384, ecdh-sha2-nistp521, diffie-hellman-group-exchange-sha256, diffie-hellman-group16-sha512, diffie-hellman-group18-sha512, diffie-hellman-group-exchange-sha1, diffie-hellman-group14-sha256, diffie-hellman-group14-sha1, diffie-hellman-group1-sha1","chacha20-poly1305@openssh.com, aes128-ctr, aes192-ctr, aes256-ctr, aes128-gcm@openssh.com, aes256-gcm@openssh.com, aes128-cbc, aes192-cbc, aes256-cbc, blowfish-cbc, cast128-cbc, 3des-cbc","umac-64-etm@openssh.com, umac-128-etm@openssh.com, hmac-sha2-256-etm@openssh.com, hmac-sha2-512-etm@openssh.com, hmac-sha1-etm@openssh.com, umac-64@openssh.com, umac-128@openssh.com, hmac-sha2-256, hmac-sha2-512, hmac-sha1","none, zlib@openssh.com","curve25519-sha256@libssh.org","ecdsa-sha2-nistp256","aes128-ctr","hmac-sha2-256","none","AAAAIQCd+X/B/OEx+FrwJSlVecOvNMuS5w2vTRz0z4prM+5VBwAAACEArU60b9CHs/d5BgyaOd7vmFygTMK5SyL90bS8VIztX/4=","AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAABKAAAAIQCd+X/B/OEx+FrwJSlVecOvNMuS5w2vTRz0z4prM+5VBwAAACEArU60b9CHs/d5BgyaOd7vmFygTMK5SyL90bS8VIztX/4=","AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDSEHaLacthwB30rtA4xJgN3G9zXkCmm2WhV/TlNBrD20fuNQAZX7XciX2YkqIHtK2dWLBYwVCCqvl//zoM42kI=","a6e4e1c16ba25d51bcddc58a6e16797144575dd18d02d9dedf75093d2b15c557",,,,,,,,,,,,,,,,,,,,"1xx7ASut7BF4ED8b592bebZBMBKTCzOsmbH4cjwx/0U=","P-256",256,"WsY12Ko6k+ez671VdpiGvGUdBrDMU7D2O848PifSYEs=","axfR8uEsQkf4vOblY6RA8ncDfYEt6zOg9KE5RdiYwpY=","T+NC4v4af5uO5+tKfA+eFivOM1drMV7Oy7ZAaDe/UfU=","/////wAAAAD//////////7zm+q2nF56E87nKwvxjJVE=","/////wAAAAEAAAAAAAAAAAAAAAD///////////////8=","NIQdotpy2HAHfSu0DjEmA3cb3NeQKabZaFX9OU0GsPY=","0fuNQAZX7XciX2YkqIHtK2dWLBYwVCCqvl//zoM42kI=",,,,,,,,,,,,,,,,,,,,"publickey",,,,, -"2022-01-10 02:20:37","170.10.0.0","tcp",22,"170-10-0-0.example.com","ssh",11976,"US","TEXAS","MARSHALL",,,"SSH-2.0-ARRIS_0.50","2.0","ARRIS_0.50",,"Y4RQS9sdRgEFwNJKVP6bZg==","diffie-hellman-group1-sha1","aes128-cbc, 3des-cbc, aes256-cbc, twofish256-cbc, twofish-cbc, twofish128-cbc, blowfish-cbc","hmac-sha1-96, hmac-sha1, hmac-md5","none","diffie-hellman-group1-sha1","ssh-rsa","aes128-cbc","hmac-sha1","none","LQj+UTJEQqdb/p/c/19yVc63eo+rnedwXKjP6eNNxxijN2cFoOjVMeqT2QTBjyoN7yRWBU2EID+3y2jUYT8mCqmqfyUv1eEbiCfLVlUyQ0X/CY9I5DDb5l6yEjNkuH2xVNNV6R7GFRwyYKAsYzfy+i9o1OORlUh3tozkkPfA9z/NlA==","AAAAB3NzaC1yc2EAAACCLQj+UTJEQqdb/p/c/19yVc63eo+rnedwXKjP6eNNxxijN2cFoOjVMeqT2QTBjyoN7yRWBU2EID+3y2jUYT8mCqmqfyUv1eEbiCfLVlUyQ0X/CY9I5DDb5l6yEjNkuH2xVNNV6R7GFRwyYKAsYzfy+i9o1OORlUh3tozkkPfA9z/NlA==","AAAAB3NzaC1yc2EAAAADAQABAAAAgwCDq1kGqqwdQVryCNcoyDbBpnL/okvM2d9NmR0OjprcToCZ2TZ5WUZt2BGwPE1QLJYskjhv7GwlfQ4qhEqHDg35wMrkO7j9LTQC7KW3xisOLuUil4FmMxPkol6s39945zBGpjw0l/BmJnUDlutxavkdd84fppFMwXNp2vbjxV1SYVc9","d53fedbfe92e631264629882b2e85bfd213ca4b07b824cd31f8de1fcb8d0ddcb",,,,,,,65537,"g6tZBqqsHUFa8gjXKMg2waZy/6JLzNnfTZkdDo6a3E6Amdk2eVlGbdgRsDxNUCyWLJI4b+xsJX0OKoRKhw4N+cDK5Du4/S00Auylt8YrDi7lIpeBZjMT5KJerN/feOcwRqY8NJfwZiZ1A5brcWr5HXfOH6aRTMFzadr248VdUmFXPQ==",1040,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"publickey, password","Arris",,,, -"2022-01-10 02:20:37","72.17.0.0","tcp",22,"072-017-0-0.example.com","ssh",33363,"US","FLORIDA","ORLANDO",517311,,"SSH-1.99-Cisco-1.25","1.99","Cisco-1.25",,"Z2fOfWsrLlh76Y0bOqa1cw==","diffie-hellman-group-exchange-sha1, diffie-hellman-group14-sha1, diffie-hellman-group1-sha1","aes128-cbc, 3des-cbc, aes192-cbc, aes256-cbc","hmac-sha1, hmac-sha1-96, hmac-md5, hmac-md5-96","none","diffie-hellman-group14-sha1","ssh-rsa","aes128-cbc","hmac-sha1","none","lrzL2DY9fVvwYg6CgB75uf2s8CLo+rL8Mp9tU1Ja3sDfBzj9QJjVDykupiy8s3usHfxMrHS2v3DhTiZjz/b5K6tVTgUBTXL94JfM4lwB+3EbLggPzKnlm1jQgnnU9c+tb7RX3IhBqU9Yj1gqxhErv9NFotgajQOOLgY0Ua5C0Ee+AIaMlLaNZe3LTejMsNUZMN5tl+sEmtutMHkGQsmjJxiJ3feF+Pys0I2+ojiiAfzqlMYar/5xOPl4Dj+HO+h91xVQ1/8nQRBc082fM7+ZJtDbRLtt4G8srlB5gew26jqfVASc/ui5gx4+BR9DG9VH8w+rJWBGfhOAaWqLFE2M3YuEWkjEmQMR1SQK1WFQ/oNiWJO2K5L3rk2LcAmyR6nQMtClVxYZ7CQOwa3uFL+JNXp9AhiiAtVaqhrEK81NJrJNh/+egTBl5STphxIShXd4KI9wyvkGlCIvNIMO94iXPVaWUXXbsGnU03+dsUkBzGf0eJ4DePInCk/RtunlSmOsjGld+rpS9g0VRxPrzbQRWuhpkgpV+CldyrI3C/rOxJRs2vSAKXocRsGwhqEKseAJzHXmiZ5ncsaGKoeB5lUkWLwcKjyok2tHVCDlzDUpE4aA/JHNEhT48det9RqtjC71yz8m0PeK2ySI/I+Qb7eBgevgduBmt+OUxgvfKi2UB6s=","AAAAB3NzaC1yc2EAAAIAlrzL2DY9fVvwYg6CgB75uf2s8CLo+rL8Mp9tU1Ja3sDfBzj9QJjVDykupiy8s3usHfxMrHS2v3DhTiZjz/b5K6tVTgUBTXL94JfM4lwB+3EbLggPzKnlm1jQgnnU9c+tb7RX3IhBqU9Yj1gqxhErv9NFotgajQOOLgY0Ua5C0Ee+AIaMlLaNZe3LTejMsNUZMN5tl+sEmtutMHkGQsmjJxiJ3feF+Pys0I2+ojiiAfzqlMYar/5xOPl4Dj+HO+h91xVQ1/8nQRBc082fM7+ZJtDbRLtt4G8srlB5gew26jqfVASc/ui5gx4+BR9DG9VH8w+rJWBGfhOAaWqLFE2M3YuEWkjEmQMR1SQK1WFQ/oNiWJO2K5L3rk2LcAmyR6nQMtClVxYZ7CQOwa3uFL+JNXp9AhiiAtVaqhrEK81NJrJNh/+egTBl5STphxIShXd4KI9wyvkGlCIvNIMO94iXPVaWUXXbsGnU03+dsUkBzGf0eJ4DePInCk/RtunlSmOsjGld+rpS9g0VRxPrzbQRWuhpkgpV+CldyrI3C/rOxJRs2vSAKXocRsGwhqEKseAJzHXmiZ5ncsaGKoeB5lUkWLwcKjyok2tHVCDlzDUpE4aA/JHNEhT48det9RqtjC71yz8m0PeK2ySI/I+Qb7eBgevgduBmt+OUxgvfKi2UB6s=","AAAAB3NzaC1yc2EAAAADAQABAAACAQDIVXBwKGhi35gabwHNZi6Bxls1BGtDVVZFhwvhTpJKTKhV4T2HnDFG7+FBpYejc92wH026Wf+uJHIpnKkVQRnnOV98zKXW68Tz+OnwT8aBQdLI+QYDC7wLwGRf+cOiXEAkpMrp2OJme+GwQ97oBccEwdu2j9vcYAFQ0+eCPNfwPrcZhwVb00kt/moLVSxWRdsDMzQiNDZf2zel+FQIAl5cCfaLSAQa1TIXy8SM13B0brnlpdyIqukQS0zUv/PL/6AsfhgLXeQBgjs1XIf6qL+ZdtQss5AKUDuJgrWDcS3nyNZQg/CAt8XdIsLntu3bCn+VGA1O/gUGLS1a9GoGd/lRArlmODNtbds74m7hxaAf/gzg0LFJx6HhwubmVCzTXEHl95KHYHKoDvCtUOgUm7zUugxWjhsLPfT6UfZCwvCY21SGVYsoEPiTT2DhuAFriM+PT83JresFHgZDosbqW0VCi2bzAKSBu/vphaqTbSdDo0xhkW9JCb3zUkW2ge/e/GrjxV4cNXRC9XQ/XYEIWmtF/gHSi0i9KweX4sN5TEkB/41vDvyDOdyPJ8Jta0I9vBolDwJ6qdMHOPlOW5oW83yCgbmUJNYkZ+MivABlc6iS/006qYiIwknHezbY5foYd8kDON7YAssOwCJcG5viII50Z1N9VsGkUv5sZMr2p9ry8Q==","06ff3cce443ed832927576d982b69d5a526d0e63334c72e87201deda61679406",,,,,,,65537,"yFVwcChoYt+YGm8BzWYugcZbNQRrQ1VWRYcL4U6SSkyoVeE9h5wxRu/hQaWHo3PdsB9Nuln/riRyKZypFUEZ5zlffMyl1uvE8/jp8E/GgUHSyPkGAwu8C8BkX/nDolxAJKTK6djiZnvhsEPe6AXHBMHbto/b3GABUNPngjzX8D63GYcFW9NJLf5qC1UsVkXbAzM0IjQ2X9s3pfhUCAJeXAn2i0gEGtUyF8vEjNdwdG655aXciKrpEEtM1L/zy/+gLH4YC13kAYI7NVyH+qi/mXbULLOQClA7iYK1g3Et58jWUIPwgLfF3SLC57bt2wp/lRgNTv4FBi0tWvRqBnf5UQK5ZjgzbW3bO+Ju4cWgH/4M4NCxSceh4cLm5lQs01xB5feSh2ByqA7wrVDoFJu81LoMVo4bCz30+lH2QsLwmNtUhlWLKBD4k09g4bgBa4jPj0/Nya3rBR4GQ6LG6ltFQotm8wCkgbv76YWqk20nQ6NMYZFvSQm981JFtoHv3vxq48VeHDV0QvV0P12BCFprRf4B0otIvSsHl+LDeUxJAf+Nbw78gzncjyfCbWtCPbwaJQ8CeqnTBzj5TluaFvN8goG5lCTWJGfjIrwAZXOokv9NOqmIiMJJx3s22OX6GHfJAzje2ALLDsAiXBub4iCOdGdTfVbBpFL+bGTK9qfa8vE=",4096,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"publickey, keyboard-interactive, password","Cisco",,,,"enterprise" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv deleted file mode 100644 index 0b125001be..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","port","hostname","tag","handshake","asn","geo","region","city","cipher_suite","ssl_poodle","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serial_number","naics","sic","freak_vulnerable","freak_cipher_suite","sector","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","http_response_type","http_code","http_reason","content_type","http_connection","www_authenticate","set_cookie","server_type","content_length","transfer_encoding","http_date","cert_valid","self_signed","cert_expired","browser_trusted","validation_level","browser_error","tlsv13_support","tlsv13_cipher","jarm" -"2022-01-10 00:01:42","96.60.0.0",10443,"96-60-0-0.example.com","ssl,vpn","TLSv1.2",4181,"US","WISCONSIN","MILWAUKEE","TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA","N",1024,"FGT60D4614030700","support","2014-06-23 09:56:32","2038-01-19 03:14:07","5A:3D:FF:06:F9:E9:25:37:57:F9:09:52:33:A4:85:15:24:2D:88:7F","168CAE",2,"sha1WithRSAEncryption","rsaEncryption","Fortinet","FortiGate","US","California","Sunnyvale",,,,,"support@fortinet.com",,,"Fortinet","Certificate Authority","US","California","Sunnyvale",,,,,"support@fortinet.com",,,517311,,"N",,,"35:AB:B6:76:2A:3D:17:B2:FB:40:45:1B:FC:0A:99:0A:6E:48:57:F7:30:0A:3B:B1:1A:E6:99:70:5B:7C:32:41","88:7B:16:DB:39:44:0C:47:0E:4A:8F:0B:C5:FB:4D:45:BC:93:5A:00:43:A1:D9:7F:05:1D:86:33:02:F8:FC:57:67:A6:1D:C0:FF:F7:D2:40:D8:9A:21:AE:4E:6D:DC:E7:FF:72:BF:13:CB:EE:A7:5F:CD:83:EA:8A:5E:FB:87:DD","99:45:1F:2E:AE:EB:88:91:27:43:33:79:FA:93:7D:CA","HTTP/1.1",200,"OK","text/html",,,,"xxxxxxxx-xxxxx",131,,"Mon, 10 Jan 2022 00:01:44 GMT","Y","N","N","N","unknown","x509: unknown error",,, -"2022-01-10 00:01:42","113.160.0.0",10443,"","ssl","TLSv1.2",45899,"VN","THAI BINH","THAI BINH","TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256","N",2048,"1078-btb-tbi-HungHa-61d39c6d5a7e2","1078-btb-tbi-HungHa-61d39c6d5a7e2","2022-01-04 01:01:34","2023-02-06 01:01:34","A9:00:BB:E1:54:4D:56:54:59:F1:B7:EA:F1:1A:D5:36:5C:63:90:8E","36974C4C6B1B3785",2,"sha256WithRSAEncryption","rsaEncryption","pfSense webConfigurator Self-Signed Certificate",,,,,,,,,,,,"pfSense webConfigurator Self-Signed Certificate",,,,,,,,,,,,517311,,"N",,,"38:85:F0:44:1E:AD:84:B8:2F:43:68:BA:AC:EE:17:13:A4:BF:86:1D:48:75:7E:22:FA:08:4C:28:5F:AC:3E:5F","AE:1B:4F:D1:E4:C0:35:9D:2A:4F:7A:37:B8:7B:11:9D:84:25:23:21:AB:EF:B2:0F:DC:C9:F2:A3:72:28:92:E1:74:72:FA:E1:09:6C:E1:F6:B6:E3:A7:61:1C:58:89:34:D7:06:5C:3D:0A:A7:F6:CC:8A:D6:24:D0:04:4C:03:02","16:93:9A:F4:35:7F:9A:85:45:71:91:C7:7C:80:88:00","HTTP/1.1",200,"OK","text/html; charset=UTF-8","keep-alive",,"PHPSESSID=e15bdfa5739c36877608eb4cf46cc388; path=/; secure; HttpO","nginx",,"chunked","Mon, 10 Jan 2022 00:01:44 GMT","N","Y","N","N","unknown","x509: unknown error",,, -"2022-01-10 00:01:42","34.224.0.0",10443,"","ssl,vpn","TLSv1.2",14618,"US","VIRGINIA","ASHBURN","TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA","N",2048,"","Entrust Certification Authority - L1K","2021-10-07 15:30:28","2022-11-06 15:30:28","AD:19:B2:1C:CB:88:70:9B:DB:8E:7E:F5:65:50:13:D6:43:6C:BE:6E","7B388364A24B88E77E5553B5C6748100",2,"sha256WithRSAEncryption","rsaEncryption","Ciena Corporation",,"US","Maryland","Hanover",,,,,,,,"Entrust, Inc.","(c) 2012 Entrust, Inc. - for authorized use only","US",,,,,,,,,,454110,,"N",,"Retail Trade","9A:64:73:0B:8A:FA:DE:22:D4:6D:5A:C6:C4:6F:D4:A4:2A:28:FA:41:1E:FF:81:DC:D4:D9:00:FD:78:DF:C4:DD","9A:B7:BD:68:7D:F3:E7:C1:B7:D3:F4:2F:01:B6:C4:77:90:A3:2B:1E:C0:89:F5:08:EC:43:87:35:60:36:D4:87:61:AA:B8:A8:B3:8A:E9:F1:04:AA:5B:67:12:FF:63:D5:14:80:77:6E:8F:7D:C3:E2:3A:F3:13:DF:08:43:6C:B0","E7:34:BC:92:84:FA:39:DE:E1:46:6C:27:DA:5A:01:F4","HTTP/1.1",200,"OK","text/html",,,,"xxxxxxxx-xxxxx",131,,"Mon, 10 Jan 2022 00:01:44 GMT","Y","N","N","Y","OV",,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv deleted file mode 100644 index ab28456b4e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv +++ /dev/null @@ -1,46 +0,0 @@ -"timestamp","ip","port","hostname","tag","handshake","asn","geo","region","city","cipher_suite","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serial_number","naics","sic","freak_vulnerable","freak_cipher_suite","sector","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","http_response_type","http_code","http_reason","content_type","http_connection","www_authenticate","set_cookie","server_type","content_length","transfer_encoding","http_date","cert_valid","self_signed","cert_expired","browser_trusted","validation_level","browser_error","device_model","device_sector","device_type","device_vendor","device_version","jarm","page_sha256fp","raw_cert","raw_cert_chain","tlsv13_cipher","tlsv13_support" -"2018-04-23 13:25:21","198.51.100.232","443",,"ssl-freak","TLSv1.0","8447","AT","WIEN","VIENNA","TLS_RSA_WITH_RC4_128_SHA","1024","usg50_B0B2DC2FA69D","usg50_B0B2DC2FA69D","2012-05-10 00:01:19","2032-05-05 00:01:19","14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2","4FAB054F","sha1WithRSAEncryption","rsaEncryption",,,,,,,,,,,,,,,,,,,,,,,,,"0","0","Y","TLS_RSA_EXPORT_WITH_RC4_40_MD5",,"57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1","E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87","1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE","HTTP/1.1","200","OK","text/html",,,,,,"chunked","Mon, 23 Apr 2018 13:25:26 GMT","Y","Y","N","N","unknown","x509: unknown error",,,,,,,,,,, -"2018-04-23 13:25:26","198.51.100.224","443","198-51-100-224.example.net","ssl-freak","TLSv1.0","12577","AT","NIEDEROSTERREICH","BADEN","TLS_RSA_WITH_RC4_128_SHA","1024","usg20w_C86C870287EC","usg20w_C86C870287EC","2010-01-01 00:00:53","2029-12-27 00:00:53","14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2","4B3D3B35","sha1WithRSAEncryption","rsaEncryption",,,,,,,,,,,,,,,,,,,,,,,,,"0","0","Y","TLS_RSA_EXPORT_WITH_RC4_40_MD5",,"57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1","E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87","1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE","HTTP/1.1","200","OK","text/html",,,,,,"chunked","Mon, 23 Apr 2018 13:25:29 GMT","Y","Y","N","N","unknown","x509: unknown error",,,,,,,,,,, -2018-04-23 13:25:21,198.51.100.232,443,,ssl-freak,TLSv1.0,8447,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg50_B0B2DC2FA69D,usg50_B0B2DC2FA69D,2012-05-10 00:01:19,2032-05-05 00:01:19,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4FAB054F,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:25:26 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:25:26,198.51.100.224,443,198-51-100-224.example.net,ssl-freak,TLSv1.0,12577,AT,NIEDEROSTERREICH,BADEN,TLS_RSA_WITH_RC4_128_SHA,1024,usg20w_C86C870287EC,usg20w_C86C870287EC,2010-01-01 00:00:53,2029-12-27 00:00:53,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4B3D3B35,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:25:29 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:25:33,198.51.100.67,443,,ssl-freak,TLSv1.0,8447,AT,NIEDEROSTERREICH,WAIDHOFEN AN DER THAYA,TLS_RSA_WITH_RC4_128_SHA,1024,Technicolor TG670,Technicolor TG670,2005-01-01 00:00:00,2024-12-31 00:00:00,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,-7A2C610E,sha1WithRSAEncryption,rsaEncryption,Technicolor,1112WT0YK,,,,,,,,,,,Technicolor,1112WT0YK,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Moved Temporarily,,keep-alive,,xAuth_SESSION_ID=bm90aGluZyBoZXJlCg==; path=/;,,0,,"Mon, 23 Apr 2018 14:25:37 GMT",N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:25:36,198.51.100.3,443,,ssl-freak,TLSv1.2,8445,AT,SALZBURG,HINTERGLEMM,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,2048,uag2100_04BF6D22A5A9,uag2100_04BF6D22A5A9,2016-03-08 20:27:08,2026-03-06 20:27:08,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,B0F07D300BDB4FC4,sha256WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_DES40_CBC_SHA,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:25:39 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:25:38,198.51.100.198,443,198-51-100-198.example.net,ssl-freak,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,2048,198-51-100-198.example.net,Go Daddy Secure Certificate Authority - G2,2016-12-29 08:51:00,2019-12-29 08:51:00,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,AEA6D3637023B56B,sha256WithRSAEncryption,rsaEncryption,,Domain Control Validated,,,,,,,,,,,198-51-100-198.example.net," Inc.""",http://certs.godaddy.com/repository/,SomeState,SomeCity,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,403,Forbidden ( The server,text/html,close,,,,2024,,,Y,N,N,Y,DV,,,,,,,,,,,, -2018-04-23 13:25:38,198.51.100.98,443,198-51-100-98.example.net,ssl-freak,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg50_FCF528743754,usg50_FCF528743754,2013-04-29 00:00:26,2033-04-24 00:00:26,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,517DB81A,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:25:40 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:25:41,198.51.100.156,443,198-51-100-156.example.net,ssl-freak,TLSv1.0,8339,AT,NIEDEROSTERREICH,SCHWECHAT,TLS_RSA_WITH_AES_128_CBC_SHA,1024,usg200_404A036775FC,usg200_404A036775FC,2010-05-01 00:04:04,2030-04-26 00:04:04,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4BDB6FF4,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_DES40_CBC_SHA,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:25:43 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:25:53,198.51.100.200,443,,ssl-freak,TLSv1.2,8447,AT,NIEDEROSTERREICH,KREMS AN DER DONAU,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_5CF4AB67FC6F,usg20_5CF4AB67FC6F,2015-12-02 00:00:47,2035-11-27 00:00:47,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,565E34AF,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:25:56 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:02,198.51.100.83,443,198-51-100-83.example.net,ssl-freak,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg20w_FCF5286F5972,usg20w_FCF5286F5972,2013-03-23 00:00:43,2033-03-18 00:00:43,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,514CF0AB,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:05 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:03,198.51.100.155,443,198-51-100-155.example.net,ssl-freak,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-155.example.net,198-51-100-155.example.net,2018-03-19 19:47:07,2023-03-19 19:47:07,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,2DF52AA905C7A2B44C2B9F0012FD5745,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,403,Forbidden,text/html,,,,Microsoft-IIS/6.0,1939,,"Mon, 23 Apr 2018 13:11:52 GMT",N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:03,198.51.100.129,443,198-51-100-129.example.net,ssl-freak,TLSv1.0,29654,AT,SALZBURG,SALZBURG,TLS_RSA_WITH_RC4_128_SHA,1024,localhost,localhost,2007-01-31 19:00:29,2008-01-31 19:00:29,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,2,sha1WithRSAEncryption,rsaEncryption,Apache HTTP Server,Test Certificate,,,,,,,,,,,Apache HTTP Server,For testing purposes only,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Apache,318,,"Mon, 23 Apr 2018 17:42:37 GMT",N,N,Y,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:26:12,198.51.100.7,443,198-51-100-7.example.net,ssl-freak,TLSv1.0,8445,AT,SALZBURG,ALTENMARKT IM PONGAU,TLS_RSA_WITH_RC4_128_SHA,2048,IMM2-5cf3fcaf3abd,IMM2-5cf3fcaf3abd,2013-03-22 14:32:06,2023-03-20 14:32:06,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,D8C631398B585F10,sha1WithRSAEncryption,rsaEncryption,System X,,US,SomeState,SomeCity,,,,,,,,System X,,US,SomeState,SomeCity,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,301,Moved Permanently,application/x-appweb-php,keep-alive,,,Mbedthis-Appweb/2.4.2,0,,"Mon, 23 Apr 2018 13:37:08 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:15,198.51.100.93,443,,ssl-freak,TLSv1.2,8447,AT,KARNTEN,SPITTAL AN DER DRAU,TLS_RSA_WITH_RC4_128_SHA,1024,usg50_B0B2DC3308EF,usg50_B0B2DC3308EF,2012-05-25 00:00:39,2032-05-20 00:00:39,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4FBECBA7,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:17 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:16,198.51.100.81,443,198-51-100-81.example.net,ssl-freak,TLSv1.0,5385,AT,VORARLBERG,FELDKIRCH,TLS_RSA_WITH_RC4_128_SHA,1024,usg100_5067F03642A5,usg100_5067F03642A5,2010-10-01 00:04:48,2030-09-26 00:04:48,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4CA525A0,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,518210,737415,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:19 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:17,198.51.100.162,443,198-51-100-162.example.net,ssl-freak,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,rc1,Peppercon CA,2003-05-08 16:30:05,2008-05-06 16:30:05,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,18,md5WithRSAEncryption,rsaEncryption,,R&D,DE,SomeState,,,,,,198-51-100-162.example.net,,,,Security Department,DE,SomeState,SomeCity,,,,,198-51-100-162.example.net,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Redirect,,,,,,,,,N,N,Y,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:26:22,198.51.100.57,443,,ssl-freak,TLSv1.0,8447,AT,STEIERMARK,GLEISDORF,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_5CF4AB661192,usg20_5CF4AB661192,2015-09-22 00:00:46,2035-09-17 00:00:46,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,56009A2E,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:25 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:28,198.51.100.146,443,198-51-100-146.example.net,ssl-freak,TLSv1.0,8447,AT,WIEN,VIENNA,TLS_ECDHE_RSA_WITH_RC4_128_SHA,1024,zywall_110_A0E4CB7CE5AF,zywall_110_A0E4CB7CE5AF,2015-01-26 17:19:56,2025-01-23 17:19:56,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,54C6773C,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:31 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:34,198.51.100.233,443,,ssl-freak,TLSv1.0,8447,AT,KARNTEN,KLAGENFURT AM WORTHERSEE,TLS_RSA_WITH_RC4_128_SHA,1024,198.51.100.174,198-51-100-174.example.net,2009-04-14 07:26:09,2025-04-15 07:26:09,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,A0571920C03C9EE0DA1168E586E0E8D440E42EA69D898AC829,sha1WithRSAEncryption,rsaEncryption,,General,DE,SomeState,SomeCity,,,,,198-51-100-174.example.net,,,,General,DE,SomeState,SomeCity,,,,,198-51-100-174.example.net,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,Close,,,LANCOM 1781A 8.50.0161 / 09.08.2011,,,,Y,N,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:35,198.51.100.106,443,198-51-100-106.example.net,ssl-freak,TLSv1.0,12793,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-106.example.net,SHT-Gruppe CA,2004-07-20 07:28:10,2006-07-20 07:38:10,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,190DBE75000000000007,sha1WithRSAEncryption,rsaEncryption,,,AT,SomeState,SomeCity,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Microsoft-IIS/6.0,1508,,"Mon, 23 Apr 2018 13:26:37 GMT",N,N,Y,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:26:37,198.51.100.191,443,,ssl-freak,TLSv1.0,8447,AT,STEIERMARK,LEBRING,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_5CF4AB669448,usg20_5CF4AB669448,2015-10-01 00:00:38,2035-09-26 00:00:38,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,560C77A6,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:40 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:42,198.51.100.235,443,198-51-100-235.example.net,ssl-freak,TLSv1.2,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg50_107BEF33651A,usg50_107BEF33651A,2014-04-24 00:00:27,2034-04-19 00:00:27,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,5358541B,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:45 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:43,198.51.100.167,443,198-51-100-167.example.net,ssl-freak,TLSv1.0,8412,AT,BURGENLAND,ELTENDORF,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-167.example.net,198-51-100-167.example.net,2008-08-19 06:57:11,2010-08-19 06:57:11,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,2,sha1WithRSAEncryption,rsaEncryption,SuSE Linux Web Server,web server,XY,unknown,unknown,,,,,198-51-100-167.example.net,,,SuSE Linux Web Server,CA,XY,SomeState,unknown,,,,,198-51-100-167.example.net,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Apache/2.2.3 (Linux/SUSE),80,,"Mon, 23 Apr 2018 13:26:45 GMT",N,N,Y,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:26:47,198.51.100.42,443,198-51-100-42.example.net,ssl-freak,TLSv1.0,8437,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-42.example.net,iLO Default Issuer (Do not trust),2013-11-05 00:00:00,2028-11-04 00:00:00,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,72FD09EF,sha1WithRSAEncryption,rsaEncryption,,,US,SomeState,SomeCity,,,,,,,,,,US,SomeState,Houston,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,,,,,,,,,N,N,N,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:26:48,198.51.100.177,443,198-51-100-177.example.net,ssl-freak,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_5CF4AB625772,usg20_5CF4AB625772,2015-03-04 00:00:39,2035-02-27 00:00:39,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,54F64B27,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:50 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:48,198.51.100.66,443,198-51-100-66.example.net,ssl-freak,TLSv1.0,5385,AT,VORARLBERG,DORNBIRN,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-66.example.net,198-51-100-66.example.net,2009-10-06 11:23:48,2015-03-29 11:23:48,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,98B18BCD61B0CD5D,sha1WithRSAEncryption,rsaEncryption,,??,??,??,??,,,,,??,,,,??,??,??,??,,,,,??,,,518210,737415,Y,TLS_RSA_EXPORT_WITH_DES40_CBC_SHA,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Found,text/html; charset=utf-8,close,,DSSignInURL=/; path=/; secure,,,,,Y,Y,Y,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:48,198.51.100.29,443,198-51-100-29.example.net,ssl-freak,TLSv1.0,6830,AT,NIEDEROSTERREICH,GUNTRAMSDORF,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_FCF52878354B,usg20_FCF52878354B,2013-05-20 00:00:39,2033-05-15 00:00:39,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,519967A7,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:50 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:49,198.51.100.235,443,,ssl-freak,TLSv1.0,8447,AT,TIROL,KITZBUHEL,TLS_RSA_WITH_RC4_128_SHA,1024,usg50_B0B2DC3AEFE7,usg50_B0B2DC3AEFE7,2012-10-30 00:02:36,2032-10-25 00:02:36,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,508F191C,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:52 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:50,198.51.100.159,443,,ssl-freak,TLSv1.0,8218,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-159.example.net,198-51-100-159.example.net,2002-01-09 20:22:25,2003-01-09 20:22:25,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,0,md5WithRSAEncryption,rsaEncryption,SomeOrganization,SomeOrganizationalUnit,--,SomeState,SomeCity,,,,,198-51-100-159.example.net,,,SomeOrganization,SomeOrganizationalUnit,--,SomeState,SomeCity,,,,,198-51-100-159.example.net,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,close,,,Apache/1.3.33 (Unix) (Red-Hat/Linux) FrontPage/4.,31,,"Mon, 23 Apr 2018 13:26:52 GMT",Y,Y,Y,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:51,198.51.100.138,443,198-51-100-138.example.net,ssl-freak,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_B0B2DC34A1F6,usg20_B0B2DC34A1F6,2012-06-16 00:00:58,2032-06-11 00:00:58,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4FDBCCBA,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:54 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:52,198.51.100.64,443,,ssl-freak,TLSv1.0,1853,AT,OBEROSTERREICH,WILHERING,TLS_RSA_WITH_RC4_128_SHA,1024,198.51.100.171,198.51.100.117,2017-08-10 10:48:40,2020-08-09 10:48:40,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,598C3A08,sha1WithRSAEncryption,rsaEncryption,,,,,SomeCity,,,,,,,,,,,,SomeCity,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,5597,,"Mon, 23 Apr 2018 13:26:54 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:55,198.51.100.189,443,198-51-100-62.example.net,ssl-freak,TLSv1.0,25255,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg20w_107BEF3A4C9E,usg20w_107BEF3A4C9E,2014-07-04 00:00:43,2034-06-29 00:00:43,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,53B5EEAB,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:57 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:56,198.51.100.17,443,198-51-100-17.example.net,ssl-freak,TLSv1.0,8447,AT,STEIERMARK,SOEDING,TLS_RSA_WITH_AES_256_CBC_SHA,1024,Vimar By-Web,Vimar By-Web,2011-10-27 09:19:55,2016-10-25 09:19:55,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,B82B13ED1FB0FD71,sha1WithRSAEncryption,rsaEncryption,,R&D,IT,SomeState,SomeCity,,,,,,,,,R&D,IT,SomeState,SomeCity,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_DES40_CBC_SHA,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Moved Temporarily,text/html,keep-alive,,,nginx/0.6.32,,chunked,"Mon, 23 Apr 2018 13:26:56 GMT",Y,Y,Y,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:56,198.51.100.111,443,,ssl-freak,TLSv1.0,8218,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-111.example.net,198-51-100-111.example.net,2002-01-09 20:22:25,2003-01-09 20:22:25,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,0,md5WithRSAEncryption,rsaEncryption,SomeOrganization,SomeOrganizationalUnit,--,SomeState,SomeCity,,,,,198-51-100-111.example.net,,,SomeOrganization,SomeOrganizationalUnit,--,SomeState,SomeCity,,,,,198-51-100-111.example.net,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,close,,,Apache/1.3.33 (Unix) (Red-Hat/Linux) FrontPage/4.,31,,"Mon, 23 Apr 2018 13:26:58 GMT",Y,Y,Y,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:56,198.51.100.179,443,198-51-100-179.example.net,ssl-freak,TLSv1.0,12605,AT,OBEROSTERREICH,LINZ,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_5CF4AB665FB9,usg20_5CF4AB665FB9,2015-09-25 00:00:42,2035-09-20 00:00:42,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,56048EAA,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:58 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:58,198.51.100.143,443,,ssl-freak,TLSv1.0,8447,AT,KARNTEN,GLAN,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_FCF5285DEDC4,usg20_FCF5285DEDC4,2012-11-09 00:00:44,2032-11-04 00:00:44,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,509C47AC,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:27:00 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:58,198.51.100.111,443,198-51-100-111.example.net,ssl-freak,TLSv1.0,1901,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,*.*,198-51-100-111.example.net,2009-01-16 12:51:43,2010-01-16 12:51:43,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,6,md5WithRSAEncryption,rsaEncryption,,,IL,SomeState,,,,,,,,,,Visonic CA,IL,SomeState,,,,,,198-51-100-111.example.net,,,518210,737415,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Found,text/html,close,,PowerLink=226002836046b4bddcd2d16b809f76d9; path=/,Apache/1.3.31 (Unix) PHP/4.3.9 mod_ssl/2.8.20 Open,,chunked,"Wed, 23 Jan 2002 10:17:09 GMT",N,N,Y,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:26:59,198.51.100.79,443,,ssl-freak,TLSv1.0,8447,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_5CF4AB65A17C,usg20_5CF4AB65A17C,2015-09-01 00:00:51,2035-08-27 00:00:51,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,55E4EAB3,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:27:02 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:59,198.51.100.90,443,,ssl-freak,TLSv1.0,8218,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-90.example.net,198-51-100-90.example.net,2002-01-09 20:22:25,2003-01-09 20:22:25,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,0,md5WithRSAEncryption,rsaEncryption,SomeOrganization,SomeOrganizationalUnit,--,SomeState,SomeCity,,,,,198-51-100-90.example.net,,,SomeOrganization,SomeOrganizationalUnit,--,SomeState,SomeCity,,,,,198-51-100-90.example.net,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,close,,,Apache/1.3.33 (Unix) (Red-Hat/Linux) FrontPage/4.,31,,"Mon, 23 Apr 2018 13:27:02 GMT",Y,Y,Y,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:27:03,198.51.100.186,443,198-51-100-186.example.net,ssl-freak,TLSv1.0,31125,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-186.example.net,198-51-100-186.example.net,2013-07-11 12:20:19,2021-07-09 12:20:19,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,D947ED19BEAB28E6,sha1WithRSAEncryption,rsaEncryption,,IT,PL,SomeState,SomeCity,,,,,198-51-100-186.example.net,,,,IT,PL,SomeState,SomeCity,,,,,198-51-100-186.example.net,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,401,Unauthorized,text/plain,close,"Basic realm=""example.com""",,Microsoft-IIS/7.5,0,,"Mon, 23 Apr 2018 14:03:40 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:27:03,198.51.100.150,443,198-51-100-150.example.net,ssl-freak,TLSv1.0,8559,AT,BURGENLAND,NEUSIEDL AM SEE,TLS_ECDHE_RSA_WITH_RC4_128_SHA,2048,198-51-100-150.example.net,COMODO RSA Domain Validation Secure Server CA,2017-02-08 00:00:00,2019-05-09 23:59:59,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,B6EF6CF436532F0252627393BD7311FD,sha256WithRSAEncryption,rsaEncryption,,Domain Control Validated,,,,,,,,,,,,,GB,SomeState,SomeCity,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:27:06 GMT",N,N,N,N,DV,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:27:03,198.51.100.141,443,198-51-100-141.example.net,ssl-freak,TLSv1.0,39372,AT,OBEROSTERREICH,HINTERSTODER,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-141.example.net,iLO Default Issuer (Do not trust),2014-01-14 00:00:00,2029-01-13 00:00:00,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,7852761B,sha1WithRSAEncryption,rsaEncryption,,,US,SomeState,SomeCity,,,,,,,,,,US,SomeState,SomeCity,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,,,,,,,,,N,N,N,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:27:04,198.51.100.194,443,198-51-100-194.example.net,ssl-freak,TLSv1.0,8447,AT,KARNTEN,GLAN,TLS_RSA_WITH_RC4_128_SHA,1024,iDRAC6 default certificate,iDRAC6 default certificate,2009-09-17 22:47:28,2019-09-15 22:47:28,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,1,sha1WithRSAEncryption,rsaEncryption,,Remote Access Group,US,SomeState,SomeCity,,,,,,,,,Remote Access Group,US,SomeState,SomeCity,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Moved Temporarily,,keep-alive,,,Mbedthis-Appweb/2.4.2,0,,"Mon, 23 Apr 2018 13:25:57 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -"2022-02-07 00:01:41","2.136.0.0",10443,"2-136-0-0.example.com","ssl,ssl-freak,ssl-poodle,vpn","TLSv1.0",12345,"ES","MADRID","MADRID","TLS_RSA_WITH_RC4_128_SHA",1024,"usg50_107BEF336340","usg50_107BEF336340","2014-04-24 00:00:32","2034-04-19 00:00:32","F5:04:98:CD:D4:67:13:E1:77:B7:38:D4:B9:43:C0:72:50:6C:0D:58",53585420,"sha1WithRSAEncryption","rsaEncryption",,,,,,,,,,,,,,,,,,,,,,,,,517311,,"Y","TLS_RSA_EXPORT_WITH_RC4_40_MD5","Communications, Service Provider, and Hosting Service","AF:3A:71:B7:1B:A2:62:4E:87:22:FF:19:3F:84:1F:7F:CC:DC:06:E0:AF:80:E2:5D:33:A5:68:9A:E3:81:25:45","14:92:CC:6B:C7:B3:09:31:50:8C:1C:8D:5B:FD:D1:BE:41:78:80:97:E0:10:11:48:1F:EE:D6:CB:4F:F0:13:D5:05:56:AC:BA:12:12:02:F7:0F:03:40:95:17:8A:5F:79:98:E1:44:EF:E6:5A:44:E3:AC:3A:F8:49:F7:AC:B6:52","E8:5F:96:16:3F:76:35:F0:07:4F:4C:2C:38:FC:27:6B","HTTP/1.1",200,"OK","text/html",,,,"",,"chunked","Mon, 07 Feb 2022 00:01:43 GMT","Y","Y","N","N","unknown","x509: unknown error",,,,,,"Zyxel","firewall","ZyWALL USG 50",,"enterprise", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv deleted file mode 100644 index 4bcc6758ac..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv +++ /dev/null @@ -1,32 +0,0 @@ -"timestamp","ip","port","hostname","tag","handshake","asn","geo","region","city","cipher_suite","ssl_poodle","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serial_number","naics","sic","sector","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","http_response_type","http_code","http_reason","content_type","http_connection","www_authenticate","set_cookie","server_type","content_length","transfer_encoding","http_date","cert_valid","self_signed","cert_expired","browser_trusted","validation_level","browser_error","tlsv13_support","tlsv13_cipher","device_model","device_sector","device_type","device_vendor","device_version","jarm","page_sha256fp","raw_cert","raw_cert_chain" -"2018-08-08 00:51:42","203.0.113.85",8443,"example.com","ssl-poodle","TLSv1.0",65540,"AT","WIEN","VIENNA","TLS_RSA_WITH_RC4_128_SHA","Y",1024,"usg20_107BEF394BA5","usg20_107BEF394BA5","2014-06-25 00:00:42","2034-06-20 00:00:42","04:FA:DE:1D:BD:4A:05:25:61:FB:F3:D6:64:74:66:44:01:22:D7:C3","53AA112A",2,"sha1WithRSAEncryption","rsaEncryption",,,,,,,,,,,,,,,,,,,,,,,,,0,0,,"16:25:9F:C7:A1:8D:64:1F:D9:25:42:BF:87:5C:4F:F3:63:14:97:21:EC:B6:67:10:F2:CA:52:37:C9:FE:49:2E","0B:2D:48:8C:4B:55:8B:F3:AB:F8:45:ED:E0:A0:63:F4:84:2F:4C:19:DC:A8:6F:7D:6A:AF:61:D7:98:AA:58:0F:CB:CA:87:D2:C3:0B:C5:DF:49:A7:84:7C:47:58:89:7D:92:B6:7B:98:7D:B1:64:4B:DC:DD:BE:9D:11:2A:D1:AE","33:E3:61:3F:5D:AA:96:99:38:A5:D6:F1:11:C7:ED:FC","HTTP/1.1",200,"OK","text/html",,,,,,"chunked","Wed, 08 Aug 2018 00:51:44 GMT","Y","Y","N","N","unknown","x509: unknown error",,,,,,,,,,, -2018-04-19 13:32:27,198.51.100.147,443,,ssl-poodle,TLSv1.0,8445,AT,SALZBURG,SALZBURG,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-147.example.net,some_issuer,2017-09-18 08:22:17,2019-09-18 08:22:17,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,746481F100000000000C,2,sha1WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,AT,Tirol,Ehrwald,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Microsoft-IIS/7.5,689,,"Thu, 19 Apr 2018 13:32:32 GMT",N,N,N,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-19 13:32:32,198.51.100.207,443,198-51-100-94.example.net,ssl-poodle,TLSv1.0,25255,AT,SALZBURG,SALZBURG,TLS_RSA_WITH_RC4_128_SHA,Y,1024,example,some_issuer,2004-06-03 11:11:43,2024-05-29 11:11:43,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,0,2,md5WithRSAEncryption,rsaEncryption,,,US,,,,,,,,,,,,US,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,close,,"rg_cookie_session_id=1654544029; path=/; expires=Fri, 01 Jan 2038",,,,"Thu, 19 Apr 2018 13:32:34 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:32,198.51.100.200,443,198-51-100-200.example.net,ssl-poodle,TLSv1.2,8445,AT,SALZBURG,,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-200.example.net,198-51-100-200.example.net,2016-10-01 14:09:12,2020-10-02 14:09:12,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,2E8C9E4A2C7D3EDC,2,sha256WithRSAEncryption,rsaEncryption,some_org_name,,AT,,,,,,,,,,some_org_name,,AT,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html; charset=utf-8,close,,,,,,,N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:33,198.51.100.239,443,198-51-100-239.example.net,ssl-poodle,TLSv1.0,8437,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,Y,1024,198-51-100-239.example.net,198-51-100-239.example.net,2011-07-27 13:30:18,2012-07-26 13:30:18,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,7C91,2,sha1WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,--,SomeState,SomeCity,,,,,198-51-100-239.example.net,,,SomeOrganization,SomeOrganizationalUnit,--,SomeState,SomeCity,,,,,198-51-100-239.example.net,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,403,Forbidden,text/html; charset=UTF-8,close,,,Apache/2.2.3 (CentOS),4958,,"Thu, 19 Apr 2018 13:32:35 GMT",Y,Y,Y,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:35,198.51.100.156,443,,ssl-poodle,TLSv1.0,8447,AT,NIEDEROSTERREICH,,TLS_RSA_WITH_RC4_128_SHA,Y,1024,example,some_issuer,2010-01-01 00:00:52,2029-12-27 00:00:52,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4B3D3B34,2,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Thu, 19 Apr 2018 13:32:37 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:36,198.51.100.122,443,198-51-100-122.example.net,ssl-poodle,TLSv1.2,36351,AT,AUSTRIA,?,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,Y,2048,198-51-100-122.example.net,COMODO RSA Organization Validation Secure Server CA,2017-04-06 00:00:00,2019-04-06 23:59:59,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,CAB81F32F3FF4766BC545A2C14DF34B5,2,sha256WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,AT,Wien,Wien,,1130,,,,,,COMODO CA Limited,,GB,Greater Manchester,Salford,,,,,,,,518210,737401,Information Technology,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Apache,0,,"Thu, 19 Apr 2018 13:32:20 GMT",Y,N,N,Y,OV,,,,,,,,,,,, -2018-04-19 13:32:37,198.51.100.58,443,198-51-100-58.example.net,ssl-poodle,TLSv1.2,12605,AT,OBEROSTERREICH,LINZ,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,example,some_issuer,2015-01-17 16:11:24,2020-01-17 16:11:24,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,6D9E2D4443F1D69E4A8865CC1C5B6963,2,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Microsoft-IIS/8.5,701,,"Thu, 19 Apr 2018 13:34:53 GMT",N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:38,198.51.100.18,443,198-51-100-18.example.net,ssl-poodle,TLSv1.2,6830,AT,OBEROSTERREICH,LINZ,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,Y,2048,198-51-100-18.example.net,TERENA SSL CA 3,2017-07-14 00:00:00,2020-07-22 12:00:00,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,0386AD387BEC13878473D23C8C786ECE,2,sha256WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,AT,,Linz,,,,,,,,TERENA,,NL,Noord-Holland,Amsterdam,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Found,,Close,,BNIS_ChallengeState=Bqyd+IQebjQwiiYNKBJkA5Ta0spL1gX5; Path=/; Exp,,61,,,Y,N,N,Y,OV,,,,,,,,,,,, -2018-04-19 13:32:38,198.51.100.246,443,,ssl-poodle,TLSv1.2,8447,AT,SALZBURG,SALZBURG,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,example,some_issuer,2014-09-01 16:18:46,2054-08-24 16:18:46,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,129FA64A4BE039B54E850F1AA65AD835,2,sha256WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Found,text/html; charset=utf-8,,,ASP.NET_SessionId=e3qfk1dfz2mtqwzoym3gul3r; path=/; HttpOnly,Microsoft-IIS/8.5,145,,"Thu, 19 Apr 2018 13:32:40 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:39,198.51.100.35,443,198-51-100-35.example.net,ssl-poodle,TLSv1.0,12605,AT,OBEROSTERREICH,LINZ,TLS_RSA_WITH_AES_128_CBC_SHA,Y,2048,198-51-100-35.example.net,Go Daddy Secure Certificate Authority - G2,2017-08-28 13:29:01,2018-09-10 06:28:49,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,90B22B4CEF57C0FC,2,sha256WithRSAEncryption,rsaEncryption,,some_org_name,,,,,,,,198-51-100-35.example.net,,,,,US,Arizona,Scottsdale,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Microsoft-IIS/7.5,266,,"Thu, 19 Apr 2018 13:35:03 GMT",Y,N,N,Y,DV,,,,,,,,,,,, -2018-04-19 13:32:39,198.51.100.142,443,,ssl-poodle,TLSv1.0,8447,AT,WIEN,VIENNA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,Y,2048,198.51.100.19,198-51-100-19.example.net,2014-12-11 09:57:33,2024-12-08 09:57:33,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,A0571DCBE5E1A2C062D8FB7001271581B5F69824157E385563FA23527E0B,2,sha256WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,DE,,,,,,,198-51-100-19.example.net,,,some_org_name,Engineering,DE,NRW,Wuerselen,,,,,198-51-100-19.example.net,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,Close,,,LANCOM,,,"Thur, 19 Apr 2018 13:32:41 GMT",Y,N,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:39,198.51.100.178,443,,ssl-poodle,TLSv1.0,8447,AT,NIEDEROSTERREICH,,TLS_RSA_WITH_RC4_128_SHA,Y,1024,example,some_issuer,2012-05-30 00:00:44,2032-05-25 00:00:44,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4FC5632C,2,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Thu, 19 Apr 2018 13:32:41 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:40,198.51.100.99,443,198-51-100-99.example.net,ssl-poodle,TLSv1.2,6830,AT,WIEN,VIENNA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-99.example.net,RapidSSL RSA CA 2018,2018-03-30 00:00:00,2019-04-29 12:00:00,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,0BDCB5D6D4C22BD2A1CF55584B6DE09C,2,sha256WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,DigiCert Inc,198-51-100-99.example.net,US,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,404,Not Found,text/html; charset=us-ascii,close,,,Microsoft-HTTPAPI/2.0,315,,"Thu, 19 Apr 2018 13:32:43 GMT",Y,N,N,Y,DV,,,,,,,,,,,, -2018-04-19 13:32:40,198.51.100.235,443,198-51-100-235.example.net,ssl-poodle,TLSv1.0,25255,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,Y,1024,Nextcloud,Nextcloud,2016-12-13 20:28:39,2017-01-12 20:28:39,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,CDE5769D28C80B6B,2,sha1WithRSAEncryption,rsaEncryption,some_org_name,,AU,Some-State,,,,,,,,,Internet Widgits Pty Ltd,,AU,Some-State,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,400,Bad Request,text/html; charset=UTF-8,close,,nc_sameSiteCookiestrict=true; path=/; httponly;secure; expires=Fr,Apache/2.4.10 (FreeBSD) OpenSSL/0.9.8zd-freebsd PH,6939,,"Thu, 19 Apr 2018 13:32:42 GMT",Y,Y,Y,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:41,198.51.100.187,443,198-51-100-187.example.net,ssl-poodle,TLSv1.2,28760,AT,OBEROSTERREICH,,TLS_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-187.example.net,Go Daddy Secure Certificate Authority - G2,2018-02-12 17:56:01,2020-02-12 17:56:01,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,03BA30FF4972177C,2,sha256WithRSAEncryption,rsaEncryption,,some_org_name,,,,,,,,198-51-100-187.example.net,,,,,US,Arizona,Scottsdale,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,400,No parameters passed t,text/html,,,,Microsoft-IIS/10.0,11,,"Thu, 19 Apr 2018 13:32:42 GMT",Y,N,N,Y,DV,,,,,,,,,,,, -2018-04-19 13:32:42,198.51.100.213,443,198-51-100-213.example.net,ssl-poodle,TLSv1.2,8447,AT,OBEROSTERREICH,,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-213.example.net,some_issuer,2016-09-22 08:12:17,2018-09-22 08:12:17,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,770000000EBB9429663601BAB700000000000E,2,sha1WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,AT,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,301,Moved Permanently,,close,,,Microsoft-IIS/8.5,0,,"Thu, 19 Apr 2018 13:32:44 GMT",N,N,N,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-19 13:32:42,198.51.100.74,443,198-51-100-74.example.net,ssl-poodle,TLSv1.0,62363,AT,STEIERMARK,,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,DC,DC,2016-12-30 17:15:38,2021-12-30 17:15:38,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,7753CCEB55990A834E15DAC5707D403A,2,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Microsoft-IIS/7.5,689,,"Thu, 19 Apr 2018 13:32:44 GMT",N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:43,198.51.100.145,443,198-51-100-145.example.net,ssl-poodle,TLSv1.0,8447,AT,KARNTEN,KLAGENFURT AM WORTHERSEE,TLS_RSA_WITH_RC4_128_SHA,Y,1024,localdomain,localdomain,2008-10-07 20:12:54,2018-10-07 20:12:54,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,91B04FFCF174CCFF,0,sha1WithRSAEncryption,rsaEncryption,some_org_name,,CA,,,,,,,198-51-100-145.example.net,,,some_org_name,,CA,Quebec,Gatineau,,,,,198-51-100-145.example.net,,,518210,737415,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.0,302,Found,text/html; charset=UTF-8,close,,"HOMEBASEID=658512b32961b9b6f8df7a3d4de7fa01; expires=Tue, 19-Jan-",Apache/2.2.3 (Red Hat),0,,"Thu, 19 Apr 2018 12:52:32 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:44,198.51.100.48,443,198-51-100-48.example.net,ssl-poodle,TLSv1.0,1901,AT,NIEDEROSTERREICH,,TLS_RSA_WITH_RC4_128_SHA,Y,1024,198-51-100-48.example.net,198-51-100-48.example.net,2013-06-15 20:10:49,2023-06-15 20:10:49,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,013F49762DAE,0,sha1WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,US,,,,,,,198-51-100-48.example.net,,,Western Digital,Branded Products,US,CS,Mountain View,,,,,,,,518210,737415,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Apache,225,,"Thu, 19 Apr 2018 03:08:06 GMT",N,N,N,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-19 13:32:45,198.51.100.94,443,198-51-100-94.example.net,ssl-poodle,TLSv1.2,6830,AT,WIEN,VIENNA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-94.example.net,RapidSSL CA,2013-04-03 17:02:33,2014-04-07 03:32:33,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,0B697D,2,sha1WithRSAEncryption,rsaEncryption,,some_org_name,,,,,,,,,,KtAjvog6HgAsml0cyxE4hpc9kv8dhgWZ,"GeoTrust, Inc.",,US,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Found,text/html; charset=utf-8,,,ASP.NET_SessionId=z5lph4ufefkvg1xzmd4q2m33; path=/; HttpOnly,Microsoft-IIS/8.0,144,,"Thu, 19 Apr 2018 13:32:48 GMT",Y,N,Y,N,unknown,x509: certificate has expired or is not yet valid,,,,,,,,,,, -2018-04-19 13:32:45,198.51.100.53,443,198-51-100-53.example.net,ssl-poodle,TLSv1.0,8447,AT,TIROL,,TLS_RSA_WITH_RC4_128_SHA,Y,1024,example,some_issuer,2008-11-13 13:47:18,2028-11-08 13:47:18,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,BE2B43544C0AFF2E,0,sha1WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,DE,,,,,,,198-51-100-53.example.net,,,some_org_name,some_org_name,DE,Niedersachsen,38162 Cremlingen (OT Schandelah),,,,,198-51-100-53.example.net,,,518210,737415,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html; charset=iso-8859-1;,,,,GoAhead-Webs,,,,Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:46,198.51.100.56,443,198-51-100-56.example.net,ssl-poodle,TLSv1.0,8445,AT,TIROL,,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-56.example.net,some_issuer,2016-11-28 08:05:12,2018-11-28 08:05:12,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,637D34F100010000000E,2,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Microsoft-IIS/7.5,689,,"Thu, 19 Apr 2018 13:32:49 GMT",N,N,N,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-19 13:32:46,198.51.100.82,443,198-51-100-82.example.net,ssl-poodle,TLSv1.0,6830,AT,OBEROSTERREICH,,TLS_RSA_WITH_AES_128_CBC_SHA,Y,1024,123AFG,7426AC8186F3,2011-01-01 00:00:06,2020-12-29 00:00:06,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,8186F3,2,sha1WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,US,,,,,,,,,,"Cisco Systems, Inc.",some_org_name,US,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.0,200,OK,text/html,close,,,Embedded HTTP Server.,107,,"Sat, 01 Jan 2011 00:00:45 GMT",N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:48,198.51.100.29,443,198-51-100-29.example.net,ssl-poodle,TLSv1.0,6830,AT,STEIERMARK,GRAZ,TLS_RSA_WITH_RC4_128_SHA,Y,1024,198.51.100.43,198.51.100.22,2018-04-18 13:32:09,2038-01-15 13:32:09,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,862D98F4B99D0042,2,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,518210,737415,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.0,200,OK,text/html; charset=utf-8,,,,,,,,Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:49,198.51.100.114,443,198-51-100-114.example.net,ssl-poodle,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_AES_256_CBC_SHA,Y,1024,198-51-100-114.example.net,198-51-100-114.example.net,2009-08-25 17:47:57,2019-05-25 17:47:57,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,FB09C7848A7F4D77,0,sha1WithRSAEncryption,rsaEncryption,some_org_name,,AT,Vienna,Vienna,,,,,198-51-100-114.example.net,,,Digispectrum,,AT,Vienna,Vienna,,,,,198-51-100-114.example.net,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html; charset=utf-8,close,,b69223925949d45306d32f1a3d23c011=6a01vehilfpml41pl3pq3oth52; path,Apache/2.2.3 (CentOS),,chunked,"Thu, 19 Apr 2018 13:32:52 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:49,198.51.100.11,443,198-51-100-11.example.net,ssl-poodle,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,Y,2048,FGT60C3G12019794,FGT60C3G12019794,2012-08-10 07:17:11,2022-08-11 07:17:11,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,-6CD83A89,2,sha1WithRSAEncryption,rsaEncryption,some_org_name,,,,,,,,,,,,Fortinet Ltd.,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,79,,"Thu, 19 Apr 2018 13:32:08 GMT",N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:49,198.51.100.49,443,198-51-100-49.example.net,ssl-poodle,TLSv1.2,8447,AT,NIEDEROSTERREICH,,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,Y,1024,localhost,localhost,2009-11-10 23:48:47,2019-11-08 23:48:47,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,B5C752C98781B503,0,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,518210,737415,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Apache/2.4.10 (Win32) OpenSSL/1.0.1i PHP/5.5.15,2190,,"Thu, 19 Apr 2018 13:32:55 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:50,198.51.100.236,443,,ssl-poodle,TLSv1.0,8447,AT,NIEDEROSTERREICH,,TLS_RSA_WITH_AES_128_CBC_SHA,Y,1024,example,some_issuer,2013-01-30 12:00:08,2023-01-28 12:00:08,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,-462A1420,2,sha1WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,US,,,,,,,,,,Netgear Inc.,Netgear Prosafe,US,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.0,200,OK,text/html,close,,,Embedded HTTP Server.,107,,"Sat, 01 Jan 2011 00:00:21 GMT",N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:50,198.51.100.224,443,198-51-100-224.example.net,ssl-poodle,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-224.example.net,some_issuer,2017-08-03 10:21:50,2019-08-03 10:21:50,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,6126D181000300000041,2,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,401,Unauthorized,text/html,,NTLM,,Microsoft-IIS/7.5,1344,,"Thu, 19 Apr 2018 13:32:52 GMT",N,N,N,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -"2022-02-07 00:01:41","206.162.0.0",10443,,"ssl,ssl-poodle,vpn","TLSv1.2",12345,"CA","BRITISH COLUMBIA","BURNABY","TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA","Y",1024,"FWF60D4615000455","support","2015-01-28 18:14:33","2038-01-19 03:14:07","C9:B0:4E:B7:79:94:B4:DD:A7:15:21:86:43:F9:6E:4B:C9:A2:87:D9","1CA40F",2,"sha1WithRSAEncryption","rsaEncryption","Fortinet","FortiGate","US","California","Sunnyvale",,,,,"support@fortinet.com",,,"Fortinet","Certificate Authority","US","California","Sunnyvale",,,,,"support@fortinet.com",,,517311,,"Communications, Service Provider, and Hosting Service","38:F7:E0:92:24:8C:CB:28:43:93:0B:91:17:30:B1:41:8F:4E:2D:E5:A8:93:AE:4D:FE:53:00:D3:0E:53:02:16","0C:F0:37:3F:A8:93:AE:4D:FE:53:00:D3:2A:E6:6D:0B:02:9D:B9:46:58:A6:9E:5A:35:40:FB:62:9C:81:47:0A:4F:15:5D:53:D9:2F:36:4A:0B:3B:10:61:A9:07:EE:94:EC:00:B8:9C:F7:E0:92:24:8C:CB:28:2C:DD:E7:07:C6","8A:B3:08:20:34:79:94:B4:DD:A7:36:D7:14:6E:33:50","HTTP/1.1",200,"OK","text/html",,,,,131,,"Mon, 07 Feb 2022 00:01:43 GMT","Y","N","N","N","unknown","x509: unknown error",,,,,,"Fortinet","firewall","FortiGate",,"enterprise", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv deleted file mode 100644 index fd671ec904..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","transaction_id","magic_cookie","message_length","message_type","mapped_family","mapped_address","mapped_port","xor_mapped_family","xor_mapped_address","xor_mapped_port","software","fingerprint","amplification","response_size" -"2010-02-10 00:00:00",192.168.0.1,udp,3478,node01.example.com,stun,64512,ZZ,Region,City,0,0,,000000000000000000000000,2112a442,88,0101,01,192.168.0.1,3243,01,192.168.0.1,3243,"Coturn-4.5.1.1 'dan Eider'",0xfaedd06e,5.40,108 -"2010-02-10 00:00:01",192.168.0.2,udp,3478,node02.example.com,stun,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",000000000000000000000000,2112a442,88,0101,01,51.77.39.195,45877,01,192.168.0.2,45877,"Coturn-4.5.1.1 'dan Eider'",0x21128641,5.40,108 -"2010-02-10 00:00:02",192.168.0.3,udp,3478,node03.example.com,stun,64512,ZZ,Region,City,0,0,,000000000000000000000000,2112a442,76,0101,01,192.168.0.3,16321,01,188.68.240.32,16321,"ApolloProxy-1.20.1.28 'sunflower'",,4.80,96 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv deleted file mode 100644 index 8f63554910..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sequence_number","ack_number","window_size","urgent_pointer","tcp_flags","raw_packet","sector" -"2022-01-10 09:18:23","66.9.0.0","tcp",80,,"synfulknock",18885,"US","NEW JERSEY","JERSEY CITY",,,0,791102,8192,0,4608,"3cfdfec601e4700f6a9a2000080045000034c3780000f706789442099555b869f7ee0050b20800000000000c123e8012200002aa0000020405b40101040201030305", -"2022-01-10 09:19:17","213.131.0.0","tcp",80,"host-213-131-55-210-customer.wanex.net","synfulknock",35805,"GE","TBILISI","TBILISI",,,0,791102,8192,0,4608,"90e2baaf0b84700f6a9a200008004500003434100000f2064382d58337d2b8698b720050916200000000000c123e8012200059d50000020405b40101040201030305", -"2022-01-10 09:27:39","213.178.0.0","tcp",80,,"synfulknock",29256,"SY","DIMASHQ","DAMASCUS",,,0,791102,8192,0,4608,"90e2bab9cfd4700f6a9a20000800450000340f1d0000ea068bdad5b2e6914a522f360050eb5200000000000c123e801220001b4a0000020405b40101040201030305" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv deleted file mode 100644 index 3309e9a3d8..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","banner" -"2019-09-04 12:27:34","198.123.245.145","tcp",5678,"example.local","telnet-alt",20255,"AA","LOCATION","LOCATION",0,0,"|MikroTik v6.5|Login:" -"2019-09-04 12:27:40","198.123.245.145","tcp",5678,"example.local","telnet-alt",20255,"AA","LOCATION","LOCATION",0,0,"|MikroTik v6.45.3 (stable)|Login:" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv deleted file mode 100644 index 3dde133d4e..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","opcode","errorcode","error","errormessage","size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,35067,node01.example.com,tftp,64512,ZZ,Region,City,0,0,5,0,"Not defined","Get not supported",22,1.57 -"2010-02-10 00:00:01",192.168.0.2,udp,56709,node02.example.com,tftp,64512,ZZ,Region,City,0,0,5,1,"File not found","File not found",19,1.36 -"2010-02-10 00:00:02",192.168.0.3,udp,32785,node03.example.com,tftp,64512,ZZ,Region,City,0,0,5,2,"Access violation","Access violation",21,1.50 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv deleted file mode 100644 index efeab02c49..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","mac","radioname","essid","modelshort","modelfull","firmware","size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,10001,node01.example.com,"ubiquiti,iot",64512,ZZ,Region,City,0,0,00156db98c3a,kachine.meta.lidia.tereixa,Kachine-Meta-Lidia-Tereixa,NS5,,XS5.ar2313.v3.5.4494.091109.1459,148,37.00 -"2010-02-10 00:00:01",192.168.0.2,udp,10001,node02.example.com,"ubiquiti,iot",64512,ZZ,Region,City,0,0,00156d7c9188,adana.mason.lanikai.ozaner,Adana-Mason-Lanikai-Ozaner,LM5,"NanoStation Loco M5",XM.ar7240.v5.6.3.28591.151130.1749,156,39.00 -"2010-02-10 00:00:02",192.168.0.3,udp,10001,node03.example.com,"ubiquiti,iot",64512,ZZ,Region,City,0,0,0418d6000fd5,tailynn.kadija.noreen.dinkar,Tailynn-Kadija-Noreen-Dinkar,P2B-400,"PowerBeam M2 400",XW.ar934x.v5.6.5.29033.160515.2108,145,36.25 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv deleted file mode 100644 index 000f5ed42d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","port","hostname","asn","geo","region","city","naics","sic","product","banner","sector" -"2019-09-04 14:51:44","198.123.245.53",5678,,5678,"AA","LOCATION","LOCATION",0,0,"Apple remote desktop vnc","RFB 003.889", -"2019-09-04 14:51:44","198.123.245.112",5678,"localhost.localdomain",5678,"AA","LOCATION","LOCATION",517311,0,"RealVNC Enterprise v5.3 or later","RFB 005.000", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv deleted file mode 100644 index 7e279ca3e6..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","response_size","amplification","error","raw_response" -"2010-02-10 00:00:00",192.168.0.1,udp,3702,node01.example.com,ws-discovery,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",989,164.83,"Validation constraint violation: SOAP message expected",c2FtcGxlIHJlc3BvbnNlIGRhdGEK -"2010-02-10 00:00:01",192.168.0.2,udp,3702,node02.example.com,ws-discovery,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",918,183.60,"Validation constraint violation: missing root element",c2FtcGxlIHJlc3BvbnNlIGRhdGEK -"2010-02-10 00:00:02",192.168.0.3,udp,3702,node03.example.com,ws-discovery,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",989,197.80,"Validation constraint violation: SOAP message expected",c2FtcGxlIHJlc3BvbnNlIGRhdGEK diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv.license deleted file mode 100644 index 9f58c89ef0..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2023 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv deleted file mode 100644 index 7e83bbaf8f..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","opcode","reported_hostname","status","size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,177,node01.example.com,xdmcp,64512,ZZ,Region,City,0,0,Willing,node01.example.com,"Linux 3.0.101-100-default",44,6.29 -"2010-02-10 00:00:01",192.168.0.2,udp,47074,node02.example.com,xdmcp,64512,ZZ,Region,City,0,0,Willing,node02.example.com,"Linux 2.6.9-103.ELsmp",48,6.86 -"2010-02-10 00:00:02",192.168.0.3,udp,177,node03.example.com,xdmcp,64512,ZZ,Region,City,0,0,Willing,node03.example.com,"1 user, load: 6,5, 6,6, 6,6",46,6.57 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/sinkhole_http_drone.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/sinkhole_http_drone.csv.license deleted file mode 100644 index 942a94035d..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/sinkhole_http_drone.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/special.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/special.csv deleted file mode 100644 index 2e7b591582..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/special.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","port","protocol","asn","geo","region","city","hostname","naics","sector","tag","public_source","status","detail","method","device_vendor" -"2010-02-10 00:00:00",192.168.0.1,,,64512,ZZ,Region,City,node01.example.com,0,"Communications, Service Provider, and Hosting Service",cyclops-blink,,"likely compromised",,, -"2010-02-10 00:00:01",192.168.0.2,,,64512,ZZ,Region,City,node02.example.com,0,"Communications, Service Provider, and Hosting Service",cyclops-blink,,"likely compromised",,, -"2010-02-10 00:00:02",192.168.0.3,,,64512,ZZ,Region,City,node03.example.com,0,"Professional, Scientific, and Technical Services",cyclops-blink,,"likely compromised",,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/special.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/special.csv.license deleted file mode 100644 index f512a890e4..0000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/special.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later From a33fa64569426ee47d039e1dd69bb2a76db52de7 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 11 Apr 2023 23:57:12 +0000 Subject: [PATCH 43/76] remove json parser - csv provides better performance --- .../shadowserver/collector_reports_api.py | 7 +- .../bots/parsers/shadowserver/parser_json.py | 171 ------------------ .../test_collector_reports_api.py | 7 +- 3 files changed, 7 insertions(+), 178 deletions(-) delete mode 100644 intelmq/bots/parsers/shadowserver/parser_json.py diff --git a/intelmq/bots/collectors/shadowserver/collector_reports_api.py b/intelmq/bots/collectors/shadowserver/collector_reports_api.py index 5e0b045c88..dc8bd6b420 100644 --- a/intelmq/bots/collectors/shadowserver/collector_reports_api.py +++ b/intelmq/bots/collectors/shadowserver/collector_reports_api.py @@ -34,7 +34,7 @@ class ShadowServerAPICollectorBot(CollectorBot, HttpMixin, CacheMixin): A list of strings or a comma-separated list of the mailing lists you want to process. types (list): A list of strings or a string of comma-separated values with the names of reporttypes you want to process. If you leave this empty, all the available reports will be downloaded and processed (i.e. 'scan', 'drones', 'intel', 'sandbox_connection', 'sinkhole_combined'). - file_format (str): File format to download ('csv' or 'json'). The default is 'json' for compatibility. Using 'csv' is recommended for best performance. + file_format (str): File format to download ('csv'). The 'json' option is not longer supported. """ country = None @@ -67,11 +67,10 @@ def init(self): self._report_list.append(self.country) if self.file_format is not None: - if not (self.file_format == 'csv' or self.file_format == 'json'): + if not (self.file_format == 'csv'): raise ValueError('Invalid file_format') else: - self.file_format = 'json' - self.logger.info("For best performance, set 'file_format' to 'csv' and use intelmq.bots.parsers.shadowserver.parser.") + self.file_format = 'csv' self.preamble = f'{{ "apikey": "{self.api_key}" ' diff --git a/intelmq/bots/parsers/shadowserver/parser_json.py b/intelmq/bots/parsers/shadowserver/parser_json.py deleted file mode 100644 index 893ad877b8..0000000000 --- a/intelmq/bots/parsers/shadowserver/parser_json.py +++ /dev/null @@ -1,171 +0,0 @@ -""" -Shadowserver JSON Parser - -SPDX-FileCopyrightText: 2020 Intelmq Team -SPDX-License-Identifier: AGPL-3.0-or-later -""" -import re -from typing import Any - -from intelmq.lib.bot import ParserBot -from intelmq.lib.exceptions import InvalidKey, InvalidValue -import intelmq.lib.message as libmessage -import intelmq.bots.parsers.shadowserver._config as config - - -class ShadowserverJSONParserBot(ParserBot): - """Parse all Shadowserver feeds in JSON format (data coming from the reports API) - Shadowserver JSON Parser - - Parameters: - feedname (str): The name of the feed - """ - __is_filename_regex = re.compile(r'^(?:\d{4}-\d{2}-\d{2}-)?(\w+)(-\w+)*\.json$') - feedname = None - _sparser_config = None - recover_line = ParserBot.recover_line_json - overwrite = True - - def init(self): - if self.feedname is not None: - feedname = self.feedname - self._sparser_config = config.get_feed_by_feedname(feedname) - if self._sparser_config: - self.logger.info('Using fixed feed name %r for parsing reports.', feedname) - else: - self.logger.info('Could not determine the feed by the feed name %r given by parameter. ' - 'Will determine the feed from the file names.', feedname) - - def parse(self, report): - report_name = report.get('extra.file_name') - if not report_name: - raise ValueError("No feedname given as parameter and the " - "processed report has no 'extra.file_name'. " - "Ensure that at least one is given. " - "Also have a look at the documentation of the bot.") - - filename_search = self.__is_filename_regex.search(report_name) - - if not filename_search: - raise ValueError(f"Report's 'extra.file_name' {report_name!r} is not valid.") - report_name = filename_search.group(1) - - self.logger.debug("Detected report's file name: %s.", report_name) - retval = config.get_feed_by_filename(report_name) - - if not retval: - raise ValueError('Could not get a config for {!r}, check the documentation.' - ''.format(report_name)) - self.feedname, self._sparser_config = retval - - return self.parse_json(report) - - def parse_line(self, line: Any, report: libmessage.Report): - conf = self._sparser_config - processedkeys = [] - - event = self.new_event(report) - event.add('feed.name', self.feedname, overwrite=self.overwrite) - - extra = {} - - for entry in conf.get('required_fields'): - intelmqkey, shadowserverkey = entry[0], entry[1] - value = self.get_value_from_config(line, entry) - - if value is not None: - event.add(intelmqkey, value) - processedkeys.append(shadowserverkey) - - # Now add optional fields. - # This action may fail, the value is added to - # extra if an add operation failed - for entry in conf.get('optional_fields'): - intelmqkey, shadowserverkey = entry[0], entry[1] - try: - value = self.get_value_from_config(line, entry) - except ValueError: - self.logger.warning('Optional key %s not found in feed %s. Possible change in data' - ' format or misconfiguration.', shadowserverkey, self.feedname) - continue - - intelmqkey, shadowserverkey = entry[0], entry[1] - if value is not None: - if intelmqkey == 'extra.': - extra[shadowserverkey] = value - processedkeys.append(shadowserverkey) - continue - elif intelmqkey and intelmqkey.startswith('extra.'): - extra[intelmqkey.replace('extra.', '', 1)] = value - processedkeys.append(shadowserverkey) - continue - elif intelmqkey is False: - # ignore it explicitly - processedkeys.append(shadowserverkey) - continue - try: - event.add(intelmqkey, value) - processedkeys.append(shadowserverkey) - except InvalidValue: - self.logger.debug('Could not add key %r in feed %r, adding it to extras.', - shadowserverkey, self.feedname) - except InvalidKey: - extra[intelmqkey] = value - processedkeys.append(shadowserverkey) - else: - processedkeys.append(shadowserverkey) - - # Now add additional constant fields. - event.update(conf.get('constant_fields', {})) - - event.add('raw', self.recover_line_json(line)) - - # Add everything which could not be resolved to extra. - for key in line: - if key not in processedkeys: - val = line[key] - if not val == "": - extra[key] = val - - if extra: - event.add('extra', extra) - - yield event - - def get_value_from_config(self, data, entry): - """ - Given a specific config, get the value for that data based on the entry - """ - conv_fun = None - - shadowserverkey = entry[1] - raw_value = data.get(shadowserverkey, None) - value = raw_value - - if raw_value is None: - raise ValueError('Key {!r} not found in feed {!r}. Possible change in data' - ' format or misconfiguration.'.format(shadowserverkey, self.feedname)) - if len(entry) > 2: - conv_fun = entry[2] - - if conv_fun is not None and raw_value is not None: - if len(entry) == 4 and entry[3]: - try: - value = conv_fun(raw_value, data) - except Exception: - self.logger.error('Could not convert shadowserverkey: %r in feed %r, ' - 'value: %r via conversion function %r.', - shadowserverkey, self.feedname, raw_value, conv_fun.__name__) - raise - else: - try: - value = conv_fun(raw_value) - except Exception: - self.logger.error('Could not convert shadowserverkey: %r in feed %r, ' - 'value: %r via conversion function %r.', - shadowserverkey, self.feedname, raw_value, conv_fun.__name__) - raise - return value - - -BOT = ShadowserverJSONParserBot diff --git a/intelmq/tests/bots/collectors/shadowserver/test_collector_reports_api.py b/intelmq/tests/bots/collectors/shadowserver/test_collector_reports_api.py index a625c9d34f..2bf6e61e9a 100644 --- a/intelmq/tests/bots/collectors/shadowserver/test_collector_reports_api.py +++ b/intelmq/tests/bots/collectors/shadowserver/test_collector_reports_api.py @@ -14,12 +14,13 @@ RANDSTR = secrets.token_urlsafe(50) ASSET_PATH = pathlib.Path(__file__).parent / 'reports-list.json' PARAMETERS = {'reports': 'anarres', 'api_key': RANDSTR, 'secret': RANDSTR, 'logging_level': 'DEBUG', 'types': ['scan_smb', 'cisco_smart_install', 'nonexistent'], 'name': 'shadowservercollector'} -REPORT = {'__type': 'Report', 'extra.file_name': '2020-08-02-scan_smb-anarres-geo.json', 'feed.accuracy': 100.0, 'feed.name': 'shadowservercollector', 'raw': 'e30='} +REPORT = {'__type': 'Report', 'extra.file_name': '2020-08-02-scan_smb-anarres-geo.csv', 'feed.accuracy': 100.0, 'feed.name': 'shadowservercollector', 'raw': 'e30='} def prepare_mocker(mocker): mocker.post('https://transform.shadowserver.org/api2/reports/list', content=ASSET_PATH.read_bytes()) - mocker.post('https://transform.shadowserver.org/api2/reports/download', text='{}') + mocker.get('https://dl.shadowserver.org/xNDSuwXrKnrLrDopU926rR75CAESMWesVCKsuyI8b8ncTv7GCX', text='{}') + mocker.get('https://dl.shadowserver.org/unnzVtn92tS9459rKIEz2J8qb7oJDv0Fa2feGUOiJLCDLqBXnN', text='{}') # Explicit skip_redis is required (although implicitly called by no_cache), otherwise fails in package build environments @@ -80,7 +81,7 @@ def test_report_sent(self, mocker): self.cache.flushdb() prepare_mocker(mocker) self.run_bot(iterations=1, parameters=PARAMETERS) - self.assertAnyLoglineEqual("Sent report: '2020-08-02-cisco_smart_install-anarres-geo.csv' (fixed: '2020-08-02-cisco_smart_install-anarres-geo.json', size: 0.00195 KiB).", 'DEBUG') + self.assertAnyLoglineEqual("Sent report: '2020-08-02-cisco_smart_install-anarres-geo.csv' (fixed: '2020-08-02-cisco_smart_install-anarres-geo.csv', size: 0.00195 KiB).", 'DEBUG') def test_report_content(self, mocker): self.cache.flushdb() From cd3338a3fc938cb14fa020996a6b71dfd7203697 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 11 Apr 2023 23:59:42 +0000 Subject: [PATCH 44/76] dynamic configuration model --- intelmq/bots/parsers/shadowserver/README.md | 7 + intelmq/bots/parsers/shadowserver/_config.py | 4202 +---------------- intelmq/bots/parsers/shadowserver/parser.py | 46 +- .../parsers/shadowserver/schema.json.test | 180 + .../parsers/shadowserver/update_schema.py | 12 + 5 files changed, 303 insertions(+), 4144 deletions(-) create mode 100644 intelmq/bots/parsers/shadowserver/schema.json.test create mode 100644 intelmq/bots/parsers/shadowserver/update_schema.py diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md index eb0ddfb4a7..297930861b 100644 --- a/intelmq/bots/parsers/shadowserver/README.md +++ b/intelmq/bots/parsers/shadowserver/README.md @@ -7,3 +7,10 @@ This module is maintained by [The Shadowserver Foundation](https://www.shadowser Please contact intelmq@shadowserver.org with any issues or concerns. +The report configuration is now stored in a _schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/_version_. + +For environments that have internet connectivity the `update_schema.py` script should be setup as a cron job to obtain the latest revision. + +For air-gapped systems automation will be required to download and copy the _schema.json_ file into this directory + +The parser will automatically reload the configuration when the file changes. diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index bea3d0c0b8..a7b80b7a6c 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -77,20 +77,34 @@ feed_idx is not complete. """ +import os import re import base64 import binascii +import json +import urllib.request +import tempfile from typing import Optional, Dict, Tuple, Any import intelmq.lib.harmonization as harmonization +class __Container: + pass + +__config = __Container() +__config.schema_file = os.path.join(os.path.dirname(__file__), 'schema.json') +__config.schema_mtime = 0.0 +__config.feedname_mapping = {} +__config.filename_mapping = {} def get_feed_by_feedname(given_feedname: str) -> Optional[Dict[str, Any]]: - return feedname_mapping.get(given_feedname, None) + reload() + return __config.feedname_mapping.get(given_feedname, None) def get_feed_by_filename(given_filename: str) -> Optional[Tuple[str, Dict[str, Any]]]: - return filename_mapping.get(given_filename, None) + reload() + return __config.filename_mapping.get(given_filename, None) def add_UTC_to_timestamp(value: str) -> str: @@ -165,11 +179,6 @@ def invalidate_zero(value: str) -> Optional[int]: return int(value) if value and int(value) != 0 else None -# TODO this function is a wild guess... -def set_tor_node(value: str) -> Optional[bool]: - return True if value else None - - def validate_ip(value: str) -> Optional[str]: """Remove "invalid" IP.""" # FIX: https://github.com/certtools/intelmq/issues/1720 # TODO: Find better fix @@ -240,4126 +249,63 @@ def scan_exchange_identifier(field): return 'exchange-server-webshell' return 'vulnerable-exchange-server' +functions = { + 'add_UTC_to_timestamp': add_UTC_to_timestamp, + 'convert_bool': convert_bool, + 'validate_to_none': validate_to_none, + 'convert_int': convert_int, + 'convert_float': convert_float, + 'convert_http_host_and_url': convert_http_host_and_url, + 'invalidate_zero': invalidate_zero, + 'validate_ip': validate_ip, + 'validate_network': validate_network, + 'validate_fqdn': validate_fqdn, + 'convert_date': convert_date, + 'convert_date_utc': convert_date_utc, + 'force_base64': force_base64, + 'scan_exchange_taxonomy': scan_exchange_taxonomy, + 'scan_exchange_type': scan_exchange_type, + 'scan_exchange_identifier': scan_exchange_identifier, + } + + +def reload (): + """ reload the configuration if it has changed """ + mtime = 0.0 + + if (os.path.isfile(__config.schema_file)): + mtime = os.path.getmtime(__config.schema_file) + if __config.schema_mtime == mtime: + return + schema_file = __config.schema_file + else: + # load a test schema if one has not been downloaded yet + schema_file = __config.schema_file + schema_file += '.test' + + __config.feedname_mapping.clear() + __config.filename_mapping.clear() + with open(schema_file) as fh: + schema = json.load(fh) + for report in schema: + __config.feedname_mapping[schema[report]['feed_name']] = (schema[report]['feed_name'], schema[report]) + __config.filename_mapping[schema[report]['file_name']] = (schema[report]['feed_name'], schema[report]) + __config.schema_mtime = mtime + +def update_schema (version): + """ download the latest configuration """ + (th, tmp) = tempfile.mkstemp() + url = 'https://interchange.shadowserver.org/intelmq/'+version + try: + urllib.request.urlretrieve(url, tmp) + except: + raise ValueError("Failed to download %r" % url) -# BEGIN CONFGEN - -# https://www.shadowserver.org/what-we-do/network-reporting/blocklist-report/ -blocklist = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ], - 'optional_fields': [ - ('source.network', 'ip', validate_network), - ('extra.', 'tag', validate_to_none), - ('source.reverse_dns', 'hostname'), - ('extra.', 'source', validate_to_none), - ('extra.', 'reason', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'blacklisted-ip', - 'classification.taxonomy': 'other', - 'classification.type': 'blacklist', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/compromised-website-report/ -compromised_website = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.application', 'application', validate_to_none), - ('source.url', 'url', convert_http_host_and_url, True), - ('source.fqdn', 'http_host', validate_fqdn), - ('source.reverse_dns', 'hostname'), - ('malware.name', 'tag'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('event_description.text', 'category', validate_to_none), - ('extra.', 'system', validate_to_none), - ('extra.', 'detected_since', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'redirect_target', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'cc_url', validate_to_none), - ('extra.', 'family', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'intrusions', - 'classification.type': 'system-compromise', - 'classification.identifier': 'compromised-website', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/device-identification-report/ -device_id = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'undetermined', - 'classification.identifier': 'device-id', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/ddos-participant-report/ -event_ddos_participant = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'duration', convert_int), - ('extra.', 'attack_src_port', convert_int), - ('extra.', 'http_usessl', convert_bool), - ('extra.', 'ip_header_seqnum', convert_int), - ('extra.', 'ip_header_ttl', convert_int), - ('extra.', 'number_of_connections', convert_int), - ('extra.', 'packet_length', convert_int), - ('extra.', 'packet_randomized', convert_bool), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'domain_source', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'dst_network', validate_to_none), - ('extra.', 'dst_netmask', validate_to_none), - ('extra.', 'attack', validate_to_none), - ('extra.', 'attack_src_ip', validate_to_none), - ('extra.', 'domain', validate_to_none), - ('extra.', 'domain_transaction_id', validate_to_none), - ('extra.', 'gcip', validate_to_none), - ('extra.', 'http_method', validate_to_none), - ('extra.', 'http_path', validate_to_none), - ('extra.', 'http_postdata', validate_to_none), - ('extra.', 'ip_header_ack', validate_to_none), - ('extra.', 'ip_header_acknum', validate_to_none), - ('extra.', 'ip_header_dont_fragment', validate_to_none), - ('extra.', 'ip_header_fin', validate_to_none), - ('extra.', 'ip_header_identity', validate_to_none), - ('extra.', 'ip_header_psh', validate_to_none), - ('extra.', 'ip_header_rst', validate_to_none), - ('extra.', 'ip_header_syn', validate_to_none), - ('extra.', 'ip_header_tos', validate_to_none), - ('extra.', 'ip_header_urg', validate_to_none), - ('extra.', 'http_agent', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'classification.identifier': 'ddos-participant', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-brute-force-events-report/ -event_honeypot_brute_force = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('classification.identifier', 'application'), - ('destination.account', 'username', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'service', validate_to_none), - ('extra.', 'start_time', convert_date_utc), - ('extra.', 'end_time', convert_date_utc), - ('extra.', 'client_version', validate_to_none), - ('extra.', 'password', validate_to_none), - ('extra.', 'payload_url', validate_to_none), - ('extra.', 'payload_md5', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'intrusion-attempts', - 'classification.type': 'brute-force', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-darknet-events-report/ -event_honeypot_darknet = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('classification.identifier', 'tag', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'count', convert_int), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-ddos-events/ -event_honeypot_ddos = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'duration', convert_int), - ('extra.', 'attack_src_port', convert_int), - ('extra.', 'http_usessl', convert_bool), - ('extra.', 'ip_header_seqnum', convert_int), - ('extra.', 'ip_header_ttl', convert_int), - ('extra.', 'number_of_connections', convert_int), - ('extra.', 'packet_length', convert_int), - ('extra.', 'packet_randomized', convert_bool), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'domain_source', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'dst_network', validate_to_none), - ('extra.', 'dst_netmask', validate_to_none), - ('extra.', 'attack', validate_to_none), - ('extra.', 'attack_src_ip', validate_to_none), - ('extra.', 'domain', validate_to_none), - ('extra.', 'domain_transaction_id', validate_to_none), - ('extra.', 'gcip', validate_to_none), - ('extra.', 'http_method', validate_to_none), - ('extra.', 'http_path', validate_to_none), - ('extra.', 'http_postdata', validate_to_none), - ('extra.', 'ip_header_ack', validate_to_none), - ('extra.', 'ip_header_acknum', validate_to_none), - ('extra.', 'ip_header_dont_fragment', validate_to_none), - ('extra.', 'ip_header_fin', validate_to_none), - ('extra.', 'ip_header_identity', validate_to_none), - ('extra.', 'ip_header_psh', validate_to_none), - ('extra.', 'ip_header_rst', validate_to_none), - ('extra.', 'ip_header_syn', validate_to_none), - ('extra.', 'ip_header_tos', validate_to_none), - ('extra.', 'ip_header_urg', validate_to_none), - ('extra.', 'http_agent', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'classification.identifier': 'honeypot-ddos', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-amplification-ddos-events-report/ -event_honeypot_ddos_amp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'avg_pps', convert_float), - ('extra.', 'max_pps', convert_float), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'request', validate_to_none), - ('extra.', 'count', convert_int), - ('extra.', 'bytes', convert_int), - ('extra.', 'end_time', convert_date_utc), - ('extra.', 'duration', convert_int), - ], - 'constant_fields': { - 'classification.identifier': 'amplification-ddos-victim', - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-ddos-target-events-report/ -event_honeypot_ddos_target = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'attack_src_port', convert_int), - ('extra.', 'http_usessl', convert_bool), - ('extra.', 'ip_header_seqnum', convert_int), - ('extra.', 'ip_header_ttl', convert_int), - ('extra.', 'number_of_connections', convert_int), - ('extra.', 'packet_length', convert_int), - ('extra.', 'packet_randomized', convert_bool), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'domain_source', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'dst_network', validate_to_none), - ('extra.', 'dst_netmask', validate_to_none), - ('extra.', 'attack', validate_to_none), - ('extra.', 'duration', convert_int), - ('extra.', 'attack_src_ip', validate_to_none), - ('extra.', 'domain', validate_to_none), - ('extra.', 'domain_transaction_id', validate_to_none), - ('extra.', 'gcip', validate_to_none), - ('extra.', 'http_method', validate_to_none), - ('extra.', 'http_path', validate_to_none), - ('extra.', 'http_postdata', validate_to_none), - ('extra.', 'ip_header_ack', validate_to_none), - ('extra.', 'ip_header_acknum', validate_to_none), - ('extra.', 'ip_header_dont_fragment', validate_to_none), - ('extra.', 'ip_header_fin', validate_to_none), - ('extra.', 'ip_header_identity', validate_to_none), - ('extra.', 'ip_header_psh', validate_to_none), - ('extra.', 'ip_header_rst', validate_to_none), - ('extra.', 'ip_header_syn', validate_to_none), - ('extra.', 'ip_header_tos', validate_to_none), - ('extra.', 'ip_header_urg', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'classification.identifier': 'honeypot-ddos-target', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-http-scanner-events/ -event_honeypot_http_scan = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('user_agent', 'http_agent', validate_to_none), - ('extra.method', 'http_request_method', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'pattern', validate_to_none), - ('destination.url', 'http_url', convert_http_host_and_url, True), - ('extra.', 'url_scheme', validate_to_none), - ('extra.', 'session_tags', validate_to_none), - ('extra.', 'vulnerability_enum', validate_to_none), - ('extra.', 'vulnerability_id', validate_to_none), - ('extra.', 'vulnerability_class', validate_to_none), - ('extra.', 'vulnerability_score', validate_to_none), - ('extra.', 'vulnerability_severity', validate_to_none), - ('extra.', 'vulnerability_version', validate_to_none), - ('extra.', 'threat_framework', validate_to_none), - ('extra.', 'threat_tactic_id', validate_to_none), - ('extra.', 'threat_technique_id', validate_to_none), - ('extra.', 'target_vendor', validate_to_none), - ('extra.', 'target_product', validate_to_none), - ('extra.', 'target_class', validate_to_none), - ('extra.', 'file_md5', validate_to_none), - ('extra.', 'file_sha256', validate_to_none), - ('extra.', 'request_raw', force_base64), - ('extra.', 'body_raw', force_base64), - ], - 'constant_fields': { - 'classification.taxonomy': 'information-gathering', - 'classification.type': 'scanner', - 'protocol.application': 'http', - 'classification.identifier': 'honeypot-http-scan', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-ics-scanner-events-report/ -event_honeypot_ics_scan = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'state', validate_to_none), - ('extra.', 'sensor_id', validate_to_none), - ('extra.', 'slave_id', validate_to_none), - ('extra.', 'function_code', validate_to_none), - ('extra.', 'request', validate_to_none), - ('extra.', 'response', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'ics', - 'classification.taxonomy': 'information-gathering', - 'classification.type': 'scanner', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/ip-spoofer-events-report/ -event_ip_spoofer = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'infection', validate_to_none), - ('source.network', 'network', validate_network), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'routedspoof', validate_to_none), - ('extra.', 'session', validate_to_none), - ('extra.', 'nat', convert_bool), - ], - 'constant_fields': { - 'classification.taxonomy': 'fraud', - 'classification.type': 'masquerade', - 'classification.identifier': 'ip-spoofer', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/sinkhole-events-report/ -event_sinkhole = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('classification.identifier', 'infection', validate_to_none), - ('malware.name', 'family', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('extra.', 'infection', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/sinkhole-dns-events-report/ -event_sinkhole_dns = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('extra.naics', 'src_naics', invalidate_zero), - ('extra.sector', 'src_sector', validate_to_none), - ('extra.dns_query_type', 'query_type'), - ('extra.dns_query', 'query'), - ('malware.name', 'family', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('extra.', 'infection', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'count', convert_int), - ], - 'constant_fields': { - 'classification.identifier': 'sinkholedns', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'dns', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/sinkhole-http-events-report/ -event_sinkhole_http = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('classification.identifier', 'tag'), - ('malware.name', 'family', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('extra.', 'infection', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('destination.url', 'http_url', convert_http_host_and_url, True), - ('destination.fqdn', 'http_host', validate_fqdn), - ('extra.', 'http_agent', validate_to_none), - ('extra.', 'forwarded_by', validate_to_none), - ('extra.', 'ssl_cipher', validate_to_none), - ('extra.', 'http_referer', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'protocol.application': 'http', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/sinkhole-http-referer-events-report/ -event_sinkhole_http_referer = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ], - 'optional_fields': [ - ('malware.name', 'family', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('extra.', 'infection', validate_to_none), - ('protocol.transport', 'protocol'), - ('extra.', 'http_referer_ip', validate_ip), - ('extra.', 'http_referer_port', convert_int), - ('extra.', 'http_referer_asn', invalidate_zero), - ('extra.', 'http_referer_geo', validate_to_none), - ('extra.', 'http_referer_region', validate_to_none), - ('extra.', 'http_referer_city', validate_to_none), - ('extra.', 'http_referer_hostname', validate_to_none), - ('extra.', 'http_referer_naics', invalidate_zero), - ('extra.', 'http_referer_sector', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('destination.url', 'http_url', convert_http_host_and_url, True), - ('destination.fqdn', 'http_host', validate_fqdn), - ('extra.', 'http_referer', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'classification.identifier': 'sinkhole-http-referer', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/malware-url-report/ -malware_url = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ], - 'optional_fields': [ - ('source.url', 'url', convert_http_host_and_url, True), - ('source.fqdn', 'host', validate_fqdn), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('malware.name', 'tag'), - ('extra.', 'source', validate_to_none), - ('malware.hash.sha256', 'sha256', validate_to_none), - ('extra.', 'application', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'malware-distribution', - 'classification.identifier': 'malware-url', - }, -} - -phish_url = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ], - 'optional_fields': [ - ('source.url', 'url', convert_http_host_and_url, True), - ('source.fqdn', 'host', validate_fqdn), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'source', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'fraud', - 'classification.type': 'phishing', - 'classification.identifier': 'phish-url', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-http-proxy-report/ -population_http_proxy = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('malware.name', 'tag'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'http', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'connection', validate_to_none), - ('extra.', 'proxy_authenticate', validate_to_none), - ('extra.', 'via', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-http-proxy', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'http', - }, -} - -# http://www.shadowserver.org/wiki/pmwiki.php/Services/Sandbox-Connection -sandbox_conn = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('destination.fqdn', 'host', validate_fqdn), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('malware.hash.md5', 'md5', validate_to_none), - ('protocol.transport', 'protocol'), - ('extra.', 'bytes_in', validate_to_none), - ('extra.', 'bytes_out', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'malware-distribution', - 'classification.identifier': 'sandbox-conn', - }, -} - -sandbox_dns = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ], - 'optional_fields': [ - ('extra.dns_query_type', 'type', validate_to_none), - ('malware.hash.md5', 'md5hash', validate_to_none), - ('extra.', 'request', validate_to_none), - ('extra.', 'response', validate_to_none), - ('extra.', 'family', validate_to_none), - ('malware.name', 'tag'), - ('extra.', 'source', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'dns', - 'classification.identifier': 'sandbox-dns', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/sandbox-url-report/ -sandbox_url = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ], - 'optional_fields': [ - ('destination.fqdn', 'host', validate_fqdn), - ('extra.http_request_method', 'method', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('malware.hash.md5', 'md5', validate_to_none), - ('destination.url', 'url', convert_http_host_and_url, True), - ('user_agent', 'user_agent', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'malware-distribution', - 'classification.identifier': 'sandbox-url', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-adb-report/ -scan_adb = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'name', validate_to_none), - ('extra.', 'model', validate_to_none), - ('extra.', 'device', validate_to_none), - ('extra.', 'features', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_sector', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-adb', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'adb', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-afp-report/ -scan_afp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'machine_type', validate_to_none), - ('extra.', 'afp_versions', validate_to_none), - ('extra.', 'uams', validate_to_none), - ('extra.', 'flags', validate_to_none), - ('extra.', 'server_name', validate_to_none), - ('extra.', 'signature', validate_to_none), - ('extra.', 'directory_service', validate_to_none), - ('extra.', 'utf8_servername', validate_to_none), - ('extra.', 'network_address', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-afp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'afp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-amqp-report/ -scan_amqp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'channel', validate_to_none), - ('extra.', 'message_length', convert_int), - ('extra.', 'class', validate_to_none), - ('extra.', 'method', validate_to_none), - ('extra.', 'version_major', validate_to_none), - ('extra.', 'version_minor', validate_to_none), - ('extra.', 'capabilities', validate_to_none), - ('extra.', 'cluster_name', validate_to_none), - ('extra.', 'platform', validate_to_none), - ('extra.', 'product', validate_to_none), - ('extra.', 'product_version', validate_to_none), - ('extra.', 'mechanisms', validate_to_none), - ('extra.', 'locales', validate_to_none), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-amqp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'amqp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-apple-remote-desktop-ard-report/ -scan_ard = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'machine_name', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-ard', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-chargen-report/ -scan_chargen = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.response_size', 'size', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'chargen', - 'classification.identifier': 'open-chargen', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-cisco-smart-install-report/ -scan_cisco_smart_install = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-cisco-smart-install', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'cisco-smart-install', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-coap-report/ -scan_coap = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'response', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-coap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'coap', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-couchdb-report/ -scan_couchdb = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'server_version', validate_to_none), - ('extra.', 'couchdb_message', validate_to_none), - ('extra.', 'couchdb_version', validate_to_none), - ('extra.', 'git_sha', validate_to_none), - ('extra.', 'features', validate_to_none), - ('extra.', 'vendor', validate_to_none), - ('extra.', 'visible_databases', validate_to_none), - ('extra.', 'error', validate_to_none), - ('extra.', 'error_reason', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'CouchDB', - 'classification.identifier': 'open-couchdb', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-cwmp-report/ -scan_cwmp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'http', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'connection', validate_to_none), - ('extra.', 'www_authenticate', validate_to_none), - ('extra.', 'set_cookie', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'date', validate_to_none), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'cwmp', - 'classification.identifier': 'open-cwmp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-db2-discovery-service-report/ -scan_db2 = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'size', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'db2_hostname', validate_to_none), - ('extra.', 'servername', validate_to_none), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'open-db2-discovery-service', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'db2', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/vulnerable-ddos-middlebox-report/ -scan_ddos_middlebox = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.application', 'tag'), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'source_port', validate_to_none), - ('extra.', 'bytes', convert_int), - ('extra.', 'amplification', convert_float), - ('extra.', 'method', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-ddos-middlebox', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/dns-open-resolvers-report/ -scan_dns = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'min_amplification', convert_float), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'dns_version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'dns-open-resolver', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'dns', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-docker-service-report/ -scan_docker = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'http', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'date', validate_to_none), - ('extra.', 'experimental', validate_to_none), - ('extra.', 'api_version', validate_to_none), - ('extra.', 'arch', validate_to_none), - ('extra.', 'go_version', validate_to_none), - ('extra.os.name', 'os', validate_to_none), - ('extra.', 'kernel_version', validate_to_none), - ('extra.', 'git_commit', validate_to_none), - ('extra.', 'min_api_version', validate_to_none), - ('extra.', 'build_time', validate_to_none), - ('extra.', 'pkg_version', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'docker', - 'classification.identifier': 'open-docker', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-dvr-dhcpdiscover-report/ -scan_dvr_dhcpdiscover = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.application', 'tag'), - ('extra.', 'video_input_channels', convert_int), - ('extra.', 'alarm_input_channels', convert_int), - ('extra.', 'video_output_channels', convert_int), - ('extra.', 'alarm_output_channels', convert_int), - ('extra.', 'remote_video_input_channels', convert_int), - ('extra.', 'ipv4_dhcp_enable', convert_bool), - ('extra.', 'ipv6_dhcp_enable', convert_bool), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_id', validate_to_none), - ('extra.', 'device_serial', validate_to_none), - ('extra.', 'machine_name', validate_to_none), - ('extra.', 'manufacturer', validate_to_none), - ('extra.', 'method', validate_to_none), - ('extra.', 'http_port', convert_int), - ('extra.', 'internal_port', convert_int), - ('extra.', 'mac_address', validate_to_none), - ('extra.', 'ipv4_address', validate_to_none), - ('extra.', 'ipv4_gateway', validate_to_none), - ('extra.', 'ipv4_subnet_mask', validate_to_none), - ('extra.', 'ipv6_address', validate_to_none), - ('extra.', 'ipv6_link_local', validate_to_none), - ('extra.', 'ipv6_gateway', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-dvr-dhcpdiscover', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-elasticsearch-report/ -scan_elasticsearch = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'build_snapshot', convert_bool), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'ok', convert_bool), - ('extra.', 'name', validate_to_none), - ('extra.', 'cluster_name', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'build_hash', validate_to_none), - ('extra.', 'build_timestamp', validate_to_none), - ('extra.', 'lucene_version', validate_to_none), - ('extra.', 'tagline', validate_to_none), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'elasticsearch', - 'classification.identifier': 'open-elasticsearch', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-erlang-port-mapper-report-daemon/ -scan_epmd = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'nodes', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'Erlang Port Mapper Daemon', - 'classification.identifier': 'open-epmd', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/vulnerable-exchange-server-report/ -scan_exchange = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('classification.taxonomy', 'tag', scan_exchange_taxonomy), - ('classification.type', 'tag', scan_exchange_type), - ('classification.identifier', 'tag', scan_exchange_identifier), - ('extra.', 'tag', validate_to_none), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'servername', validate_to_none), - ('destination.url', 'url', convert_http_host_and_url, True), - ], - 'constant_fields': { - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-ftp-report/ -scan_ftp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'banner', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'auth_tls_response', validate_to_none), - ('extra.', 'auth_ssl_response', validate_to_none), - ('extra.', 'tlsv13_support', validate_to_none), - ('extra.', 'tlsv13_cipher', validate_to_none), - ('extra.', 'jarm', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_sector', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-ftp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ftp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-hadoop-report/ -scan_hadoop = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'total_disk', convert_int), - ('extra.', 'used_disk', convert_int), - ('extra.', 'free_disk', convert_int), - ('source.reverse_dns', 'hostname'), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'server_type', validate_to_none), - ('extra.', 'clusterid', validate_to_none), - ('extra.', 'livenodes', validate_to_none), - ('extra.', 'namenodeaddress', validate_to_none), - ('extra.', 'volumeinfo', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-hadoop', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'hadoop', - 'protocol.transport': 'tcp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-http-report/ -scan_http = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'http', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'connection', validate_to_none), - ('extra.', 'www_authenticate', validate_to_none), - ('extra.', 'set_cookie', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-http', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'http', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-http-proxy-report/ -scan_http_proxy = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'http', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'connection', validate_to_none), - ('extra.', 'proxy_authenticate', validate_to_none), - ('extra.', 'via', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), - ], - 'constant_fields': { - 'classification.identifier': 'open-http-proxy', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'http', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/vulnerable-http-report/ -scan_http_vulnerable = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'http', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'connection', validate_to_none), - ('extra.', 'www_authenticate', validate_to_none), - ('extra.', 'set_cookie', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), - ('extra.', 'version', validate_to_none), - ('extra.', 'build_date', validate_to_none), - ('extra.', 'detail', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-http', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'http', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-ics-report/ -scan_ics = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.application', 'tag'), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_id', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'raw_response', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-ics', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-ipmi-report/ -scan_ipmi = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'none_auth', convert_bool), - ('extra.', 'md2_auth', convert_bool), - ('extra.', 'md5_auth', convert_bool), - ('extra.', 'passkey_auth', convert_bool), - ('extra.', 'oem_auth', convert_bool), - ('extra.', 'permessage_auth', convert_bool), - ('extra.', 'userlevel_auth', convert_bool), - ('extra.', 'usernames', convert_bool), - ('extra.', 'nulluser', convert_bool), - ('extra.', 'anon_login', convert_bool), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'ipmi_version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'defaultkg', validate_to_none), - ('extra.', 'error', validate_to_none), - ('extra.', 'deviceid', validate_to_none), - ('extra.', 'devicerev', validate_to_none), - ('extra.', 'firmwarerev', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'manufacturerid', validate_to_none), - ('extra.', 'manufacturername', validate_to_none), - ('extra.', 'productid', validate_to_none), - ('extra.', 'productname', validate_to_none), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ipmi', - 'protocol.transport': 'udp', - 'classification.identifier': 'open-ipmi', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-ipp-report/ -scan_ipp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'ipp_version', validate_to_none), - ('extra.', 'cups_version', validate_to_none), - ('extra.', 'printer_uris', validate_to_none), - ('extra.', 'printer_name', validate_to_none), - ('extra.', 'printer_info', validate_to_none), - ('extra.', 'printer_more_info', validate_to_none), - ('extra.', 'printer_make_and_model', validate_to_none), - ('extra.', 'printer_firmware_name', validate_to_none), - ('extra.', 'printer_firmware_string_version', validate_to_none), - ('extra.', 'printer_firmware_version', validate_to_none), - ('extra.', 'printer_organization', validate_to_none), - ('extra.', 'printer_organization_unit', validate_to_none), - ('extra.', 'printer_uuid', validate_to_none), - ('extra.', 'printer_wifi_ssid', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ipp', - 'classification.identifier': 'open-ipp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/vulnerable-isakmp-report/ -scan_isakmp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'spi_size', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'initiator_spi', validate_to_none), - ('extra.', 'responder_spi', validate_to_none), - ('extra.', 'next_payload', validate_to_none), - ('extra.', 'exchange_type', validate_to_none), - ('extra.', 'flags', validate_to_none), - ('extra.', 'message_id', validate_to_none), - ('extra.', 'next_payload2', validate_to_none), - ('extra.', 'domain_of_interpretation', validate_to_none), - ('extra.', 'protocol_id', validate_to_none), - ('extra.', 'notify_message_type', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'open-ike', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ipsec', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-kubernetes-api-server-report/ -scan_kubernetes = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'http', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'date', validate_to_none), - ('extra.', 'major', validate_to_none), - ('extra.', 'minor', validate_to_none), - ('extra.', 'git_version', validate_to_none), - ('extra.', 'git_commit', validate_to_none), - ('extra.', 'git_tree_state', validate_to_none), - ('extra.', 'build_date', validate_to_none), - ('extra.', 'go_version', validate_to_none), - ('extra.', 'compiler', validate_to_none), - ('extra.', 'platform', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'browser_trusted', convert_bool), - ('extra.', 'browser_error', validate_to_none), - ('extra.', 'raw_cert', validate_to_none), - ('extra.', 'raw_cert_chain', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'kubernetes', - 'classification.identifier': 'open-kubernetes', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-ldap-tcp-report/ -scan_ldap_tcp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.local_hostname', 'dns_host_name', validate_to_none), - ('extra.', 'domain_controller_functionality', convert_int), - ('extra.', 'domain_functionality', convert_int), - ('extra.', 'forest_functionality', convert_int), - ('extra.', 'highest_committed_usn', convert_int), - ('extra.', 'is_global_catalog_ready', convert_bool), - ('extra.', 'is_synchronized', convert_bool), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'size', convert_int), - ('extra.', 'configuration_naming_context', validate_to_none), - ('extra.', 'current_time', validate_to_none), - ('extra.', 'default_naming_context', validate_to_none), - ('extra.', 'ds_service_name', validate_to_none), - ('extra.', 'ldap_service_name', validate_to_none), - ('extra.', 'naming_contexts', validate_to_none), - ('extra.', 'root_domain_naming_context', validate_to_none), - ('extra.', 'schema_naming_context', validate_to_none), - ('extra.', 'server_name', validate_to_none), - ('extra.', 'subschema_subentry', validate_to_none), - ('extra.', 'supported_capabilities', validate_to_none), - ('extra.', 'supported_control', validate_to_none), - ('extra.', 'supported_ldap_policies', validate_to_none), - ('extra.', 'supported_ldap_version', validate_to_none), - ('extra.', 'supported_sasl_mechanisms', validate_to_none), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ldap', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-ldap-report/ -scan_ldap_udp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.local_hostname', 'dns_host_name', validate_to_none), - ('extra.', 'domain_controller_functionality', convert_int), - ('extra.', 'domain_functionality', convert_int), - ('extra.', 'forest_functionality', convert_int), - ('extra.', 'highest_committed_usn', convert_int), - ('extra.', 'is_global_catalog_ready', convert_bool), - ('extra.', 'is_synchronized', convert_bool), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'size', convert_int), - ('extra.', 'configuration_naming_context', validate_to_none), - ('extra.', 'current_time', validate_to_none), - ('extra.', 'default_naming_context', validate_to_none), - ('extra.', 'ds_service_name', validate_to_none), - ('extra.', 'ldap_service_name', validate_to_none), - ('extra.', 'naming_contexts', validate_to_none), - ('extra.', 'root_domain_naming_context', validate_to_none), - ('extra.', 'schema_naming_context', validate_to_none), - ('extra.', 'server_name', validate_to_none), - ('extra.', 'subschema_subentry', validate_to_none), - ('extra.', 'supported_capabilities', validate_to_none), - ('extra.', 'supported_control', validate_to_none), - ('extra.', 'supported_ldap_policies', validate_to_none), - ('extra.', 'supported_ldap_version', validate_to_none), - ('extra.', 'supported_sasl_mechanisms', validate_to_none), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ldap', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-mdns-report/ -scan_mdns = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'mdns_name', validate_to_none), - ('extra.', 'mdns_ipv4', validate_to_none), - ('extra.', 'mdns_ipv6', validate_to_none), - ('extra.', 'services', validate_to_none), - ('extra.', 'workstation_name', validate_to_none), - ('extra.', 'workstation_ipv4', validate_to_none), - ('extra.', 'workstation_ipv6', validate_to_none), - ('extra.', 'workstation_info', validate_to_none), - ('extra.', 'http_name', validate_to_none), - ('extra.', 'http_ipv4', validate_to_none), - ('extra.', 'http_ipv6', validate_to_none), - ('extra.', 'http_ptr', validate_to_none), - ('extra.', 'http_info', validate_to_none), - ('extra.', 'http_target', validate_to_none), - ('extra.', 'http_port', convert_int), - ('extra.', 'spotify_name', validate_to_none), - ('extra.', 'spotify_ipv4', validate_to_none), - ('extra.', 'spotify_ipv6', validate_to_none), - ('extra.', 'opc_ua_discovery', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'mdns', - 'classification.identifier': 'open-mdns', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-memcached-report/ -scan_memcached = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'pid', convert_int), - ('extra.', 'pointer_size', convert_int), - ('extra.', 'uptime', convert_int), - ('extra.', 'curr_connections', convert_int), - ('extra.', 'total_connections', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'time', validate_to_none), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'memcached', - 'classification.identifier': 'open-memcached', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-mongodb-report/ -scan_mongodb = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'gitversion', validate_to_none), - ('extra.', 'sysinfo', validate_to_none), - ('extra.', 'opensslversion', validate_to_none), - ('extra.', 'allocator', validate_to_none), - ('extra.', 'javascriptengine', validate_to_none), - ('extra.', 'bits', validate_to_none), - ('extra.', 'maxbsonobjectsize', validate_to_none), - ('extra.', 'ok', convert_bool), - ('extra.', 'visible_databases', validate_to_none), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'mongodb', - 'classification.identifier': 'open-mongodb', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-mqtt-report/ -scan_mqtt = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'anonymous_access', convert_bool), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'raw_response', validate_to_none), - ('extra.', 'hex_code', validate_to_none), - ('extra.', 'code', validate_to_none), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serialNumber', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'mqtt', - 'classification.identifier': 'open-mqtt', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-mqtt-report/ -scan_mqtt_anon = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'raw_response', validate_to_none), - ('extra.', 'hex_code', validate_to_none), - ('extra.', 'code', validate_to_none), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serialNumber', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'mqtt', - 'classification.identifier': 'open-mqtt-anon', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-MSSQL -scan_mssql = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.local_hostname', 'server_name', validate_to_none), - ('extra.', 'tcp_port', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'instance_name', validate_to_none), - ('extra.', 'named_pipe', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'mssql', - 'classification.identifier': 'open-mssql', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-mysql-server-report/ -scan_mysql = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'client_can_handle_expired_passwords', convert_bool), - ('extra.', 'client_compress', convert_bool), - ('extra.', 'client_connect_attrs', convert_bool), - ('extra.', 'client_connect_with_db', convert_bool), - ('extra.', 'client_deprecated_eof', convert_bool), - ('extra.', 'client_found_rows', convert_bool), - ('extra.', 'client_ignore_sigpipe', convert_bool), - ('extra.', 'client_ignore_space', convert_bool), - ('extra.', 'client_interactive', convert_bool), - ('extra.', 'client_local_files', convert_bool), - ('extra.', 'client_long_flag', convert_bool), - ('extra.', 'client_long_password', convert_bool), - ('extra.', 'client_multi_results', convert_bool), - ('extra.', 'client_multi_statements', convert_bool), - ('extra.', 'client_no_schema', convert_bool), - ('extra.', 'client_odbc', convert_bool), - ('extra.', 'client_plugin_auth', convert_bool), - ('extra.', 'client_plugin_auth_len_enc_client_data', convert_bool), - ('extra.', 'client_protocol_41', convert_bool), - ('extra.', 'client_ps_multi_results', convert_bool), - ('extra.', 'client_reserved', convert_bool), - ('extra.', 'client_secure_connection', convert_bool), - ('extra.', 'client_session_track', convert_bool), - ('extra.', 'client_ssl', convert_bool), - ('extra.', 'client_transactions', convert_bool), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'mysql_protocol_version', validate_to_none), - ('extra.', 'server_version', validate_to_none), - ('extra.', 'error_code', validate_to_none), - ('extra.', 'error_id', validate_to_none), - ('extra.', 'error_message', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'browser_trusted', convert_bool), - ('extra.', 'browser_error', validate_to_none), - ('extra.', 'raw_cert', validate_to_none), - ('extra.', 'raw_cert_chain', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'mysql', - 'classification.identifier': 'open-mysql', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-NATPMP -scan_nat_pmp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'opcode', validate_to_none), - ('extra.', 'uptime', convert_int), - ('extra.', 'external_ip', validate_to_none), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'open-natpmp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'natpmp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-netbios-report/ -scan_netbios = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.account', 'username'), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'mac_address', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'workgroup', validate_to_none), - ('extra.', 'machine_name', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'open-netbios-nameservice', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'netbios-nameservice', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/netcore-netis-router-vulnerability-scan-report/ -scan_netis_router = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'response', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'open-netis', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.transport': 'udp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/ntp-version-report/ -scan_ntp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'clk_wander', convert_float), - ('extra.', 'frequency', convert_float), - ('extra.', 'jitter', convert_float), - ('extra.', 'leap', convert_float), - ('extra.', 'offset', convert_float), - ('extra.', 'peer', convert_int), - ('extra.', 'poll', convert_int), - ('extra.', 'precision', convert_int), - ('extra.', 'rootdelay', convert_float), - ('extra.', 'rootdispersion', convert_float), - ('extra.', 'stratum', convert_int), - ('extra.', 'tc', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'version', validate_to_none), - ('extra.', 'clock', validate_to_none), - ('extra.', 'error', validate_to_none), - ('extra.', 'mintc', validate_to_none), - ('extra.', 'noise', validate_to_none), - ('extra.', 'phase', validate_to_none), - ('extra.', 'processor', validate_to_none), - ('extra.', 'refid', validate_to_none), - ('extra.', 'reftime', validate_to_none), - ('extra.', 'stability', validate_to_none), - ('extra.', 'state', validate_to_none), - ('extra.', 'system', validate_to_none), - ('extra.', 'tai', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'ntp-version', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ntp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/ntp-monitor-report/ -scan_ntpmonitor = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'packets', convert_int), - ('extra.', 'size', convert_int), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'ntp-monitor', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ntp', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-Portmapper -scan_portmapper = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'programs', validate_to_none), - ('extra.', 'mountd_port', validate_to_none), - ('extra.', 'exports', validate_to_none), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'portmapper', - 'classification.identifier': 'open-portmapper', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-postgresql-server-report/ -scan_postgres = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'startup_error_line', convert_int), - ('extra.', 'client_ssl', convert_bool), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'supported_protocols', validate_to_none), - ('extra.', 'protocol_error_code', validate_to_none), - ('extra.', 'protocol_error_file', validate_to_none), - ('extra.', 'protocol_error_line', validate_to_none), - ('extra.', 'protocol_error_message', validate_to_none), - ('extra.', 'protocol_error_routine', validate_to_none), - ('extra.', 'protocol_error_severity', validate_to_none), - ('extra.', 'protocol_error_severity_v', validate_to_none), - ('extra.', 'startup_error_code', validate_to_none), - ('extra.', 'startup_error_file', validate_to_none), - ('extra.', 'startup_error_message', validate_to_none), - ('extra.', 'startup_error_routine', validate_to_none), - ('extra.', 'startup_error_severity', validate_to_none), - ('extra.', 'startup_error_severity_v', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'browser_trusted', convert_bool), - ('extra.', 'browser_error', validate_to_none), - ('extra.', 'raw_cert', validate_to_none), - ('extra.', 'raw_cert_chain', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'postgres', - 'classification.identifier': 'open-postgres', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-QOTD -scan_qotd = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'quote', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'qotd', - 'classification.identifier': 'open-qotd', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-quic-report/ -scan_quic = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'version_field_1', validate_to_none), - ('extra.', 'version_field_2', validate_to_none), - ('extra.', 'version_field_3', validate_to_none), - ('extra.', 'version_field_4', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'classification.identifier': 'open-quic', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-radmin-report/ -scan_radmin = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-radmin', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-rdp-report/ -scan_rdp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'cve20190708_vulnerable', convert_bool), - ('extra.', 'bluekeep_vulnerable', convert_bool), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'rdp_protocol', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'tlsv13_support', validate_to_none), - ('extra.', 'tlsv13_cipher', validate_to_none), - ('extra.', 'jarm', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'rdp', - 'protocol.transport': 'tcp', - 'classification.identifier': 'open-rdp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-ms-rdpeudp/ -scan_rdpeudp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sessionid', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-msrdpeudp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-Redis -scan_redis = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'git_sha1', validate_to_none), - ('extra.', 'git_dirty_flag', validate_to_none), - ('extra.', 'build_id', validate_to_none), - ('extra.', 'mode', validate_to_none), - ('extra.os.name', 'os', validate_to_none), - ('extra.', 'architecture', validate_to_none), - ('extra.', 'multiplexing_api', validate_to_none), - ('extra.', 'gcc_version', validate_to_none), - ('extra.', 'process_id', validate_to_none), - ('extra.', 'run_id', validate_to_none), - ('extra.', 'uptime', convert_int), - ('extra.', 'connected_clients', validate_to_none), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'redis', - 'classification.identifier': 'open-redis', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-rsync-report/ -scan_rsync = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'module', validate_to_none), - ('extra.', 'motd', validate_to_none), - ('extra.', 'has_password', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-rsync', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'rsync', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-sip-report/ -scan_sip = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'sip', validate_to_none), - ('extra.', 'sip_code', validate_to_none), - ('extra.', 'sip_reason', validate_to_none), - ('user_agent', 'user_agent', validate_to_none), - ('extra.', 'sip_via', validate_to_none), - ('extra.', 'sip_to', validate_to_none), - ('extra.', 'sip_from', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'content_type', validate_to_none), - ('extra.sip_server', 'server', validate_to_none), - ('extra.sip_contact', 'contact', validate_to_none), - ('extra.sip_cseq', 'cseq', validate_to_none), - ('extra.sip_call_id', 'call_id', validate_to_none), - ('extra.sip_allow', 'allow', validate_to_none), - ('extra.', 'amplification', convert_float), - ('extra.', 'response_size', convert_int), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'sip', - 'classification.identifier': 'open-sip', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-slp-service-report/ -scan_slp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'function', validate_to_none), - ('extra.', 'function_text', validate_to_none), - ('extra.', 'flags', validate_to_none), - ('extra.', 'next_extension_offset', validate_to_none), - ('extra.', 'xid', validate_to_none), - ('extra.', 'language_tag_length', validate_to_none), - ('extra.', 'language_tag', validate_to_none), - ('extra.', 'error_code', validate_to_none), - ('extra.', 'error_code_text', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'raw_response', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'slp', - 'classification.identifier': 'open-slp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-smb-report/ -scan_smb = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'smb_implant', convert_bool), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'arch', validate_to_none), - ('extra.', 'key', validate_to_none), - ('extra.', 'smbv1_support', validate_to_none), - ('extra.', 'smb_major_number', validate_to_none), - ('extra.', 'smb_minor_number', validate_to_none), - ('extra.', 'smb_revision', validate_to_none), - ('extra.', 'smb_version_string', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'smb', - 'protocol.transport': 'tcp', - 'classification.identifier': 'open-smb', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-smtp-report/ -scan_smtp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'banner', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'smtp', - 'classification.identifier': 'open-smtp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/vulnerable-smtp-report/ -scan_smtp_vulnerable = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'banner', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'smtp', - 'classification.identifier': 'vulnerable-smtp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-snmp-report/ -scan_snmp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'sysdesc', validate_to_none), - ('extra.', 'sysname', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'version', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_sector', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('extra.', 'community', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'snmp', - 'classification.identifier': 'open-snmp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-socks4-5-proxy-report/ -scan_socks = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.application', 'tag'), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-socks', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-SSDP -scan_ssdp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'header', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'systime', validate_to_none), - ('extra.', 'cache_control', validate_to_none), - ('extra.', 'location', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'search_target', validate_to_none), - ('extra.', 'unique_service_name', validate_to_none), - ('extra.', 'host', validate_to_none), - ('extra.', 'nts', validate_to_none), - ('extra.', 'nt', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'server_port', validate_to_none), - ('extra.', 'instance', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'updated_at', validate_to_none), - ('extra.', 'resource_identifier', validate_to_none), - ('extra.', 'amplification', convert_float), - ('extra.', 'response_size', convert_int), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ssdp', - 'classification.identifier': 'open-ssdp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-ssh-report/ -scan_ssh = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'serverid_raw', validate_to_none), - ('extra.', 'serverid_version', validate_to_none), - ('extra.', 'serverid_software', validate_to_none), - ('extra.', 'serverid_comment', validate_to_none), - ('extra.', 'server_cookie', validate_to_none), - ('extra.', 'available_kex', validate_to_none), - ('extra.', 'available_ciphers', validate_to_none), - ('extra.', 'available_mac', validate_to_none), - ('extra.', 'available_compression', validate_to_none), - ('extra.', 'selected_kex', validate_to_none), - ('extra.', 'algorithm', validate_to_none), - ('extra.', 'selected_cipher', validate_to_none), - ('extra.', 'selected_mac', validate_to_none), - ('extra.', 'selected_compression', validate_to_none), - ('extra.', 'server_signature_value', validate_to_none), - ('extra.', 'server_signature_raw', validate_to_none), - ('extra.', 'server_host_key', validate_to_none), - ('extra.', 'server_host_key_sha256', validate_to_none), - ('extra.', 'rsa_prime', validate_to_none), - ('extra.', 'rsa_prime_length', validate_to_none), - ('extra.', 'rsa_generator', validate_to_none), - ('extra.', 'rsa_generator_length', validate_to_none), - ('extra.', 'rsa_public_key', validate_to_none), - ('extra.', 'rsa_public_key_length', validate_to_none), - ('extra.', 'rsa_exponent', validate_to_none), - ('extra.', 'rsa_modulus', validate_to_none), - ('extra.', 'rsa_length', validate_to_none), - ('extra.', 'dss_prime', validate_to_none), - ('extra.', 'dss_prime_length', validate_to_none), - ('extra.', 'dss_generator', validate_to_none), - ('extra.', 'dss_generator_length', validate_to_none), - ('extra.', 'dss_public_key', validate_to_none), - ('extra.', 'dss_public_key_length', validate_to_none), - ('extra.', 'dss_dsa_public_g', validate_to_none), - ('extra.', 'dss_dsa_public_p', validate_to_none), - ('extra.', 'dss_dsa_public_q', validate_to_none), - ('extra.', 'dss_dsa_public_y', validate_to_none), - ('extra.', 'ecdsa_curve25519', validate_to_none), - ('extra.', 'ecdsa_curve', validate_to_none), - ('extra.', 'ecdsa_public_key_length', validate_to_none), - ('extra.', 'ecdsa_public_key_b', validate_to_none), - ('extra.', 'ecdsa_public_key_gx', validate_to_none), - ('extra.', 'ecdsa_public_key_gy', validate_to_none), - ('extra.', 'ecdsa_public_key_n', validate_to_none), - ('extra.', 'ecdsa_public_key_p', validate_to_none), - ('extra.', 'ecdsa_public_key_x', validate_to_none), - ('extra.', 'ecdsa_public_key_y', validate_to_none), - ('extra.', 'ed25519_curve25519', validate_to_none), - ('extra.', 'ed25519_cert_public_key_nonce', validate_to_none), - ('extra.', 'ed25519_cert_public_key_bytes', validate_to_none), - ('extra.', 'ed25519_cert_public_key_raw', validate_to_none), - ('extra.', 'ed25519_cert_public_key_sha256', validate_to_none), - ('extra.', 'ed25519_cert_public_key_serial', validate_to_none), - ('extra.', 'ed25519_cert_public_key_type_id', validate_to_none), - ('extra.', 'ed25519_cert_public_key_type_name', validate_to_none), - ('extra.', 'ed25519_cert_public_key_keyid', validate_to_none), - ('extra.', 'ed25519_cert_public_key_principles', validate_to_none), - ('extra.', 'ed25519_cert_public_key_valid_after', validate_to_none), - ('extra.', 'ed25519_cert_public_key_valid_before', validate_to_none), - ('extra.', 'ed25519_cert_public_key_duration', validate_to_none), - ('extra.', 'ed25519_cert_public_key_sigkey_bytes', validate_to_none), - ('extra.', 'ed25519_cert_public_key_sigkey_raw', validate_to_none), - ('extra.', 'ed25519_cert_public_key_sigkey_sha256', validate_to_none), - ('extra.', 'ed25519_cert_public_key_sigkey_value', validate_to_none), - ('extra.', 'ed25519_cert_public_key_sig_raw', validate_to_none), - ('extra.', 'banner', validate_to_none), - ('extra.', 'userauth_methods', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'classification.identifier': 'open-ssh', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-ssl-report/ -scan_ssl = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'ssl_poodle', convert_bool), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'freak_vulnerable', convert_bool), - ('extra.', 'freak_cipher_suite', validate_to_none), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'http_response_type', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'http_connection', validate_to_none), - ('extra.', 'www_authenticate', validate_to_none), - ('extra.', 'set_cookie', validate_to_none), - ('extra.', 'server_type', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'browser_trusted', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'browser_error', validate_to_none), - ('extra.', 'tlsv13_support', validate_to_none), - ('extra.', 'tlsv13_cipher', validate_to_none), - ('extra.', 'jarm', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'https', - 'classification.identifier': 'open-ssl', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Ssl-Freak-Scan -scan_ssl_freak = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'freak_vulnerable', convert_bool), - ('extra.', 'freak_cipher_suite', validate_to_none), - ('extra.', 'sector', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'http_response_type', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'http_connection', validate_to_none), - ('extra.', 'www_authenticate', validate_to_none), - ('extra.', 'set_cookie', validate_to_none), - ('extra.', 'server_type', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'browser_trusted', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'browser_error', validate_to_none), - ('extra.', 'tlsv13_support', validate_to_none), - ('extra.', 'tlsv13_cipher', validate_to_none), - ('extra.', 'raw_cert', validate_to_none), - ('extra.', 'raw_cert_chain', validate_to_none), - ('extra.', 'jarm', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_sector', validate_to_none), - ('extra.', 'page_sha256fp', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'https', - 'classification.identifier': 'ssl-freak', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Ssl-Scan -scan_ssl_poodle = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'ssl_poodle', convert_bool), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'http_response_type', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'http_connection', validate_to_none), - ('extra.', 'www_authenticate', validate_to_none), - ('extra.', 'set_cookie', validate_to_none), - ('extra.', 'server_type', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'browser_trusted', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'browser_error', validate_to_none), - ('extra.', 'tlsv13_support', validate_to_none), - ('extra.', 'tlsv13_cipher', validate_to_none), - ('extra.', 'raw_cert', validate_to_none), - ('extra.', 'raw_cert_chain', validate_to_none), - ('extra.', 'jarm', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_sector', validate_to_none), - ('extra.', 'page_sha256fp', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'https', - 'classification.identifier': 'ssl-poodle', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-stun-service-report/ -scan_stun = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'mapped_port', convert_int), - ('extra.', 'xor_mapped_port', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'transaction_id', validate_to_none), - ('extra.', 'magic_cookie', validate_to_none), - ('extra.', 'message_length', convert_int), - ('extra.', 'message_type', validate_to_none), - ('extra.', 'mapped_family', validate_to_none), - ('extra.', 'mapped_address', validate_to_none), - ('extra.', 'xor_mapped_family', validate_to_none), - ('extra.', 'xor_mapped_address', validate_to_none), - ('extra.', 'software', validate_to_none), - ('extra.', 'fingerprint', validate_to_none), - ('extra.', 'amplification', convert_float), - ('extra.', 'response_size', convert_int), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'Session Traversal Utilities for NAT', - 'classification.identifier': 'open-stun', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/synful-scan-report/ -scan_synfulknock = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'ack_number', convert_int), - ('extra.', 'window_size', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'sequence_number', validate_to_none), - ('extra.', 'urgent_pointer', validate_to_none), - ('extra.', 'tcp_flags', validate_to_none), - ('extra.', 'raw_packet', validate_to_none), - ('extra.source.sector', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-synfulknock', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-telnet-report/ -scan_telnet = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'banner', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'telnet', - 'classification.identifier': 'open-telnet', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-TFTP -scan_tftp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'opcode', validate_to_none), - ('extra.', 'errorcode', validate_to_none), - ('extra.', 'error', validate_to_none), - ('extra.', 'errormessage', validate_to_none), - ('extra.', 'size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'tftp', - 'classification.identifier': 'open-tftp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-ubiquiti-report/ -scan_ubiquiti = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.mac_address', 'mac', validate_to_none), - ('extra.radio_name', 'radioname', validate_to_none), - ('extra.model', 'modelshort', validate_to_none), - ('extra.model_full', 'modelfull', validate_to_none), - ('extra.firmwarerev', 'firmware', validate_to_none), - ('extra.response_size', 'size', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'essid', validate_to_none), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-ubiquiti-discovery-service', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-vnc-report/ -scan_vnc = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.reverse_dns', 'hostname'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'product', validate_to_none), - ('extra.', 'banner', validate_to_none), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'vnc', - 'protocol.transport': 'tcp', - 'classification.identifier': 'open-vnc', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-ws-discovery-service-report/ -scan_ws_discovery = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ('extra.', 'error', validate_to_none), - ('extra.', 'raw_response', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ws-discovery', - 'classification.identifier': 'open-ws-discovery', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-xdmcp-service-report/ -scan_xdmcp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'opcode', validate_to_none), - ('extra.', 'reported_hostname', validate_to_none), - ('extra.', 'status', validate_to_none), - ('extra.', 'size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'xdmcp', - 'classification.identifier': 'open-xdmcp', - }, -} - -# http://www.shadowserver.org/wiki/pmwiki.php/Services/Spam-URL -spam_url = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ], - 'optional_fields': [ - ('source.url', 'url', convert_http_host_and_url, True), - ('source.fqdn', 'host', validate_fqdn), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'source', validate_to_none), - ('extra.', 'sender', validate_to_none), - ('extra.', 'subject', validate_to_none), - ('malware.hash.md5', 'md5', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'abusive-content', - 'classification.type': 'spam', - 'classification.identifier': 'spam-url', - }, -} - -special = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('source.reverse_dns', 'hostname'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('malware.name', 'tag'), - ('extra.', 'public_source', validate_to_none), - ('extra.', 'status', validate_to_none), - ('extra.', 'detail', validate_to_none), - ('extra.', 'method', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'special', - }, -} - -mapping = ( - # feed name, file name, function - ('Blocklist', 'blocklist', blocklist), - ('Compromised-Website', 'compromised_website', compromised_website), - ('Device-Identification IPv4', 'device_id', device_id), - ('Device-Identification IPv6', 'device_id6', device_id), - ('DDoS-Participant', 'event4_ddos_participant', event_ddos_participant), - ('Honeypot-Brute-Force-Events', 'event4_honeypot_brute_force', event_honeypot_brute_force), - ('Honeypot-Darknet', 'event4_honeypot_darknet', event_honeypot_darknet), - ('Honeypot-DDoS', 'event4_honeypot_ddos', event_honeypot_ddos), - ('Honeypot-Amplification-DDoS-Events', 'event4_honeypot_ddos_amp', event_honeypot_ddos_amp), - ('Honeypot-DDoS-Target', 'event4_honeypot_ddos_target', event_honeypot_ddos_target), - ('Honeypot-HTTP-Scan', 'event4_honeypot_http_scan', event_honeypot_http_scan), - ('Honeypot-ICS-Scanner', 'event4_honeypot_ics_scan', event_honeypot_ics_scan), - ('IP-Spoofer-Events', 'event4_ip_spoofer', event_ip_spoofer), - ('Microsoft-Sinkhole-Events IPv4', 'event4_microsoft_sinkhole', event_sinkhole), - ('Microsoft-Sinkhole-Events-HTTP IPv4', 'event4_microsoft_sinkhole_http', event_sinkhole_http), - ('Sinkhole-Events IPv4', 'event4_sinkhole', event_sinkhole), - ('Sinkhole-DNS', 'event4_sinkhole_dns', event_sinkhole_dns), - ('Sinkhole-Events-HTTP IPv4', 'event4_sinkhole_http', event_sinkhole_http), - ('Sinkhole-Events-HTTP-Referer IPv4', 'event4_sinkhole_http_referer', event_sinkhole_http_referer), - ('Sinkhole-Events IPv6', 'event6_sinkhole', event_sinkhole), - ('Sinkhole-Events-HTTP IPv6', 'event6_sinkhole_http', event_sinkhole_http), - ('Sinkhole-Events-HTTP-Referer IPv6', 'event6_sinkhole_http_referer', event_sinkhole_http_referer), - ('Malware-URL', 'malware_url', malware_url), - ('Phish-URL', 'phish_url', phish_url), - ('IPv6-Accessible-HTTP-Proxy', 'population6_http_proxy', population_http_proxy), - ('Accessible-HTTP-Proxy', 'population_http_proxy', population_http_proxy), - ('Sandbox-Connections', 'sandbox_conn', sandbox_conn), - ('Sandbox-DNS', 'sandbox_dns', sandbox_dns), - ('Sandbox-URL', 'sandbox_url', sandbox_url), - ('IPv6-Accessible-CWMP', 'scan6_cwmp', scan_cwmp), - ('IPv6-DNS-Open-Resolvers', 'scan6_dns', scan_dns), - ('IPv6-Vulnerable-Exchange', 'scan6_exchange', scan_exchange), - ('IPv6-Accessible-FTP', 'scan6_ftp', scan_ftp), - ('IPv6-Accessible-HTTP', 'scan6_http', scan_http), - ('IPv6-Open-HTTP-Proxy', 'scan6_http_proxy', scan_http_proxy), - ('IPv6-Vulnerable-HTTP', 'scan6_http_vulnerable', scan_http_vulnerable), - ('IPv6-Open-IPP', 'scan6_ipp', scan_ipp), - ('IPv6-Open-LDAP-TCP', 'scan6_ldap_tcp', scan_ldap_tcp), - ('IPv6-Open-MQTT', 'scan6_mqtt', scan_mqtt), - ('IPv6-Open-Anonymous-MQTT', 'scan6_mqtt_anon', scan_mqtt_anon), - ('IPv6-Accessible-MySQL', 'scan6_mysql', scan_mysql), - ('IPv6-NTP-Version', 'scan6_ntp', scan_ntp), - ('IPv6-NTP-Monitor', 'scan6_ntpmonitor', scan_ntpmonitor), - ('IPv6-Accessible-PostgreSQL', 'scan6_postgres', scan_postgres), - ('IPv6-Accessible-RDP', 'scan6_rdp', scan_rdp), - ('IPv6-Accessible-SLP', 'scan6_slp', scan_slp), - ('IPv6-Accessible-SMB', 'scan6_smb', scan_smb), - ('IPv6-Accessible-SMTP', 'scan6_smtp', scan_smtp), - ('IPv6-Vulnerable-SMTP', 'scan6_smtp_vulnerable', scan_smtp_vulnerable), - ('IPv6-Open-SNMP', 'scan6_snmp', scan_snmp), - ('IPv6-Accessible-SSH', 'scan6_ssh', scan_ssh), - ('IPv6-Accessible-SSL', 'scan6_ssl', scan_ssl), - ('SSL-FREAK-Vulnerable-Servers IPv6', 'scan6_ssl_freak', scan_ssl_freak), - ('SSL-POODLE-Vulnerable-Servers IPv6', 'scan6_ssl_poodle', scan_ssl_poodle), - ('IPv6-Accessible-Session-Traversal-Utilities-for-NAT', 'scan6_stun', scan_stun), - ('IPv6-Accessible-Telnet', 'scan6_telnet', scan_telnet), - ('IPv6-Accessible-VNC', 'scan6_vnc', scan_vnc), - ('Accessible-ADB', 'scan_adb', scan_adb), - ('Accessible-AFP', 'scan_afp', scan_afp), - ('Accessible-AMQP', 'scan_amqp', scan_amqp), - ('Accessible-ARD', 'scan_ard', scan_ard), - ('Open-Chargen', 'scan_chargen', scan_chargen), - ('Accessible-Cisco-Smart-Install', 'scan_cisco_smart_install', scan_cisco_smart_install), - ('Accessible-CoAP', 'scan_coap', scan_coap), - ('Accessible-CouchDB', 'scan_couchdb', scan_couchdb), - ('Accessible-CWMP', 'scan_cwmp', scan_cwmp), - ('Open-DB2-Discovery-Service', 'scan_db2', scan_db2), - ('Vulnerable-DDoS-Middlebox', 'scan_ddos_middlebox', scan_ddos_middlebox), - ('DNS-Open-Resolvers', 'scan_dns', scan_dns), - ('Accessible-Docker', 'scan_docker', scan_docker), - ('Accessible-DVR-DHCPDiscover', 'scan_dvr_dhcpdiscover', scan_dvr_dhcpdiscover), - ('Open-Elasticsearch', 'scan_elasticsearch', scan_elasticsearch), - ('Accessible-Erlang-Port-Mapper-Daemon', 'scan_epmd', scan_epmd), - ('Vulnerable-Exchange-Server', 'scan_exchange', scan_exchange), - ('Accessible-FTP', 'scan_ftp', scan_ftp), - ('Accessible-Hadoop', 'scan_hadoop', scan_hadoop), - ('Accessible-HTTP', 'scan_http', scan_http), - ('Open-HTTP-Proxy', 'scan_http_proxy', scan_http_proxy), - ('Vulnerable-HTTP', 'scan_http_vulnerable', scan_http_vulnerable), - ('Accessible-ICS', 'scan_ics', scan_ics), - ('Open-IPMI', 'scan_ipmi', scan_ipmi), - ('Open-IPP', 'scan_ipp', scan_ipp), - ('Vulnerable-ISAKMP', 'scan_isakmp', scan_isakmp), - ('Accessible-Kubernetes-API', 'scan_kubernetes', scan_kubernetes), - ('Open-LDAP-TCP', 'scan_ldap_tcp', scan_ldap_tcp), - ('Open-LDAP', 'scan_ldap_udp', scan_ldap_udp), - ('Open-mDNS', 'scan_mdns', scan_mdns), - ('Open-Memcached', 'scan_memcached', scan_memcached), - ('Open-MongoDB', 'scan_mongodb', scan_mongodb), - ('Open-MQTT', 'scan_mqtt', scan_mqtt), - ('Open-Anonymous-MQTT', 'scan_mqtt_anon', scan_mqtt_anon), - ('Open-MSSQL', 'scan_mssql', scan_mssql), - ('Accessible-MySQL', 'scan_mysql', scan_mysql), - ('Open-NATPMP', 'scan_nat_pmp', scan_nat_pmp), - ('Open-NetBIOS-Nameservice', 'scan_netbios', scan_netbios), - ('Open-Netis', 'scan_netis_router', scan_netis_router), - ('NTP-Version', 'scan_ntp', scan_ntp), - ('NTP-Monitor', 'scan_ntpmonitor', scan_ntpmonitor), - ('Open-Portmapper', 'scan_portmapper', scan_portmapper), - ('Accessible-PostgreSQL', 'scan_postgres', scan_postgres), - ('Open-QOTD', 'scan_qotd', scan_qotd), - ('Accessible-QUIC', 'scan_quic', scan_quic), - ('Accessible-Radmin', 'scan_radmin', scan_radmin), - ('Accessible-RDP', 'scan_rdp', scan_rdp), - ('Accessible-MS-RDPEUDP', 'scan_rdpeudp', scan_rdpeudp), - ('Open-Redis', 'scan_redis', scan_redis), - ('Accessible-Rsync', 'scan_rsync', scan_rsync), - ('Accessible-SIP', 'scan_sip', scan_sip), - ('Accessible-SLP', 'scan_slp', scan_slp), - ('Accessible-SMB', 'scan_smb', scan_smb), - ('Accessible-SMTP', 'scan_smtp', scan_smtp), - ('Vulnerable-SMTP', 'scan_smtp_vulnerable', scan_smtp_vulnerable), - ('Open-SNMP', 'scan_snmp', scan_snmp), - ('Accessible-SOCKS4/5-Proxy', 'scan_socks', scan_socks), - ('Open-SSDP', 'scan_ssdp', scan_ssdp), - ('Accessible-SSH', 'scan_ssh', scan_ssh), - ('Accessible-SSL', 'scan_ssl', scan_ssl), - ('SSL-FREAK-Vulnerable-Servers', 'scan_ssl_freak', scan_ssl_freak), - ('SSL-POODLE-Vulnerable-Servers IPv4', 'scan_ssl_poodle', scan_ssl_poodle), - ('Accessible-Session-Traversal-Utilities-for-NAT', 'scan_stun', scan_stun), - ('SYNful-Knock', 'scan_synfulknock', scan_synfulknock), - ('Accessible-Telnet', 'scan_telnet', scan_telnet), - ('Open-TFTP', 'scan_tftp', scan_tftp), - ('Accessible-Ubiquiti-Discovery-Service', 'scan_ubiquiti', scan_ubiquiti), - ('Accessible-VNC', 'scan_vnc', scan_vnc), - ('Accessible-WS-Discovery-Service', 'scan_ws_discovery', scan_ws_discovery), - ('Open-XDMCP', 'scan_xdmcp', scan_xdmcp), - ('Spam-URL', 'spam_url', spam_url), - ('Special', 'special', special), - ('Accessible-RDPEUDP', 'scan_rdpeudp', scan_rdpeudp), - ('Sinkhole-Events', 'event4_sinkhole', event_sinkhole), - ('Sinkhole-Events-HTTP', 'event4_sinkhole_http', event_sinkhole_http), - ('Sinkhole-Events-HTTP-Referer', 'event4_sinkhole_http_referer', event_sinkhole_http_referer), -) -# END CONFGEN + try: + with open(tmp) as fh: + schema = json.load(fh) + except: + # leave tempfile behind for diagnosis + raise ValueError("Failed to validate %r" % tmp) -feedname_mapping = {feedname: function for feedname, filename, function in mapping} -filename_mapping = {filename: (feedname, function) for feedname, filename, function in mapping} + os.replace(tmp, __config.schema_file) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index 70ba3b4bb6..f14549141a 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -22,6 +22,7 @@ """ import copy import re +import os from intelmq.lib.bot import ParserBot from intelmq.lib.exceptions import InvalidKey, InvalidValue @@ -29,7 +30,13 @@ class ShadowserverParserBot(ParserBot): - """Parse all ShadowServer feeds""" + """ + Parse all ShadowServer feeds + + Parameters: + schema_file (str): Path to the report schema file + + """ recover_line = ParserBot.recover_line_csv_dict _csv_params = {'dialect': 'unix'} @@ -124,10 +131,17 @@ def parse_line(self, row, report): value = raw_value if conv_func is not None and raw_value is not None: - if len(item) == 4 and item[3]: - value = conv_func(raw_value, row) - else: - value = conv_func(raw_value) + try: + if len(item) == 4 and item[3]: + value = config.functions[conv_func](raw_value, row) + else: + value = config.functions[conv_func](raw_value) + except Exception: + """ fail early and often in this case. We want to be able to convert everything """ + self.logger.error('Could not convert shadowkey: %r in feed %r, ' + 'value: %r via conversion function %r.', + shadowkey, self.feedname, raw_value, conv_func) + raise if value is not None: event.add(intelmqkey, value) @@ -153,17 +167,17 @@ def parse_line(self, row, report): value = raw_value if conv_func is not None and raw_value is not None: - if len(item) == 4 and item[3]: - value = conv_func(raw_value, row) - else: - try: - value = conv_func(raw_value) - except Exception: - """ fail early and often in this case. We want to be able to convert everything """ - self.logger.error('Could not convert shadowkey: %r in feed %r, ' - 'value: %r via conversion function %r.', - shadowkey, self.feedname, raw_value, conv_func.__name__) - raise + try: + if len(item) == 4 and item[3]: + value = config.functions[conv_func](raw_value, row) + else: + value = config.functions[conv_func](raw_value) + except Exception: + """ fail early and often in this case. We want to be able to convert everything """ + self.logger.error('Could not convert shadowkey: %r in feed %r, ' + 'value: %r via conversion function %r.', + shadowkey, self.feedname, raw_value, conv_func) + raise if value is not None: if intelmqkey == 'extra.': diff --git a/intelmq/bots/parsers/shadowserver/schema.json.test b/intelmq/bots/parsers/shadowserver/schema.json.test new file mode 100644 index 0000000000..2cfb8bb1d3 --- /dev/null +++ b/intelmq/bots/parsers/shadowserver/schema.json.test @@ -0,0 +1,180 @@ +{ + "test_smb" : { + "constant_fields" : { + "classification.identifier" : "test-smb", + "classification.taxonomy" : "vulnerable", + "classification.type" : "vulnerable-system", + "protocol.application" : "smb", + "protocol.transport" : "tcp" + }, + "feed_name" : "Test-Accessible-SMB", + "file_name" : "test_smb", + "optional_fields" : [ + [ + "extra.", + "smb_implant", + "convert_bool" + ], + [ + "source.reverse_dns", + "hostname" + ], + [ + "extra.", + "tag" + ], + [ + "source.asn", + "asn", + "invalidate_zero" + ], + [ + "source.geolocation.cc", + "geo" + ], + [ + "source.geolocation.region", + "region" + ], + [ + "source.geolocation.city", + "city" + ], + [ + "extra.source.naics", + "naics", + "invalidate_zero" + ], + [ + "extra.source.sic", + "sic", + "invalidate_zero" + ], + [ + "extra.", + "arch", + "validate_to_none" + ], + [ + "extra.", + "key", + "validate_to_none" + ], + [ + "extra.", + "smbv1_support", + "validate_to_none" + ], + [ + "extra.", + "smb_major_number", + "validate_to_none" + ], + [ + "extra.", + "smb_minor_number", + "validate_to_none" + ], + [ + "extra.", + "smb_revision", + "validate_to_none" + ], + [ + "extra.", + "smb_version_string", + "validate_to_none" + ] + ], + "required_fields" : [ + [ + "time.source", + "timestamp", + "add_UTC_to_timestamp" + ], + [ + "source.ip", + "ip", + "validate_ip" + ], + [ + "source.port", + "port", + "convert_int" + ] + ] + }, + "test_telnet" : { + "constant_fields" : { + "classification.identifier" : "test-telnet", + "classification.taxonomy" : "vulnerable", + "classification.type" : "vulnerable-system", + "protocol.application" : "telnet" + }, + "feed_name" : "Test-Accessible-Telnet", + "file_name" : "test_telnet", + "optional_fields" : [ + [ + "protocol.transport", + "protocol" + ], + [ + "source.reverse_dns", + "hostname" + ], + [ + "extra.", + "tag", + "validate_to_none" + ], + [ + "source.asn", + "asn", + "invalidate_zero" + ], + [ + "source.geolocation.cc", + "geo" + ], + [ + "source.geolocation.region", + "region" + ], + [ + "source.geolocation.city", + "city" + ], + [ + "extra.", + "naics", + "invalidate_zero" + ], + [ "extra.", + "sic", + "invalidate_zero" + ], + [ + "extra.", + "banner", + "validate_to_none" + ] + ], + "required_fields" : [ + [ + "time.source", + "timestamp", + "add_UTC_to_timestamp" + ], + [ + "source.ip", + "ip", + "validate_ip" + ], + [ + "source.port", + "port", + "convert_int" + ] + ] + } +} diff --git a/intelmq/bots/parsers/shadowserver/update_schema.py b/intelmq/bots/parsers/shadowserver/update_schema.py new file mode 100644 index 0000000000..040f672593 --- /dev/null +++ b/intelmq/bots/parsers/shadowserver/update_schema.py @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: 2023 The Shadowserver Foundation +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +# -*- coding: utf-8 -*- + +import os +import intelmq.bots.parsers.shadowserver._config as config + +if __name__ == '__main__': # pragma: no cover + exec(open(os.path.join(os.path.dirname(__file__), '../../../version.py')).read()) # defines __version__ + config.update_schema(__version__) From b081509850f40f8626379ca2100f495dbfd52b96 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Wed, 12 Apr 2023 00:01:32 +0000 Subject: [PATCH 45/76] revised tests --- .../bots/parsers/shadowserver/test_broken.py | 12 +- .../bots/parsers/shadowserver/test_mapping.py | 8 +- .../parsers/shadowserver/test_parameters.py | 37 +++--- .../parsers/shadowserver/test_report_smb.py | 124 ++++++++++++++++++ .../shadowserver/test_report_switch.py | 16 +-- .../shadowserver/test_report_telnet.py | 87 ++++++++++++ .../shadowserver/testdata/test_smb.csv | 4 + .../testdata/test_smb.csv.license | 2 + .../shadowserver/testdata/test_telnet.csv | 3 + .../testdata/test_telnet.csv.license | 2 + 10 files changed, 260 insertions(+), 35 deletions(-) create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_report_smb.py create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv.license create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv.license diff --git a/intelmq/tests/bots/parsers/shadowserver/test_broken.py b/intelmq/tests/bots/parsers/shadowserver/test_broken.py index 472dd0b90c..2b803142eb 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_broken.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_broken.py @@ -13,12 +13,12 @@ REPORT1 = {"raw": utils.base64_encode('adasdasdasdasd\nadasdasdafgf'), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_http-test-test.csv", + "extra.file_name": "2019-01-01-test_smb-test-test.csv", } REPORT2 = {"raw": utils.base64_encode('timestamp,ip,port\n2018-08-01T00:00:00+00,127.0.0.1,80'), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ftp-test-test.csv", + "extra.file_name": "2019-01-01-test_telnet-test-test.csv", } REPORT3 = {"raw": utils.base64_encode('adasdasdasdasd\nadasdasdafgf'), "__type": "Report", @@ -48,10 +48,10 @@ def test_broken(self): """ self.input_message = REPORT1 self.run_bot(allowed_error_count=1) - self.assertLogMatches(pattern="Detected report's file name: 'scan_http'.", + self.assertLogMatches(pattern="Detected report's file name: 'test_smb'.", levelname="DEBUG") self.assertLogMatches(pattern="Failed to parse line.") - self.assertLogMatches(pattern="ValueError: Required column 'timestamp' not found in feed 'Accessible-HTTP'. Possible change in data format or misconfiguration.") + self.assertLogMatches(pattern="ValueError: Required column 'timestamp' not found in feed 'Test-Accessible-SMB'. Possible change in data format or misconfiguration.") self.assertLogMatches(pattern=r"Sent 0 events and found 1 problem\(s\)\.", levelname="INFO") @@ -61,9 +61,9 @@ def test_half_broken(self): """ self.input_message = REPORT2 self.run_bot(allowed_warning_count=63) - self.assertLogMatches(pattern="Detected report's file name: 'scan_ftp'.", + self.assertLogMatches(pattern="Detected report's file name: 'test_telnet'.", levelname="DEBUG") - self.assertLogMatches(pattern="Optional key 'jarm' not found in feed 'Accessible-FTP'.", + self.assertLogMatches(pattern="Optional key 'banner' not found in feed 'Test-Accessible-Telnet'.", levelname="WARNING") self.assertLogMatches(pattern=r"Sent 1 events and found 0 problem\(s\)\.", levelname="INFO") diff --git a/intelmq/tests/bots/parsers/shadowserver/test_mapping.py b/intelmq/tests/bots/parsers/shadowserver/test_mapping.py index f58aed66eb..6a2af94475 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_mapping.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_mapping.py @@ -11,22 +11,22 @@ with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_telnet.csv')) as handle: + 'testdata/test_telnet.csv')) as handle: TELNET_FILE = handle.read() EXAMPLE_TELNET = { "raw": utils.base64_encode(TELNET_FILE), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_telnet.csv", + "extra.file_name": "2019-01-01-test_telnet.csv", } with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_vnc.csv')) as handle: + 'testdata/test_smb.csv')) as handle: TELNET_FILE = handle.read() EXAMPLE_VNC = { "raw": utils.base64_encode(TELNET_FILE), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_vnc.csv", + "extra.file_name": "2019-01-01-test_smb.csv", } diff --git a/intelmq/tests/bots/parsers/shadowserver/test_parameters.py b/intelmq/tests/bots/parsers/shadowserver/test_parameters.py index a5ea81f199..677cd0319b 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_parameters.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_parameters.py @@ -12,38 +12,41 @@ from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_dns.csv')) as handle: + 'testdata/test_smb.csv')) as handle: EXAMPLE_FILE = handle.read() EXAMPLE_LINES = EXAMPLE_FILE.splitlines() EXAMPLE_REPORT = {"raw": utils.base64_encode(EXAMPLE_FILE), "__type": "Report", "time.observation": "2018-07-30T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_dns-test-test.csv", + "extra.file_name": "2019-01-01-test_smb-test-test.csv", 'feed.name': 'report feedname', } EVENTS = [{ '__type': 'Event', 'feed.name': 'report feedname', - "classification.identifier": "dns-open-resolver", + "classification.identifier": 'test-smb', "classification.taxonomy": "vulnerable", "classification.type": "vulnerable-system", - "extra.dns_version": "dnsmasq-2.66", - "extra.min_amplification": 4.619, - "extra.tag": "openresolver", - "protocol.application": "dns", - "protocol.transport": "udp", + "extra.smb_implant": False, + "extra.smb_major_number": '2', + "extra.smb_minor_number": '1', + "extra.smb_version_string": 'SMB 2.1', + "extra.smbv1_support": 'N', + "extra.tag": "smb", + "protocol.application": "smb", + "protocol.transport": "tcp", 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - "source.asn": 25255, - "source.geolocation.cc": "AT", - "source.geolocation.city": "VIENNA", - "source.geolocation.region": "WIEN", - "source.ip": "198.51.100.179", - "source.port": 53, - "source.reverse_dns": "198-51-100-189.example.net", + "source.asn": 64512, + "source.geolocation.cc": "ZZ", + "source.geolocation.city": "City", + "source.geolocation.region": "Region", + "source.ip": "192.168.0.1", + "source.port": 445, + "source.reverse_dns": "node01.example.com", "time.observation": "2018-07-30T00:00:00+00:00", - "time.source": "2018-04-14T00:14:34+00:00" + "time.source": "2010-02-10T00:00:00+00:00" }, ] @@ -70,7 +73,7 @@ def test_overwrite_feed_name(self): self.run_bot(prepare=False) for i, EVENT in enumerate(EVENTS): event = EVENT.copy() - event['feed.name'] = 'DNS-Open-Resolvers' + event['feed.name'] = 'Test-Accessible-SMB' self.assertMessageEqual(i, event) diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py b/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py new file mode 100644 index 0000000000..c7eefdf0a9 --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py @@ -0,0 +1,124 @@ +# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +# -*- coding: utf-8 -*- + +import os +import unittest + +import intelmq.lib.test as test +import intelmq.lib.utils as utils +from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot + +with open(os.path.join(os.path.dirname(__file__), + 'testdata/test_smb.csv')) as handle: + EXAMPLE_FILE = handle.read() +EXAMPLE_LINES = EXAMPLE_FILE.splitlines() + +EXAMPLE_REPORT = {'feed.name': 'Test-Accessible-SMB', + "raw": utils.base64_encode(EXAMPLE_FILE), + "__type": "Report", + "time.observation": "2015-01-01T00:00:00+00:00", + "extra.file_name": "2019-01-01-test_smb-test-geo.csv", + } +EVENTS = [ +{ + '__type' : 'Event', + 'classification.identifier' : 'test-smb', + 'classification.taxonomy' : 'vulnerable', + 'classification.type' : 'vulnerable-system', + 'extra.smb_implant' : False, + 'extra.smb_major_number' : '2', + 'extra.smb_minor_number' : '1', + 'extra.smb_version_string' : 'SMB 2.1', + 'extra.smbv1_support' : 'N', + 'extra.tag' : 'smb', + 'feed.name' : 'Test-Accessible-SMB', + 'protocol.application' : 'smb', + 'protocol.transport' : 'tcp', + 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), + 'source.asn' : 64512, + 'source.geolocation.cc' : 'ZZ', + 'source.geolocation.city' : 'City', + 'source.geolocation.region' : 'Region', + 'source.ip' : '192.168.0.1', + 'source.port' : 445, + 'source.reverse_dns' : 'node01.example.com', + 'time.observation' : '2015-01-01T00:00:00+00:00', + 'time.source' : '2010-02-10T00:00:00+00:00' +}, + +{ + '__type' : 'Event', + 'classification.identifier' : 'test-smb', + 'classification.taxonomy' : 'vulnerable', + 'classification.type' : 'vulnerable-system', + 'extra.smb_implant' : False, + 'extra.smb_major_number' : '2', + 'extra.smb_minor_number' : '1', + 'extra.smb_version_string' : 'SMB 2.1', + 'extra.smbv1_support' : 'N', + 'extra.tag' : 'smb', + 'feed.name' : 'Test-Accessible-SMB', + 'protocol.application' : 'smb', + 'protocol.transport' : 'tcp', + 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), + 'source.asn' : 64512, + 'source.geolocation.cc' : 'ZZ', + 'source.geolocation.city' : 'City', + 'source.geolocation.region' : 'Region', + 'source.ip' : '192.168.0.2', + 'source.port' : 445, + 'source.reverse_dns' : 'node02.example.com', + 'time.observation' : '2015-01-01T00:00:00+00:00', + 'time.source' : '2010-02-10T00:00:01+00:00' +}, + +{ + '__type' : 'Event', + 'classification.identifier' : 'test-smb', + 'classification.taxonomy' : 'vulnerable', + 'classification.type' : 'vulnerable-system', + 'extra.smb_implant' : False, + 'extra.smb_major_number' : '2', + 'extra.smb_minor_number' : '1', + 'extra.smb_version_string' : 'SMB 2.1', + 'extra.smbv1_support' : 'N', + 'extra.tag' : 'smb', + 'feed.name' : 'Test-Accessible-SMB', + 'protocol.application' : 'smb', + 'protocol.transport' : 'tcp', + 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), + 'source.asn' : 64512, + 'source.geolocation.cc' : 'ZZ', + 'source.geolocation.city' : 'City', + 'source.geolocation.region' : 'Region', + 'source.ip' : '192.168.0.3', + 'source.port' : 445, + 'source.reverse_dns' : 'node03.example.com', + 'time.observation' : '2015-01-01T00:00:00+00:00', + 'time.source' : '2010-02-10T00:00:02+00:00' +} + ] + + +class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): + """ + A TestCase for a ShadowserverParserBot. + """ + + @classmethod + def set_bot(cls): + cls.bot_reference = ShadowserverParserBot + cls.default_input_message = EXAMPLE_REPORT + + def test_event(self): + """ Test if correct Event has been produced. """ + self.run_bot() + for i, EVENT in enumerate(EVENTS): + self.assertMessageEqual(i, EVENT) + + +if __name__ == '__main__': # pragma: no cover + unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py b/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py index 0a34a69f0a..570d612fb4 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py @@ -12,24 +12,24 @@ from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ftp.csv')) as handle: + 'testdata/test_smb.csv')) as handle: EXAMPLE_LINES = handle.read().splitlines()[:2] -FIRST_REPORT = {'feed.name': 'Accessible FTP', +FIRST_REPORT = {'feed.name': 'Test-Accessible-SMB', "raw": utils.base64_encode('\n'.join(EXAMPLE_LINES)), "__type": "Report", "time.observation": "2019-03-25T00:00:00+00:00", - "extra.file_name": "2019-03-25-scan_ftp-test-test.csv", + "extra.file_name": "2019-03-25-test_smb-test-test.csv", } -with open(os.path.join(os.path.dirname(__file__), 'testdata/blocklist.csv')) as handle: +with open(os.path.join(os.path.dirname(__file__), 'testdata/test_telnet.csv')) as handle: EXAMPLE_LINES = handle.read().splitlines()[:2] SECOND_REPORT = { - 'feed.name': 'Blocklist', + 'feed.name': 'Test-Accessible-Telnet', "raw": utils.base64_encode('\n'.join(EXAMPLE_LINES)), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-blocklist-test-geo.csv", + "extra.file_name": "2019-01-01-test_telnet-test-geo.csv", } @@ -48,9 +48,9 @@ def test_event(self): """ Test if the parser correctly detects and handles different report types. """ self.input_message = [FIRST_REPORT, SECOND_REPORT] self.run_bot(iterations=2) - self.assertLogMatches("Detected report's file name: 'scan_ftp'", + self.assertLogMatches("Detected report's file name: 'test_smb'", levelname='DEBUG') - self.assertLogMatches("Detected report's file name: 'blocklist'", + self.assertLogMatches("Detected report's file name: 'test_telnet'", levelname='DEBUG') diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py b/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py new file mode 100644 index 0000000000..6d539ac4a7 --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py @@ -0,0 +1,87 @@ +# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +# -*- coding: utf-8 -*- + +import os +import unittest + +import intelmq.lib.test as test +import intelmq.lib.utils as utils +from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot + +with open(os.path.join(os.path.dirname(__file__), + 'testdata/test_telnet.csv')) as handle: + EXAMPLE_FILE = handle.read() +EXAMPLE_LINES = EXAMPLE_FILE.splitlines() + +EXAMPLE_REPORT = {'feed.name': 'Test-Accessible-Telnet', + "raw": utils.base64_encode(EXAMPLE_FILE), + "__type": "Report", + "time.observation": "2015-01-01T00:00:00+00:00", + "extra.file_name": "2019-01-01-test_telnet-test-geo.csv", + } +EVENTS = [{'__type': 'Event', + 'feed.name': 'Test-Accessible-Telnet', + "classification.identifier": "test-telnet", + "classification.taxonomy": "vulnerable", + "classification.type": "vulnerable-system", + "extra.banner": "|MikroTik v6.5|Login:", + "extra.tag": "telnet-alt", + "protocol.application": "telnet", + "protocol.transport": "tcp", + 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], + EXAMPLE_LINES[1]])), + "source.asn": 20255, + "source.geolocation.cc": "AA", + "source.geolocation.city": "LOCATION", + "source.geolocation.region": "LOCATION", + "source.ip": "198.123.245.145", + "source.port": 5678, + "source.reverse_dns": "example.local", + "time.observation": "2015-01-01T00:00:00+00:00", + "time.source": "2019-09-04T12:27:34+00:00" + }, + {'__type': 'Event', + 'feed.name': 'Test-Accessible-Telnet', + "classification.identifier": "test-telnet", + "classification.taxonomy": "vulnerable", + "classification.type": "vulnerable-system", + "extra.banner": "|MikroTik v6.45.3 (stable)|Login:", + "extra.tag": "telnet-alt", + "protocol.application": "telnet", + "protocol.transport": "tcp", + 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], + EXAMPLE_LINES[2]])), + "source.asn": 20255, + "source.geolocation.cc": "AA", + "source.geolocation.city": "LOCATION", + "source.geolocation.region": "LOCATION", + "source.ip": "198.123.245.145", + "source.port": 5678, + "source.reverse_dns": "example.local", + "time.observation": "2015-01-01T00:00:00+00:00", + "time.source": "2019-09-04T12:27:40+00:00" + }] + + +class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): + """ + A TestCase for a ShadowserverParserBot. + """ + + @classmethod + def set_bot(cls): + cls.bot_reference = ShadowserverParserBot + cls.default_input_message = EXAMPLE_REPORT + + def test_event(self): + """ Test if correct Event has been produced. """ + self.run_bot() + for i, EVENT in enumerate(EVENTS): + self.assertMessageEqual(i, EVENT) + + +if __name__ == '__main__': # pragma: no cover + unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv new file mode 100644 index 0000000000..fc7fe2fff6 --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv @@ -0,0 +1,4 @@ +"timestamp","ip","port","hostname","tag","asn","geo","region","city","naics","sic","smb_implant","arch","key","smbv1_support","smb_major_number","smb_minor_number","smb_revision","smb_version_string" +"2010-02-10 00:00:00",192.168.0.1,445,node01.example.com,smb,64512,ZZ,Region,City,0,0,N,,,N,2,1,0,"SMB 2.1" +"2010-02-10 00:00:01",192.168.0.2,445,node02.example.com,smb,64512,ZZ,Region,City,0,0,N,,,N,2,1,0,"SMB 2.1" +"2010-02-10 00:00:02",192.168.0.3,445,node03.example.com,smb,64512,ZZ,Region,City,0,0,N,,,N,2,1,0,"SMB 2.1" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv.license new file mode 100644 index 0000000000..f512a890e4 --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv.license @@ -0,0 +1,2 @@ +SPDX-FileCopyrightText: 2022 The Shadowserver Foundation +SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv new file mode 100644 index 0000000000..3309e9a3d8 --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv @@ -0,0 +1,3 @@ +"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","banner" +"2019-09-04 12:27:34","198.123.245.145","tcp",5678,"example.local","telnet-alt",20255,"AA","LOCATION","LOCATION",0,0,"|MikroTik v6.5|Login:" +"2019-09-04 12:27:40","198.123.245.145","tcp",5678,"example.local","telnet-alt",20255,"AA","LOCATION","LOCATION",0,0,"|MikroTik v6.45.3 (stable)|Login:" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv.license new file mode 100644 index 0000000000..942a94035d --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv.license @@ -0,0 +1,2 @@ +SPDX-FileCopyrightText: 2019 Guillermo Rodriguez +SPDX-License-Identifier: AGPL-3.0-or-later From c6108d6b219a1588cd45ba6bf7ec89dd6a5c5a42 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Mon, 8 May 2023 15:05:12 +0000 Subject: [PATCH 46/76] Updated to reset report type on reload #2361 --- intelmq/bots/parsers/shadowserver/README.md | 2 +- intelmq/bots/parsers/shadowserver/_config.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md index 297930861b..bb6216b9a7 100644 --- a/intelmq/bots/parsers/shadowserver/README.md +++ b/intelmq/bots/parsers/shadowserver/README.md @@ -11,6 +11,6 @@ The report configuration is now stored in a _schema.json_ file downloaded from h For environments that have internet connectivity the `update_schema.py` script should be setup as a cron job to obtain the latest revision. -For air-gapped systems automation will be required to download and copy the _schema.json_ file into this directory +For air-gapped systems automation will be required to download and copy the _schema.json_ file into this directory. The parser will automatically reload the configuration when the file changes. diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index a7b80b7a6c..29382d2782 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -272,15 +272,14 @@ def scan_exchange_identifier(field): def reload (): """ reload the configuration if it has changed """ mtime = 0.0 + schema_file = __config.schema_file - if (os.path.isfile(__config.schema_file)): + if os.path.isfile(__config.schema_file): mtime = os.path.getmtime(__config.schema_file) if __config.schema_mtime == mtime: return - schema_file = __config.schema_file else: # load a test schema if one has not been downloaded yet - schema_file = __config.schema_file schema_file += '.test' __config.feedname_mapping.clear() From 308ec67e4227634cece6276ac47e53adff7aed63 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 23 May 2023 01:12:47 +0000 Subject: [PATCH 47/76] Added schema download on startup and additional logging --- intelmq/bots/parsers/shadowserver/_config.py | 33 +++++++++++++------ intelmq/bots/parsers/shadowserver/parser.py | 1 + .../parsers/shadowserver/update_schema.py | 3 +- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index 29382d2782..f766be3221 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -106,6 +106,8 @@ def get_feed_by_filename(given_filename: str) -> Optional[Tuple[str, Dict[str, A reload() return __config.filename_mapping.get(given_filename, None) +def set_logger(logger): + __config.logger = logger def add_UTC_to_timestamp(value: str) -> str: return value + ' UTC' @@ -272,29 +274,38 @@ def scan_exchange_identifier(field): def reload (): """ reload the configuration if it has changed """ mtime = 0.0 - schema_file = __config.schema_file if os.path.isfile(__config.schema_file): mtime = os.path.getmtime(__config.schema_file) if __config.schema_mtime == mtime: return else: - # load a test schema if one has not been downloaded yet - schema_file += '.test' + __config.logger.info("The schema file does not exist.") + + if __config.schema_mtime == 0.0 and mtime == 0.0 and not os.environ.get('INTELMQ_SKIP_INTERNET'): + __config.logger.info("Attempting to download schema.") + update_schema() __config.feedname_mapping.clear() __config.filename_mapping.clear() - with open(schema_file) as fh: - schema = json.load(fh) - for report in schema: - __config.feedname_mapping[schema[report]['feed_name']] = (schema[report]['feed_name'], schema[report]) - __config.filename_mapping[schema[report]['file_name']] = (schema[report]['feed_name'], schema[report]) + for schema_file in [ __config.schema_file, ".".join([__config.schema_file, 'test']) ]: + if os.path.isfile(schema_file): + with open(schema_file) as fh: + schema = json.load(fh) + for report in schema: + if report == "_meta": + __config.logger.info("Loading schema %s." % schema[report]['date_created']) + for msg in schema[report]['change_log']: + __config.logger.info(msg) + else: + __config.feedname_mapping[schema[report]['feed_name']] = (schema[report]['feed_name'], schema[report]) + __config.filename_mapping[schema[report]['file_name']] = (schema[report]['feed_name'], schema[report]) __config.schema_mtime = mtime -def update_schema (version): +def update_schema (): """ download the latest configuration """ (th, tmp) = tempfile.mkstemp() - url = 'https://interchange.shadowserver.org/intelmq/'+version + url = 'https://interchange.shadowserver.org/intelmq/v1' try: urllib.request.urlretrieve(url, tmp) except: @@ -307,4 +318,6 @@ def update_schema (version): # leave tempfile behind for diagnosis raise ValueError("Failed to validate %r" % tmp) + if os.path.exists(__config.schema_file): + os.replace(__config.schema_file, ".".join([__config.schema_file, 'bak'])) os.replace(tmp, __config.schema_file) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index f14549141a..2f20262bfa 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -47,6 +47,7 @@ class ShadowserverParserBot(ParserBot): overwrite = False def init(self): + config.set_logger(self.logger) if self.feedname is not None: self._sparser_config = config.get_feed_by_feedname(self.feedname) if self._sparser_config: diff --git a/intelmq/bots/parsers/shadowserver/update_schema.py b/intelmq/bots/parsers/shadowserver/update_schema.py index 040f672593..a7975147ed 100644 --- a/intelmq/bots/parsers/shadowserver/update_schema.py +++ b/intelmq/bots/parsers/shadowserver/update_schema.py @@ -8,5 +8,4 @@ import intelmq.bots.parsers.shadowserver._config as config if __name__ == '__main__': # pragma: no cover - exec(open(os.path.join(os.path.dirname(__file__), '../../../version.py')).read()) # defines __version__ - config.update_schema(__version__) + config.update_schema() From 9ecf36616a2cec50de0eb5a562403ea2e212de8c Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 23 May 2023 23:32:53 +0000 Subject: [PATCH 48/76] Added version support to the schema update function. --- intelmq/bots/parsers/shadowserver/README.md | 6 ++-- intelmq/bots/parsers/shadowserver/_config.py | 32 +++++++++++++++++--- intelmq/bots/parsers/shadowserver/parser.py | 4 +++ 3 files changed, 34 insertions(+), 8 deletions(-) diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md index bb6216b9a7..c757020e94 100644 --- a/intelmq/bots/parsers/shadowserver/README.md +++ b/intelmq/bots/parsers/shadowserver/README.md @@ -7,10 +7,10 @@ This module is maintained by [The Shadowserver Foundation](https://www.shadowser Please contact intelmq@shadowserver.org with any issues or concerns. -The report configuration is now stored in a _schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/_version_. +The report configuration is now stored in a _schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/v1. -For environments that have internet connectivity the `update_schema.py` script should be setup as a cron job to obtain the latest revision. +For environments that have internet connectivity the `update_schema.py` script can be called from a cron job to obtain the latest revision. -For air-gapped systems automation will be required to download and copy the _schema.json_ file into this directory. +For air-gapped systems automation will be required to download and copy the _schema.json_ file into this directory. The parser will automatically reload the configuration when the file changes. diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index f766be3221..bb67db525a 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -97,6 +97,11 @@ class __Container: __config.feedname_mapping = {} __config.filename_mapping = {} +def set_logger(logger): + """ Sets the logger instance. """ + __config.logger = logger + + def get_feed_by_feedname(given_feedname: str) -> Optional[Dict[str, Any]]: reload() return __config.feedname_mapping.get(given_feedname, None) @@ -106,8 +111,6 @@ def get_feed_by_filename(given_filename: str) -> Optional[Tuple[str, Dict[str, A reload() return __config.filename_mapping.get(given_filename, None) -def set_logger(logger): - __config.logger = logger def add_UTC_to_timestamp(value: str) -> str: return value + ' UTC' @@ -304,20 +307,39 @@ def reload (): def update_schema (): """ download the latest configuration """ - (th, tmp) = tempfile.mkstemp() + if os.environ.get('INTELMQ_SKIP_INTERNET'): + return None + + (th, tmp) = tempfile.mkstemp(dir=os.path.dirname(__file__)) url = 'https://interchange.shadowserver.org/intelmq/v1' try: urllib.request.urlretrieve(url, tmp) except: raise ValueError("Failed to download %r" % url) + new_version = '' + old_version = '' + try: with open(tmp) as fh: schema = json.load(fh) + new_version = schema['_meta']['date_created'] except: # leave tempfile behind for diagnosis raise ValueError("Failed to validate %r" % tmp) if os.path.exists(__config.schema_file): - os.replace(__config.schema_file, ".".join([__config.schema_file, 'bak'])) - os.replace(tmp, __config.schema_file) + old_version = '' + try: + with open(__config.schema_file) as fh: + schema = json.load(fh) + old_version = schema['_meta']['date_created'] + if new_version != old_version: + os.replace(__config.schema_file, ".".join([__config.schema_file, 'bak'])) + except: + pass + + if new_version != old_version: + os.replace(tmp, __config.schema_file) + else: + os.unlink(tmp) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index 2f20262bfa..71489e2ec1 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -48,6 +48,10 @@ class ShadowserverParserBot(ParserBot): def init(self): config.set_logger(self.logger) + try: + config.update_schema() + except Exception as e: + logger.warning(f"Schema update failed: {e}.") if self.feedname is not None: self._sparser_config = config.get_feed_by_feedname(self.feedname) if self._sparser_config: From 9c4a1a4dfd47a3be3bc5dd1cc77228464a426450 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Sun, 28 May 2023 23:13:54 +0000 Subject: [PATCH 49/76] Documentation and style updates. --- CHANGELOG.md | 6 + .../shadowserver/collector_reports_api.py | 2 +- intelmq/bots/parsers/shadowserver/README.md | 39 ++++- intelmq/bots/parsers/shadowserver/_config.py | 52 +++--- intelmq/bots/parsers/shadowserver/parser.py | 2 +- .../bots/parsers/shadowserver/test_broken.py | 4 +- .../bots/parsers/shadowserver/test_mapping.py | 1 - .../parsers/shadowserver/test_report_smb.py | 151 +++++++++--------- .../shadowserver/test_report_switch.py | 10 +- .../shadowserver/test_report_telnet.py | 4 +- 10 files changed, 154 insertions(+), 117 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 72d9501937..ea36275bc0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -118,15 +118,21 @@ CHANGELOG ### Bots #### Collectors +<<<<<<< HEAD - `intelmq.bots.collector.rt`: - restrict `python-rt` to be below version 3.0 due to introduced breaking changes, - added support for `Subject NOT LIKE` queries, - added support for multiple values in ticket subject queries. - `intelmq.bots.collectors.rsync`: Support for optional private key, relative time parsing for the source path, extra rsync parameters and strict host key checking (PR#2241 by Mateo Durante). +======= +- `intelmq.bots.collectors.shadowserver.collector_reports_api`: + - The 'json' option is no longer supported as the 'csv' option provides better performance. Please see intelmq/bots/parsers/shadowserver/README.md for a sample configuration. (PR#2372) +>>>>>>> Documentation and style updates. #### Parsers - `intelmq.bots.parsers.shadowserver._config`: - Reset detected `feedname` at shutdown to re-detect the feedname on reloads (PR#2361 by @elsif2, fixes #2360). + - Switch to dynamic configuration to decouple report schema changes from IntelMQ releases. Please see intelmq/bots/parsers/shadowserver/README.md for details. (PR#2372) - `intelmq.bots.parsers.shadowserver._config`: - Added 'IPv6-Vulnerable-Exchange' alias and 'Accessible-WS-Discovery-Service' report. (PR#2338) - Removed unused `p0f_genre` and `p0f_detail` from the 'DNS-Open-Resolvers' report. (PR#2338) diff --git a/intelmq/bots/collectors/shadowserver/collector_reports_api.py b/intelmq/bots/collectors/shadowserver/collector_reports_api.py index dc8bd6b420..5e7117bd23 100644 --- a/intelmq/bots/collectors/shadowserver/collector_reports_api.py +++ b/intelmq/bots/collectors/shadowserver/collector_reports_api.py @@ -34,7 +34,7 @@ class ShadowServerAPICollectorBot(CollectorBot, HttpMixin, CacheMixin): A list of strings or a comma-separated list of the mailing lists you want to process. types (list): A list of strings or a string of comma-separated values with the names of reporttypes you want to process. If you leave this empty, all the available reports will be downloaded and processed (i.e. 'scan', 'drones', 'intel', 'sandbox_connection', 'sinkhole_combined'). - file_format (str): File format to download ('csv'). The 'json' option is not longer supported. + file_format (str): File format to download ('csv'). The 'json' option is no longer supported. """ country = None diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md index c757020e94..ae38dcb8cc 100644 --- a/intelmq/bots/parsers/shadowserver/README.md +++ b/intelmq/bots/parsers/shadowserver/README.md @@ -7,10 +7,45 @@ This module is maintained by [The Shadowserver Foundation](https://www.shadowser Please contact intelmq@shadowserver.org with any issues or concerns. -The report configuration is now stored in a _schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/v1. +The report configuration is now stored in a _schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/v1/schema. -For environments that have internet connectivity the `update_schema.py` script can be called from a cron job to obtain the latest revision. +For environments that have internet connectivity the `update_schema.py` script should be called from a cron job to obtain the latest revision. +The parser will attempt to download a schema update on startup unless INTELMQ_SKIP_INTERNET is set. For air-gapped systems automation will be required to download and copy the _schema.json_ file into this directory. The parser will automatically reload the configuration when the file changes. + + +## Sample configuration: + +``` +shadowserver-collector: + description: Our bot responsible for getting reports from Shadowserver + enabled: true + group: Collector + module: intelmq.bots.collectors.shadowserver.collector_reports_api + name: Shadowserver_Collector + parameters: + destination_queues: + _default: [shadowserver-parser-queue] + file_format: csv + api_key: "$API_KEY_received_from_the_shadowserver_foundation" + secret: "$SECRET_received_from_the_shadowserver_foundation" + run_mode: continuous +``` + +``` +shadowserver-parser: + bot_id: shadowserver-parser + name: Shadowserver Parser + enabled: true + group: Parser + groupname: parsers + module: intelmq.bots.parsers.shadowserver.parser + parameters: + destination_queues: + _default: [file-output-queue] + run_mode: continuous +``` + diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index bb67db525a..5219fdb344 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -88,15 +88,18 @@ import intelmq.lib.harmonization as harmonization + class __Container: pass + __config = __Container() __config.schema_file = os.path.join(os.path.dirname(__file__), 'schema.json') __config.schema_mtime = 0.0 __config.feedname_mapping = {} __config.filename_mapping = {} + def set_logger(logger): """ Sets the logger instance. """ __config.logger = logger @@ -254,27 +257,28 @@ def scan_exchange_identifier(field): return 'exchange-server-webshell' return 'vulnerable-exchange-server' + functions = { - 'add_UTC_to_timestamp': add_UTC_to_timestamp, - 'convert_bool': convert_bool, - 'validate_to_none': validate_to_none, - 'convert_int': convert_int, - 'convert_float': convert_float, - 'convert_http_host_and_url': convert_http_host_and_url, - 'invalidate_zero': invalidate_zero, - 'validate_ip': validate_ip, - 'validate_network': validate_network, - 'validate_fqdn': validate_fqdn, - 'convert_date': convert_date, - 'convert_date_utc': convert_date_utc, - 'force_base64': force_base64, - 'scan_exchange_taxonomy': scan_exchange_taxonomy, - 'scan_exchange_type': scan_exchange_type, - 'scan_exchange_identifier': scan_exchange_identifier, - } - - -def reload (): + 'add_UTC_to_timestamp': add_UTC_to_timestamp, + 'convert_bool': convert_bool, + 'validate_to_none': validate_to_none, + 'convert_int': convert_int, + 'convert_float': convert_float, + 'convert_http_host_and_url': convert_http_host_and_url, + 'invalidate_zero': invalidate_zero, + 'validate_ip': validate_ip, + 'validate_network': validate_network, + 'validate_fqdn': validate_fqdn, + 'convert_date': convert_date, + 'convert_date_utc': convert_date_utc, + 'force_base64': force_base64, + 'scan_exchange_taxonomy': scan_exchange_taxonomy, + 'scan_exchange_type': scan_exchange_type, + 'scan_exchange_identifier': scan_exchange_identifier, +} + + +def reload(): """ reload the configuration if it has changed """ mtime = 0.0 @@ -291,7 +295,7 @@ def reload (): __config.feedname_mapping.clear() __config.filename_mapping.clear() - for schema_file in [ __config.schema_file, ".".join([__config.schema_file, 'test']) ]: + for schema_file in [__config.schema_file, ".".join([__config.schema_file, 'test'])]: if os.path.isfile(schema_file): with open(schema_file) as fh: schema = json.load(fh) @@ -305,13 +309,14 @@ def reload (): __config.filename_mapping[schema[report]['file_name']] = (schema[report]['feed_name'], schema[report]) __config.schema_mtime = mtime -def update_schema (): + +def update_schema(): """ download the latest configuration """ if os.environ.get('INTELMQ_SKIP_INTERNET'): return None (th, tmp) = tempfile.mkstemp(dir=os.path.dirname(__file__)) - url = 'https://interchange.shadowserver.org/intelmq/v1' + url = 'https://interchange.shadowserver.org/intelmq/v1/schema' try: urllib.request.urlretrieve(url, tmp) except: @@ -329,7 +334,6 @@ def update_schema (): raise ValueError("Failed to validate %r" % tmp) if os.path.exists(__config.schema_file): - old_version = '' try: with open(__config.schema_file) as fh: schema = json.load(fh) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index 71489e2ec1..668a815341 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -51,7 +51,7 @@ def init(self): try: config.update_schema() except Exception as e: - logger.warning(f"Schema update failed: {e}.") + self.logger.warning("Schema update failed: %s." % e) if self.feedname is not None: self._sparser_config = config.get_feed_by_feedname(self.feedname) if self._sparser_config: diff --git a/intelmq/tests/bots/parsers/shadowserver/test_broken.py b/intelmq/tests/bots/parsers/shadowserver/test_broken.py index 2b803142eb..3797f03cd5 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_broken.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_broken.py @@ -24,12 +24,12 @@ "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", "extra.file_name": "2019-01-01-some_string-test-test.csv", -} + } REPORT4 = {"raw": utils.base64_encode('adasdasdasdasd\nadasdasdafgf'), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", "extra.file_name": "2020.wrong-filename.csv", -} + } class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): diff --git a/intelmq/tests/bots/parsers/shadowserver/test_mapping.py b/intelmq/tests/bots/parsers/shadowserver/test_mapping.py index 6a2af94475..d296dfdc26 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_mapping.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_mapping.py @@ -52,6 +52,5 @@ def test_changed_feed(self): self.run_bot(iterations=2) - if __name__ == '__main__': # pragma: no cover unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py b/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py index c7eefdf0a9..93d592d15c 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py @@ -22,85 +22,78 @@ "time.observation": "2015-01-01T00:00:00+00:00", "extra.file_name": "2019-01-01-test_smb-test-geo.csv", } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'test-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Test-Accessible-SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 445, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'test-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Test-Accessible-SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 445, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'test-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Test-Accessible-SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 445, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} - ] +EVENTS = [{'__type': 'Event', + 'classification.identifier': 'test-smb', + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + 'extra.smb_implant': False, + 'extra.smb_major_number': '2', + 'extra.smb_minor_number': '1', + 'extra.smb_version_string': 'SMB 2.1', + 'extra.smbv1_support': 'N', + 'extra.tag': 'smb', + 'feed.name': 'Test-Accessible-SMB', + 'protocol.application': 'smb', + 'protocol.transport': 'tcp', + 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), + 'source.asn': 64512, + 'source.geolocation.cc': 'ZZ', + 'source.geolocation.city': 'City', + 'source.geolocation.region': 'Region', + 'source.ip': '192.168.0.1', + 'source.port': 445, + 'source.reverse_dns': 'node01.example.com', + 'time.observation': '2015-01-01T00:00:00+00:00', + 'time.source': '2010-02-10T00:00:00+00:00' + }, + {'__type': 'Event', + 'classification.identifier': 'test-smb', + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + 'extra.smb_implant': False, + 'extra.smb_major_number': '2', + 'extra.smb_minor_number': '1', + 'extra.smb_version_string': 'SMB 2.1', + 'extra.smbv1_support': 'N', + 'extra.tag': 'smb', + 'feed.name': 'Test-Accessible-SMB', + 'protocol.application': 'smb', + 'protocol.transport': 'tcp', + 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), + 'source.asn': 64512, + 'source.geolocation.cc': 'ZZ', + 'source.geolocation.city': 'City', + 'source.geolocation.region': 'Region', + 'source.ip': '192.168.0.2', + 'source.port': 445, + 'source.reverse_dns': 'node02.example.com', + 'time.observation': '2015-01-01T00:00:00+00:00', + 'time.source': '2010-02-10T00:00:01+00:00' + }, + {'__type': 'Event', + 'classification.identifier': 'test-smb', + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + 'extra.smb_implant': False, + 'extra.smb_major_number': '2', + 'extra.smb_minor_number': '1', + 'extra.smb_version_string': 'SMB 2.1', + 'extra.smbv1_support': 'N', + 'extra.tag': 'smb', + 'feed.name': 'Test-Accessible-SMB', + 'protocol.application': 'smb', + 'protocol.transport': 'tcp', + 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), + 'source.asn': 64512, + 'source.geolocation.cc': 'ZZ', + 'source.geolocation.city': 'City', + 'source.geolocation.region': 'Region', + 'source.ip': '192.168.0.3', + 'source.port': 445, + 'source.reverse_dns': 'node03.example.com', + 'time.observation': '2015-01-01T00:00:00+00:00', + 'time.source': '2010-02-10T00:00:02+00:00' + }] class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py b/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py index 570d612fb4..a9be8a0a13 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py @@ -16,11 +16,11 @@ EXAMPLE_LINES = handle.read().splitlines()[:2] FIRST_REPORT = {'feed.name': 'Test-Accessible-SMB', - "raw": utils.base64_encode('\n'.join(EXAMPLE_LINES)), - "__type": "Report", - "time.observation": "2019-03-25T00:00:00+00:00", - "extra.file_name": "2019-03-25-test_smb-test-test.csv", - } + "raw": utils.base64_encode('\n'.join(EXAMPLE_LINES)), + "__type": "Report", + "time.observation": "2019-03-25T00:00:00+00:00", + "extra.file_name": "2019-03-25-test_smb-test-test.csv", + } with open(os.path.join(os.path.dirname(__file__), 'testdata/test_telnet.csv')) as handle: EXAMPLE_LINES = handle.read().splitlines()[:2] diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py b/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py index 6d539ac4a7..df9cf25dca 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py @@ -42,7 +42,7 @@ "source.reverse_dns": "example.local", "time.observation": "2015-01-01T00:00:00+00:00", "time.source": "2019-09-04T12:27:34+00:00" - }, + }, {'__type': 'Event', 'feed.name': 'Test-Accessible-Telnet', "classification.identifier": "test-telnet", @@ -63,7 +63,7 @@ "source.reverse_dns": "example.local", "time.observation": "2015-01-01T00:00:00+00:00", "time.source": "2019-09-04T12:27:40+00:00" - }] + }] class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): From e4f9ac4670a21a1bdc582d441e243d38f8f91331 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 30 May 2023 16:05:26 +0000 Subject: [PATCH 50/76] Added schema.json.test.license. --- intelmq/bots/parsers/shadowserver/schema.json.test.license | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 intelmq/bots/parsers/shadowserver/schema.json.test.license diff --git a/intelmq/bots/parsers/shadowserver/schema.json.test.license b/intelmq/bots/parsers/shadowserver/schema.json.test.license new file mode 100644 index 0000000000..9f58c89ef0 --- /dev/null +++ b/intelmq/bots/parsers/shadowserver/schema.json.test.license @@ -0,0 +1,2 @@ +SPDX-FileCopyrightText: 2023 The Shadowserver Foundation +SPDX-License-Identifier: AGPL-3.0-or-later From 460344fa4b26b7b69f7930a2e014183ae3da63e1 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Thu, 27 Jul 2023 20:19:25 +0000 Subject: [PATCH 51/76] Updates in response to feedback. --- .../shadowserver/collector_reports_api.py | 9 +++- intelmq/bots/parsers/shadowserver/README.md | 21 ++++++-- intelmq/bots/parsers/shadowserver/_config.py | 53 +++++++++++++------ intelmq/bots/parsers/shadowserver/parser.py | 45 +++++++++++++--- .../parsers/shadowserver/update_schema.py | 11 ---- .../shadowserver/test_download_schema.py | 28 ++++++++++ 6 files changed, 130 insertions(+), 37 deletions(-) delete mode 100644 intelmq/bots/parsers/shadowserver/update_schema.py create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_download_schema.py diff --git a/intelmq/bots/collectors/shadowserver/collector_reports_api.py b/intelmq/bots/collectors/shadowserver/collector_reports_api.py index 5e7117bd23..05bffa898e 100644 --- a/intelmq/bots/collectors/shadowserver/collector_reports_api.py +++ b/intelmq/bots/collectors/shadowserver/collector_reports_api.py @@ -68,12 +68,19 @@ def init(self): if self.file_format is not None: if not (self.file_format == 'csv'): - raise ValueError('Invalid file_format') + raise ValueError("Invalid file_format '%s'. Must be 'csv'." % self.file_format) else: self.file_format = 'csv' self.preamble = f'{{ "apikey": "{self.api_key}" ' + def check(parameters: dict): + for key in parameters: + if key == 'file_format' and parameters[key] != 'csv': + return [["error", "Invalid file_format '%s'. Must be 'csv'." % parameters[key]]] + elif key == 'country': + return [["warning", "Deprecated parameter 'country' found. Please use 'reports' instead. The backwards-compatibility will be removed in IntelMQ version 4.0.0."]] + def _headers(self, data): return {'HMAC2': hmac.new(self.secret.encode(), data.encode('utf-8'), digestmod=hashlib.sha256).hexdigest()} diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md index ae38dcb8cc..cd750d00b3 100644 --- a/intelmq/bots/parsers/shadowserver/README.md +++ b/intelmq/bots/parsers/shadowserver/README.md @@ -7,16 +7,28 @@ This module is maintained by [The Shadowserver Foundation](https://www.shadowser Please contact intelmq@shadowserver.org with any issues or concerns. -The report configuration is now stored in a _schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/v1/schema. +The report configuration is now stored in a _shadowserver-schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/v1/schema. -For environments that have internet connectivity the `update_schema.py` script should be called from a cron job to obtain the latest revision. -The parser will attempt to download a schema update on startup unless INTELMQ_SKIP_INTERNET is set. +The parser will attempt to download a schema update on startup when the *auto_update* option is enabled. -For air-gapped systems automation will be required to download and copy the _schema.json_ file into this directory. +Schema downloads can also be scheduled as a cron job: + +``` +02 01 * * * intelmq.bots.parsers.shadowserver.parser --update-schema +``` + +For air-gapped systems automation will be required to download and copy the file to VAR_STATE_PATH/shadowserver-schema.json. The parser will automatically reload the configuration when the file changes. +## Schema contract + +Once set the `classification.identifier`, `classification.taxonomy`, and `classification.type` fields will remain static. + +Once set report fields will not be deleted. + + ## Sample configuration: ``` @@ -46,6 +58,7 @@ shadowserver-parser: parameters: destination_queues: _default: [file-output-queue] + auto_update: true run_mode: continuous ``` diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index 5219fdb344..afe3a6b11f 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -82,11 +82,12 @@ import base64 import binascii import json -import urllib.request import tempfile from typing import Optional, Dict, Tuple, Any import intelmq.lib.harmonization as harmonization +from intelmq.lib.utils import create_request_session +from intelmq import VAR_STATE_PATH class __Container: @@ -94,8 +95,10 @@ class __Container: __config = __Container() -__config.schema_file = os.path.join(os.path.dirname(__file__), 'schema.json') +__config.schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json') +__config.schema_base = os.path.join(os.path.dirname(__file__), 'schema.json.test') __config.schema_mtime = 0.0 +__config.auto_update = False __config.feedname_mapping = {} __config.filename_mapping = {} @@ -105,13 +108,16 @@ def set_logger(logger): __config.logger = logger +def enable_auto_update(enable): + """ Enable automatic schema update. """ + __config.auto_update = enable + + def get_feed_by_feedname(given_feedname: str) -> Optional[Dict[str, Any]]: - reload() return __config.feedname_mapping.get(given_feedname, None) def get_feed_by_filename(given_filename: str) -> Optional[Tuple[str, Dict[str, Any]]]: - reload() return __config.filename_mapping.get(given_filename, None) @@ -289,19 +295,18 @@ def reload(): else: __config.logger.info("The schema file does not exist.") - if __config.schema_mtime == 0.0 and mtime == 0.0 and not os.environ.get('INTELMQ_SKIP_INTERNET'): - __config.logger.info("Attempting to download schema.") + if __config.schema_mtime == 0.0 and mtime == 0.0 and __config.auto_update: update_schema() __config.feedname_mapping.clear() __config.filename_mapping.clear() - for schema_file in [__config.schema_file, ".".join([__config.schema_file, 'test'])]: + for schema_file in [__config.schema_file, __config.schema_base]: if os.path.isfile(schema_file): with open(schema_file) as fh: schema = json.load(fh) for report in schema: if report == "_meta": - __config.logger.info("Loading schema %s." % schema[report]['date_created']) + __config.logger.info("Loading schema %r." % schema[report]['date_created']) for msg in schema[report]['change_log']: __config.logger.info(msg) else: @@ -313,37 +318,55 @@ def reload(): def update_schema(): """ download the latest configuration """ if os.environ.get('INTELMQ_SKIP_INTERNET'): - return None + return False - (th, tmp) = tempfile.mkstemp(dir=os.path.dirname(__file__)) + # download the schema to a temp file + (th, tmp) = tempfile.mkstemp(dir=VAR_STATE_PATH) url = 'https://interchange.shadowserver.org/intelmq/v1/schema' + __config.logger.info("Attempting to download schema from %r" % url) + __config.logger.debug("Using temp file %r for the download." % tmp) try: - urllib.request.urlretrieve(url, tmp) + with create_request_session() as session: + with session.get(url, stream=True) as r: + r.raise_for_status() + with open(tmp, 'wb') as f: + for chunk in r.iter_content(chunk_size=8192): + f.write(chunk) except: - raise ValueError("Failed to download %r" % url) + __config.logger.error("Failed to download %r" % url) + return False + __config.logger.info("Download successful.") new_version = '' old_version = '' try: + # validate the downloaded file with open(tmp) as fh: schema = json.load(fh) new_version = schema['_meta']['date_created'] except: # leave tempfile behind for diagnosis - raise ValueError("Failed to validate %r" % tmp) + __config.logger.error("Failed to validate %r" % tmp) + return False if os.path.exists(__config.schema_file): + # compare the new version against the old; rename the existing file try: with open(__config.schema_file) as fh: schema = json.load(fh) old_version = schema['_meta']['date_created'] if new_version != old_version: os.replace(__config.schema_file, ".".join([__config.schema_file, 'bak'])) - except: - pass + except Exception as e: + __config.logger.error("Unable to replace schema file: %s" % str(e)) + return False if new_version != old_version: os.replace(tmp, __config.schema_file) + __config.logger.info("New schema version is %r." % new_version) + return True else: os.unlink(tmp) + + return False diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index 668a815341..2e383a004e 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -26,6 +26,8 @@ from intelmq.lib.bot import ParserBot from intelmq.lib.exceptions import InvalidKey, InvalidValue +from intelmq.bin.intelmqctl import IntelMQController +import intelmq.lib.utils as utils import intelmq.bots.parsers.shadowserver._config as config @@ -34,8 +36,7 @@ class ShadowserverParserBot(ParserBot): Parse all ShadowServer feeds Parameters: - schema_file (str): Path to the report schema file - + auto_update (boolean): Enable automatic schema download """ recover_line = ParserBot.recover_line_csv_dict @@ -45,13 +46,15 @@ class ShadowserverParserBot(ParserBot): feedname = None _mode = None overwrite = False + auto_update = False def init(self): config.set_logger(self.logger) - try: - config.update_schema() - except Exception as e: - self.logger.warning("Schema update failed: %s." % e) + if self.auto_update: + config.enable_auto_update(True) + self.logger.debug("Feature 'auto_update' is enabled.") + config.reload() + if self.feedname is not None: self._sparser_config = config.get_feed_by_feedname(self.feedname) if self._sparser_config: @@ -228,5 +231,35 @@ def parse_line(self, row, report): def shutdown(self): self.feedname = None + @classmethod + def _create_argparser(cls): + argparser = super()._create_argparser() + argparser.add_argument("--update-schema", action='store_true', help='downloads latest report schema') + argparser.add_argument("--verbose", action='store_true', help='be verbose') + return argparser + + @classmethod + def run(cls, parsed_args=None): + if not parsed_args: + parsed_args = cls._create_argparser().parse_args() + if parsed_args.update_schema: + logger = utils.log(__name__, log_path=None) + if parsed_args.verbose: + logger.setLevel('INFO') + else: + logger.setLevel('ERROR') + config.set_logger(logger) + if config.update_schema(): + runtime_conf = utils.get_bots_settings() + try: + ctl = IntelMQController() + for bot in runtime_conf: + if runtime_conf[bot]["module"] == __name__ and runtime_conf[bot]['parameters'].get('auto_update', True): + ctl.bot_reload(bot) + except Exception as e: + logger.error("Failed to signal bot: %r" % str(e)) + else: + super().run(parsed_args=parsed_args) + BOT = ShadowserverParserBot diff --git a/intelmq/bots/parsers/shadowserver/update_schema.py b/intelmq/bots/parsers/shadowserver/update_schema.py deleted file mode 100644 index a7975147ed..0000000000 --- a/intelmq/bots/parsers/shadowserver/update_schema.py +++ /dev/null @@ -1,11 +0,0 @@ -# SPDX-FileCopyrightText: 2023 The Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import intelmq.bots.parsers.shadowserver._config as config - -if __name__ == '__main__': # pragma: no cover - config.update_schema() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py new file mode 100644 index 0000000000..e685876826 --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: 2023 The Shadowserver Foundation +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +# -*- coding: utf-8 -*- +""" +Created on Thu Jul 27 19:44:44 2023 + +""" + +import unittest +import os +import logging +from intelmq import VAR_STATE_PATH +import intelmq.bots.parsers.shadowserver._config as config +import intelmq.lib.utils as utils +import intelmq.lib.test as test + +@test.skip_internet() +class TestShadowserverSchemaDownload(unittest.TestCase): + + def test_download(self): + schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json') + config.set_logger(utils.log('test-bot', log_path=None)) + if os.path.exists(schema_file): + os.unlink(schema_file) + self.assertEqual(True, config.update_schema()) + self.assertEqual(True, os.path.exists(schema_file)) From fec1fd2a22f1d26578ec5d9aeed752fe760c14ee Mon Sep 17 00:00:00 2001 From: elsif2 Date: Fri, 28 Jul 2023 14:17:41 +0000 Subject: [PATCH 52/76] Removed file_format parameter --- .../shadowserver/collector_reports_api.py | 20 ++++--------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/intelmq/bots/collectors/shadowserver/collector_reports_api.py b/intelmq/bots/collectors/shadowserver/collector_reports_api.py index 05bffa898e..66169d96f1 100644 --- a/intelmq/bots/collectors/shadowserver/collector_reports_api.py +++ b/intelmq/bots/collectors/shadowserver/collector_reports_api.py @@ -34,7 +34,6 @@ class ShadowServerAPICollectorBot(CollectorBot, HttpMixin, CacheMixin): A list of strings or a comma-separated list of the mailing lists you want to process. types (list): A list of strings or a string of comma-separated values with the names of reporttypes you want to process. If you leave this empty, all the available reports will be downloaded and processed (i.e. 'scan', 'drones', 'intel', 'sandbox_connection', 'sinkhole_combined'). - file_format (str): File format to download ('csv'). The 'json' option is no longer supported. """ country = None @@ -42,7 +41,6 @@ class ShadowServerAPICollectorBot(CollectorBot, HttpMixin, CacheMixin): secret = None types = None reports = None - file_format = None rate_limit: int = 86400 redis_cache_db: int = 12 redis_cache_host: str = "127.0.0.1" # TODO: type could be ipadress @@ -66,18 +64,12 @@ def init(self): self.logger.warn("Deprecated parameter 'country' found. Please use 'reports' instead. The backwards-compatibility will be removed in IntelMQ version 4.0.0.") self._report_list.append(self.country) - if self.file_format is not None: - if not (self.file_format == 'csv'): - raise ValueError("Invalid file_format '%s'. Must be 'csv'." % self.file_format) - else: - self.file_format = 'csv' - self.preamble = f'{{ "apikey": "{self.api_key}" ' def check(parameters: dict): for key in parameters: - if key == 'file_format' and parameters[key] != 'csv': - return [["error", "Invalid file_format '%s'. Must be 'csv'." % parameters[key]]] + if key == 'file_format': + return [["error", "The file_format parameter is no longer supported. All reports are CSV."]] elif key == 'country': return [["warning", "Deprecated parameter 'country' found. Please use 'reports' instead. The backwards-compatibility will be removed in IntelMQ version 4.0.0."]] @@ -129,11 +121,7 @@ def _report_download(self, reportid: str): data = self.preamble data += f',"id": "{reportid}"}}' self.logger.debug('Downloading report with data: %s.', data) - - if (self.file_format == 'json'): - response = self.http_session().post(APIROOT + 'reports/download', data=data, headers=self._headers(data)) - else: - response = self.http_session().get(DLROOT + reportid) + response = self.http_session().get(DLROOT + reportid) response.raise_for_status() return response.text @@ -150,7 +138,7 @@ def process(self): for item in reportslist: filename = item['file'] - filename_fixed = FILENAME_PATTERN.sub('.' + self.file_format, filename, count=1) + filename_fixed = FILENAME_PATTERN.sub('.csv', filename, count=1) if self.cache_get(filename): self.logger.debug('Processed file %r (fixed: %r) already.', filename, filename_fixed) continue From fe2a37c6c6526950e3602647303ec4a4efa79c86 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Thu, 24 Aug 2023 16:04:21 +0000 Subject: [PATCH 53/76] Minor changes based on feedback 2023-08-24 --- CHANGELOG.md | 2 - intelmq/bots/parsers/shadowserver/README.md | 2 + intelmq/bots/parsers/shadowserver/_config.py | 49 ++++++++++--------- intelmq/bots/parsers/shadowserver/parser.py | 6 ++- .../bots/parsers/shadowserver/test_broken.py | 5 ++ .../bots/parsers/shadowserver/test_mapping.py | 1 + .../parsers/shadowserver/test_parameters.py | 3 +- .../parsers/shadowserver/test_report_smb.py | 1 + .../shadowserver/test_report_switch.py | 1 + .../shadowserver/test_report_telnet.py | 1 + 10 files changed, 45 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea36275bc0..8cee9e520c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -124,10 +124,8 @@ CHANGELOG - added support for `Subject NOT LIKE` queries, - added support for multiple values in ticket subject queries. - `intelmq.bots.collectors.rsync`: Support for optional private key, relative time parsing for the source path, extra rsync parameters and strict host key checking (PR#2241 by Mateo Durante). -======= - `intelmq.bots.collectors.shadowserver.collector_reports_api`: - The 'json' option is no longer supported as the 'csv' option provides better performance. Please see intelmq/bots/parsers/shadowserver/README.md for a sample configuration. (PR#2372) ->>>>>>> Documentation and style updates. #### Parsers - `intelmq.bots.parsers.shadowserver._config`: diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md index cd750d00b3..4969acb6d0 100644 --- a/intelmq/bots/parsers/shadowserver/README.md +++ b/intelmq/bots/parsers/shadowserver/README.md @@ -28,6 +28,8 @@ Once set the `classification.identifier`, `classification.taxonomy`, and `classi Once set report fields will not be deleted. +The schema revision history is maintained at https://github.com/The-Shadowserver-Foundation/report_schema/. + ## Sample configuration: diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index afe3a6b11f..4bfadb9d98 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -95,8 +95,10 @@ class __Container: __config = __Container() +__config.schema_url = 'https://interchange.shadowserver.org/intelmq/v1/schema' __config.schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json') __config.schema_base = os.path.join(os.path.dirname(__file__), 'schema.json.test') +__config.schema_active = __config.schema_file __config.schema_mtime = 0.0 __config.auto_update = False __config.feedname_mapping = {} @@ -108,6 +110,13 @@ def set_logger(logger): __config.logger = logger +def enable_test_mode(enable): + """ Set which schema to load. """ + if enable: + __config.schema_active = __config.schema_base + else: + __config.schema_active = __config.schema_file + def enable_auto_update(enable): """ Enable automatic schema update. """ __config.auto_update = enable @@ -300,40 +309,36 @@ def reload(): __config.feedname_mapping.clear() __config.filename_mapping.clear() - for schema_file in [__config.schema_file, __config.schema_base]: - if os.path.isfile(schema_file): - with open(schema_file) as fh: - schema = json.load(fh) - for report in schema: - if report == "_meta": - __config.logger.info("Loading schema %r." % schema[report]['date_created']) - for msg in schema[report]['change_log']: - __config.logger.info(msg) - else: - __config.feedname_mapping[schema[report]['feed_name']] = (schema[report]['feed_name'], schema[report]) - __config.filename_mapping[schema[report]['file_name']] = (schema[report]['feed_name'], schema[report]) + if os.path.isfile(__config.schema_active): + with open(__config.schema_active) as fh: + schema = json.load(fh) + for report in schema: + if report == "_meta": + __config.logger.info("Loading schema %r.", schema[report]['date_created']) + for msg in schema[report]['change_log']: + __config.logger.info(msg) + else: + __config.feedname_mapping[schema[report]['feed_name']] = (schema[report]['feed_name'], schema[report]) + __config.filename_mapping[schema[report]['file_name']] = (schema[report]['feed_name'], schema[report]) __config.schema_mtime = mtime def update_schema(): """ download the latest configuration """ - if os.environ.get('INTELMQ_SKIP_INTERNET'): - return False # download the schema to a temp file (th, tmp) = tempfile.mkstemp(dir=VAR_STATE_PATH) - url = 'https://interchange.shadowserver.org/intelmq/v1/schema' - __config.logger.info("Attempting to download schema from %r" % url) - __config.logger.debug("Using temp file %r for the download." % tmp) + __config.logger.info("Attempting to download schema from %r", __config.schema_url) + __config.logger.debug("Using temp file %r for the download.", tmp) try: with create_request_session() as session: - with session.get(url, stream=True) as r: + with session.get(__config.schema_url, stream=True) as r: r.raise_for_status() with open(tmp, 'wb') as f: for chunk in r.iter_content(chunk_size=8192): f.write(chunk) except: - __config.logger.error("Failed to download %r" % url) + __config.logger.error("Failed to download %r", __config.schema_url) return False __config.logger.info("Download successful.") @@ -347,7 +352,7 @@ def update_schema(): new_version = schema['_meta']['date_created'] except: # leave tempfile behind for diagnosis - __config.logger.error("Failed to validate %r" % tmp) + __config.logger.error("Failed to validate %r", tmp) return False if os.path.exists(__config.schema_file): @@ -359,12 +364,12 @@ def update_schema(): if new_version != old_version: os.replace(__config.schema_file, ".".join([__config.schema_file, 'bak'])) except Exception as e: - __config.logger.error("Unable to replace schema file: %s" % str(e)) + __config.logger.error("Unable to replace schema file: %s", str(e)) return False if new_version != old_version: os.replace(tmp, __config.schema_file) - __config.logger.info("New schema version is %r." % new_version) + __config.logger.info("New schema version is %r.", new_version) return True else: os.unlink(tmp) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index 2e383a004e..fd9fa6b2cf 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -37,6 +37,7 @@ class ShadowserverParserBot(ParserBot): Parameters: auto_update (boolean): Enable automatic schema download + test_mode (boolean): Use test schema """ recover_line = ParserBot.recover_line_csv_dict @@ -47,9 +48,12 @@ class ShadowserverParserBot(ParserBot): _mode = None overwrite = False auto_update = False + test_mode = False def init(self): config.set_logger(self.logger) + if self.test_mode: + config.enable_test_mode(True) if self.auto_update: config.enable_auto_update(True) self.logger.debug("Feature 'auto_update' is enabled.") @@ -254,7 +258,7 @@ def run(cls, parsed_args=None): try: ctl = IntelMQController() for bot in runtime_conf: - if runtime_conf[bot]["module"] == __name__ and runtime_conf[bot]['parameters'].get('auto_update', True): + if runtime_conf[bot]["module"] == __name__: ctl.bot_reload(bot) except Exception as e: logger.error("Failed to signal bot: %r" % str(e)) diff --git a/intelmq/tests/bots/parsers/shadowserver/test_broken.py b/intelmq/tests/bots/parsers/shadowserver/test_broken.py index 3797f03cd5..54a85e7802 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_broken.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_broken.py @@ -46,6 +46,7 @@ def test_broken(self): """ Test a report which does not have valid fields """ + self.prepare_bot(parameters={'test_mode': True}) self.input_message = REPORT1 self.run_bot(allowed_error_count=1) self.assertLogMatches(pattern="Detected report's file name: 'test_smb'.", @@ -59,6 +60,7 @@ def test_half_broken(self): """ Test a report which does not have an optional field. """ + self.prepare_bot(parameters={'test_mode': True}) self.input_message = REPORT2 self.run_bot(allowed_warning_count=63) self.assertLogMatches(pattern="Detected report's file name: 'test_telnet'.", @@ -72,6 +74,7 @@ def test_no_config(self): """ Test a report which does not have a valid extra.file_name """ + self.prepare_bot(parameters={'test_mode': True}) self.input_message = REPORT3 self.run_bot(allowed_error_count=1) self.assertLogMatches(pattern="ValueError: Could not get a config for 'some_string', check the documentation.") @@ -80,6 +83,7 @@ def test_invalid_filename(self): """ Test a report which does not have a valid extra.file_name """ + self.prepare_bot(parameters={'test_mode': True}) self.input_message = REPORT4 self.run_bot(allowed_error_count=1) self.assertLogMatches(pattern="ValueError: Report's 'extra.file_name' '2020.wrong-filename.csv' is not valid.") @@ -89,6 +93,7 @@ def test_no_report_name(self): Test a report without file_name and no given feedname as parameter. Error message should be verbose. """ + self.prepare_bot(parameters={'test_mode': True}) self.run_bot(allowed_error_count=1) self.assertLogMatches(pattern="ValueError: No feedname given as parameter and the " "processed report has no 'extra.file_name'. " diff --git a/intelmq/tests/bots/parsers/shadowserver/test_mapping.py b/intelmq/tests/bots/parsers/shadowserver/test_mapping.py index d296dfdc26..b764de8274 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_mapping.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_mapping.py @@ -48,6 +48,7 @@ def test_changed_feed(self): Tests if the parser correctly re-detects the feed for the second received report #1493 """ + self.prepare_bot(parameters={'test_mode': True}) self.input_message = (EXAMPLE_TELNET, EXAMPLE_VNC) self.run_bot(iterations=2) diff --git a/intelmq/tests/bots/parsers/shadowserver/test_parameters.py b/intelmq/tests/bots/parsers/shadowserver/test_parameters.py index 677cd0319b..45a4a87354 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_parameters.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_parameters.py @@ -63,13 +63,14 @@ def set_bot(cls): def test_default(self): """ Test if feed name is not overwritten has been produced. """ + self.prepare_bot(parameters={'test_mode': True}) self.run_bot() for i, EVENT in enumerate(EVENTS): self.assertMessageEqual(i, EVENT) def test_overwrite_feed_name(self): """ Test if feed name is overwritten if asked to do so. """ - self.prepare_bot(parameters={'overwrite': True}) + self.prepare_bot(parameters={'test_mode': True, 'overwrite': True}) self.run_bot(prepare=False) for i, EVENT in enumerate(EVENTS): event = EVENT.copy() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py b/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py index 93d592d15c..aa6940061b 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py @@ -108,6 +108,7 @@ def set_bot(cls): def test_event(self): """ Test if correct Event has been produced. """ + self.prepare_bot(parameters={'test_mode': True}) self.run_bot() for i, EVENT in enumerate(EVENTS): self.assertMessageEqual(i, EVENT) diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py b/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py index a9be8a0a13..488f5a51a1 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py @@ -46,6 +46,7 @@ def set_bot(cls): def test_event(self): """ Test if the parser correctly detects and handles different report types. """ + self.prepare_bot(parameters={'test_mode': True}) self.input_message = [FIRST_REPORT, SECOND_REPORT] self.run_bot(iterations=2) self.assertLogMatches("Detected report's file name: 'test_smb'", diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py b/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py index df9cf25dca..b2499c589d 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py @@ -78,6 +78,7 @@ def set_bot(cls): def test_event(self): """ Test if correct Event has been produced. """ + self.prepare_bot(parameters={'test_mode': True}) self.run_bot() for i, EVENT in enumerate(EVENTS): self.assertMessageEqual(i, EVENT) From ec066ce06a06dd87912ad3b4337c84fe12821eba Mon Sep 17 00:00:00 2001 From: elsif2 Date: Thu, 24 Aug 2023 16:26:59 +0000 Subject: [PATCH 54/76] Added VAR_STATE_PATH check. --- intelmq/bots/parsers/shadowserver/_config.py | 1 + .../parsers/shadowserver/test_download_schema.py | 13 +++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index 4bfadb9d98..6ffffdae86 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -117,6 +117,7 @@ def enable_test_mode(enable): else: __config.schema_active = __config.schema_file + def enable_auto_update(enable): """ Enable automatic schema update. """ __config.auto_update = enable diff --git a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py index e685876826..f9512ca98c 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py @@ -20,9 +20,10 @@ class TestShadowserverSchemaDownload(unittest.TestCase): def test_download(self): - schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json') - config.set_logger(utils.log('test-bot', log_path=None)) - if os.path.exists(schema_file): - os.unlink(schema_file) - self.assertEqual(True, config.update_schema()) - self.assertEqual(True, os.path.exists(schema_file)) + if os.path.isdir(VAR_STATE_PATH): + schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json') + config.set_logger(utils.log('test-bot', log_path=None)) + if os.path.exists(schema_file): + os.unlink(schema_file) + self.assertEqual(True, config.update_schema()) + self.assertEqual(True, os.path.exists(schema_file)) From d1427f3365aa03c6df3c8befd0f270db3e94d96f Mon Sep 17 00:00:00 2001 From: elsif2 Date: Fri, 25 Aug 2023 15:37:51 +0000 Subject: [PATCH 55/76] Changes based on feedback 2023-08-25. --- CHANGELOG.md | 6 +- docs/user/bots.rst | 171 ++++++------------ intelmq/bots/parsers/shadowserver/README.md | 57 ------ intelmq/bots/parsers/shadowserver/_config.py | 10 +- .../shadowserver/test_download_schema.py | 8 +- 5 files changed, 72 insertions(+), 180 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cee9e520c..9fdc102258 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -118,20 +118,18 @@ CHANGELOG ### Bots #### Collectors -<<<<<<< HEAD - `intelmq.bots.collector.rt`: - restrict `python-rt` to be below version 3.0 due to introduced breaking changes, - added support for `Subject NOT LIKE` queries, - added support for multiple values in ticket subject queries. - `intelmq.bots.collectors.rsync`: Support for optional private key, relative time parsing for the source path, extra rsync parameters and strict host key checking (PR#2241 by Mateo Durante). - `intelmq.bots.collectors.shadowserver.collector_reports_api`: - - The 'json' option is no longer supported as the 'csv' option provides better performance. Please see intelmq/bots/parsers/shadowserver/README.md for a sample configuration. (PR#2372) + - The 'json' option is no longer supported as the 'csv' option provides better performance. #### Parsers - `intelmq.bots.parsers.shadowserver._config`: - Reset detected `feedname` at shutdown to re-detect the feedname on reloads (PR#2361 by @elsif2, fixes #2360). - - Switch to dynamic configuration to decouple report schema changes from IntelMQ releases. Please see intelmq/bots/parsers/shadowserver/README.md for details. (PR#2372) -- `intelmq.bots.parsers.shadowserver._config`: + - Switch to dynamic configuration to decouple report schema changes from IntelMQ releases. - Added 'IPv6-Vulnerable-Exchange' alias and 'Accessible-WS-Discovery-Service' report. (PR#2338) - Removed unused `p0f_genre` and `p0f_detail` from the 'DNS-Open-Resolvers' report. (PR#2338) - Added 'Accessible-SIP' report. (PR#2348) diff --git a/docs/user/bots.rst b/docs/user/bots.rst index 2fbe27df8e..a758ff8ad8 100644 --- a/docs/user/bots.rst +++ b/docs/user/bots.rst @@ -673,6 +673,23 @@ The resulting reports contain the following special field: * `extra.file_name`: The name of the downloaded file, with fixed filename extension. The API returns file names with the extension `.csv`, although the files are JSON, not CSV. Therefore, for clarity and better error detection in the parser, the file name in `extra.file_name` uses `.json` as extension. +**Sample configuration** + +.. code-block:: yaml + + shadowserver-collector: + description: Our bot responsible for getting reports from Shadowserver + enabled: true + group: Collector + module: intelmq.bots.collectors.shadowserver.collector_reports_api + name: Shadowserver_Collector + parameters: + destination_queues: + _default: [shadowserver-parser-queue] + file_format: csv + api_key: "$API_KEY_received_from_the_shadowserver_foundation" + secret: "$SECRET_received_from_the_shadowserver_foundation" + run_mode: continuous .. _intelmq.bots.collectors.shodan.collector_stream: @@ -1557,17 +1574,15 @@ This does not affect URLs which already include the scheme. .. _intelmq.bots.parsers.shadowserver.parser: -.. _intelmq.bots.parsers.shadowserver.parser_json: Shadowserver ^^^^^^^^^^^^ -There are two Shadowserver parsers, one for data in ``CSV`` format (``intelmq.bots.parsers.shadowserver.parser``) and one for data in ``JSON`` format (``intelmq.bots.parsers.shadowserver.parser_json``). -The latter was added in IntelMQ 2.3 and is meant to be used together with the Shadowserver API collector. +The Shadowserver parser operates on ``CSV`` formatted data. **Information** -* `name:` `intelmq.bots.parsers.shadowserver.parser` (for CSV data) or `intelmq.bots.parsers.shadowserver.parser_json` (for JSON data) +* `name:` `intelmq.bots.parsers.shadowserver.parser` * `public:` yes * `description:` Parses different reports from Shadowserver. @@ -1603,107 +1618,45 @@ A list of possible feeds can be found in the table below in the column "feed nam **Supported reports** -These are the supported feed name and their corresponding file name for automatic detection: - - ======================================= ========================= - feed name file name - ======================================= ========================= - Accessible-ADB `scan_adb` - Accessible-AFP `scan_afp` - Accessible-AMQP `scan_amqp` - Accessible-ARD `scan_ard` - Accessible-Cisco-Smart-Install `cisco_smart_install` - Accessible-CoAP `scan_coap` - Accessible-CWMP `scan_cwmp` - Accessible-MS-RDPEUDP `scan_msrdpeudp` - Accessible-FTP `scan_ftp` - Accessible-Hadoop `scan_hadoop` - Accessible-HTTP `scan_http` - Accessible-Radmin `scan_radmin` - Accessible-RDP `scan_rdp` - Accessible-Rsync `scan_rsync` - Accessible-SMB `scan_smb` - Accessible-Telnet `scan_telnet` - Accessible-Ubiquiti-Discovery-Service `scan_ubiquiti` - Accessible-VNC `scan_vnc` - Blacklisted-IP (deprecated) `blacklist` - Blocklist `blocklist` - Compromised-Website `compromised_website` - Device-Identification IPv4 / IPv6 `device_id`/`device_id6` - DNS-Open-Resolvers `scan_dns` - Honeypot-Amplification-DDoS-Events `event4_honeypot_ddos_amp` - Honeypot-Brute-Force-Events `event4_honeypot_brute_force` - Honeypot-Darknet `event4_honeypot_darknet` - Honeypot-HTTP-Scan `event4_honeypot_http_scan` - HTTP-Scanners `hp_http_scan` - ICS-Scanners `hp_ics_scan` - IP-Spoofer-Events `event4_ip_spoofer` - Microsoft-Sinkhole-Events IPv4 `event4_microsoft_sinkhole` - Microsoft-Sinkhole-Events-HTTP IPv4 `event4_microsoft_sinkhole_http` - NTP-Monitor `scan_ntpmonitor` - NTP-Version `scan_ntp` - Open-Chargen `scan_chargen` - Open-DB2-Discovery-Service `scan_db2` - Open-Elasticsearch `scan_elasticsearch` - Open-IPMI `scan_ipmi` - Open-IPP `scan_ipp` - Open-LDAP `scan_ldap` - Open-LDAP-TCP `scan_ldap_tcp` - Open-mDNS `scan_mdns` - Open-Memcached `scan_memcached` - Open-MongoDB `scan_mongodb` - Open-MQTT `scan_mqtt` - Open-MSSQL `scan_mssql` - Open-NATPMP `scan_nat_pmp` - Open-NetBIOS-Nameservice `scan_netbios` - Open-Netis `netis_router` - Open-Portmapper `scan_portmapper` - Open-QOTD `scan_qotd` - Open-Redis `scan_redis` - Open-SNMP `scan_snmp` - Open-SSDP `scan_ssdp` - Open-TFTP `scan_tftp` - Open-XDMCP `scan_xdmcp` - Outdated-DNSSEC-Key `outdated_dnssec_key` - Outdated-DNSSEC-Key-IPv6 `outdated_dnssec_key_v6` - Sandbox-URL `cwsandbox_url` - Sinkhole-DNS `sinkhole_dns` - Sinkhole-Events `event4_sinkhole`/`event6_sinkhole` - Sinkhole-Events IPv4 `event4_sinkhole` - Sinkhole-Events IPv6 `event6_sinkhole` - Sinkhole-HTTP-Events `event4_sinkhole_http`/`event6_sinkhole_http` - Sinkhole-HTTP-Events IPv4 `event4_sinkhole_http` - Sinkhole-HTTP-Events IPv6 `event6_sinkhole_http` - Sinkhole-Events-HTTP-Referer `event4_sinkhole_http_referer`/`event6_sinkhole_http_referer` - Sinkhole-Events-HTTP-Referer IPv4 `event4_sinkhole_http_referer` - Sinkhole-Events-HTTP-Referer IPv6 `event6_sinkhole_http_referer` - Spam-URL `spam_url` - SSL-FREAK-Vulnerable-Servers `scan_ssl_freak` - SSL-POODLE-Vulnerable-Servers `scan_ssl_poodle`/`scan6_ssl_poodle` - Vulnerable-Exchange-Server `*` `scan_exchange` - Vulnerable-ISAKMP `scan_isakmp` - Vulnerable-HTTP `scan_http` - Vulnerable-SMTP `scan_smtp_vulnerable` - ======================================= ========================= - -`*` This report can also contain data on active webshells (column `tag` is `exchange;webshell`), and are therefore not only vulnerable but also actively infected. - -In addition, the following legacy reports are supported: - - =========================== =================================================== ======================== - feed name successor feed name file name - =========================== =================================================== ======================== - Amplification-DDoS-Victim Honeypot-Amplification-DDoS-Events ``ddos_amplification`` - CAIDA-IP-Spoofer IP-Spoofer-Events ``caida_ip_spoofer`` - Darknet Honeypot-Darknet ``darknet`` - Drone Sinkhole-Events ``botnet_drone`` - Drone-Brute-Force Honeypot-Brute-Force-Events, Sinkhole-HTTP-Events ``drone_brute_force`` - Microsoft-Sinkhole Sinkhole-HTTP-Events ``microsoft_sinkhole`` - Sinkhole-HTTP-Drone Sinkhole-HTTP-Events ``sinkhole_http_drone`` - IPv6-Sinkhole-HTTP-Drone Sinkhole-HTTP-Events ``sinkhole6_http`` - =========================== =================================================== ======================== - -More information on these legacy reports can be found in `Changes in Sinkhole and Honeypot Report Types and Formats `_. +The report configuration is stored in a `shadowserver-schema.json` file downloaded from https://interchange.shadowserver.org/intelmq/v1/schema. + +The parser will attempt to download a schema update on startup when the *auto_update* option is enabled. + +Schema downloads can also be scheduled as a cron job: + +.. code-block:: bash + + 02 01 * * * intelmq.bots.parsers.shadowserver.parser --update-schema + + +For air-gapped systems automation will be required to download and copy the file to VAR_STATE_PATH/shadowserver-schema.json. + +The parser will automatically reload the configuration when the file changes. + +**Schema contract** + +Once set in the schema, the `classification.identifier`, `classification.taxonomy`, and `classification.type` fields will remain static for a specific report. + +Report fields will not be removed from a report. + +The schema revision history is maintained at https://github.com/The-Shadowserver-Foundation/report_schema/. + +**Sample configuration** + +.. code-block:: yaml + + shadowserver-parser: + bot_id: shadowserver-parser + name: Shadowserver Parser + enabled: true + group: Parser + groupname: parsers + module: intelmq.bots.parsers.shadowserver.parser + parameters: + destination_queues: + _default: [file-output-queue] + auto_update: true + run_mode: continuous **Development** @@ -1715,14 +1668,6 @@ The parser consists of two files: Both files are required for the parser to work properly. -**Add new Feedformats** - -Add a new feed format and conversions if required to the file -``_config.py``. Don't forget to update the ``mapping`` dict. -It is required to look up the correct configuration. - -Look at the documentation in the bot's ``_config.py`` file for more information. - .. _intelmq.bots.parsers.shodan.parser: diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md index 4969acb6d0..eb0ddfb4a7 100644 --- a/intelmq/bots/parsers/shadowserver/README.md +++ b/intelmq/bots/parsers/shadowserver/README.md @@ -7,60 +7,3 @@ This module is maintained by [The Shadowserver Foundation](https://www.shadowser Please contact intelmq@shadowserver.org with any issues or concerns. -The report configuration is now stored in a _shadowserver-schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/v1/schema. - -The parser will attempt to download a schema update on startup when the *auto_update* option is enabled. - -Schema downloads can also be scheduled as a cron job: - -``` -02 01 * * * intelmq.bots.parsers.shadowserver.parser --update-schema -``` - -For air-gapped systems automation will be required to download and copy the file to VAR_STATE_PATH/shadowserver-schema.json. - -The parser will automatically reload the configuration when the file changes. - - -## Schema contract - -Once set the `classification.identifier`, `classification.taxonomy`, and `classification.type` fields will remain static. - -Once set report fields will not be deleted. - -The schema revision history is maintained at https://github.com/The-Shadowserver-Foundation/report_schema/. - - -## Sample configuration: - -``` -shadowserver-collector: - description: Our bot responsible for getting reports from Shadowserver - enabled: true - group: Collector - module: intelmq.bots.collectors.shadowserver.collector_reports_api - name: Shadowserver_Collector - parameters: - destination_queues: - _default: [shadowserver-parser-queue] - file_format: csv - api_key: "$API_KEY_received_from_the_shadowserver_foundation" - secret: "$SECRET_received_from_the_shadowserver_foundation" - run_mode: continuous -``` - -``` -shadowserver-parser: - bot_id: shadowserver-parser - name: Shadowserver Parser - enabled: true - group: Parser - groupname: parsers - module: intelmq.bots.parsers.shadowserver.parser - parameters: - destination_queues: - _default: [file-output-queue] - auto_update: true - run_mode: continuous -``` - diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index 6ffffdae86..279093dfe3 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -95,6 +95,7 @@ class __Container: __config = __Container() +__config.var_state_path = VAR_STATE_PATH __config.schema_url = 'https://interchange.shadowserver.org/intelmq/v1/schema' __config.schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json') __config.schema_base = os.path.join(os.path.dirname(__file__), 'schema.json.test') @@ -328,7 +329,7 @@ def update_schema(): """ download the latest configuration """ # download the schema to a temp file - (th, tmp) = tempfile.mkstemp(dir=VAR_STATE_PATH) + (th, tmp) = tempfile.mkstemp(dir=__config.var_state_path) __config.logger.info("Attempting to download schema from %r", __config.schema_url) __config.logger.debug("Using temp file %r for the download.", tmp) try: @@ -376,3 +377,10 @@ def update_schema(): os.unlink(tmp) return False + + +def prepare_update_schema_test(path): + """ Reconfigure internal settings to perform a schema update test. """ + __config.var_state_path = path + __config.schema_file = os.path.join(path, 'shadowserver-schema.json') + return __config.schema_file diff --git a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py index f9512ca98c..5246e6bb67 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py @@ -10,8 +10,8 @@ import unittest import os +import tempfile import logging -from intelmq import VAR_STATE_PATH import intelmq.bots.parsers.shadowserver._config as config import intelmq.lib.utils as utils import intelmq.lib.test as test @@ -20,10 +20,8 @@ class TestShadowserverSchemaDownload(unittest.TestCase): def test_download(self): - if os.path.isdir(VAR_STATE_PATH): - schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json') + with tempfile.TemporaryDirectory() as tmp_dir: + schema_file = config.prepare_update_schema_test(tmp_dir) config.set_logger(utils.log('test-bot', log_path=None)) - if os.path.exists(schema_file): - os.unlink(schema_file) self.assertEqual(True, config.update_schema()) self.assertEqual(True, os.path.exists(schema_file)) From ae54e7cf783f770f0f7b25dd919f21d890964c3d Mon Sep 17 00:00:00 2001 From: elsif2 Date: Fri, 25 Aug 2023 15:51:38 +0000 Subject: [PATCH 56/76] Added INTELMQ_SKIP_INTERNET check --- .../bots/parsers/shadowserver/test_download_schema.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py index 5246e6bb67..203a3c0b12 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py @@ -20,8 +20,9 @@ class TestShadowserverSchemaDownload(unittest.TestCase): def test_download(self): - with tempfile.TemporaryDirectory() as tmp_dir: - schema_file = config.prepare_update_schema_test(tmp_dir) - config.set_logger(utils.log('test-bot', log_path=None)) - self.assertEqual(True, config.update_schema()) - self.assertEqual(True, os.path.exists(schema_file)) + if not os.environ.get('INTELMQ_SKIP_INTERNET'): + with tempfile.TemporaryDirectory() as tmp_dir: + schema_file = config.prepare_update_schema_test(tmp_dir) + config.set_logger(utils.log('test-bot', log_path=None)) + self.assertEqual(True, config.update_schema()) + self.assertEqual(True, os.path.exists(schema_file)) From e4e50637c0da38f32f6b8bbb95aa71875d0c4ad9 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Fri, 25 Aug 2023 16:11:21 +0000 Subject: [PATCH 57/76] Added debug logging for CI test. --- intelmq/bots/parsers/shadowserver/_config.py | 3 ++- .../tests/bots/parsers/shadowserver/test_download_schema.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index 279093dfe3..d573d12c61 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -339,8 +339,9 @@ def update_schema(): with open(tmp, 'wb') as f: for chunk in r.iter_content(chunk_size=8192): f.write(chunk) - except: + except Exception as e: __config.logger.error("Failed to download %r", __config.schema_url) + __config.logger.debug(str(e)) return False __config.logger.info("Download successful.") diff --git a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py index 203a3c0b12..abcd0ca2a4 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py @@ -23,6 +23,6 @@ def test_download(self): if not os.environ.get('INTELMQ_SKIP_INTERNET'): with tempfile.TemporaryDirectory() as tmp_dir: schema_file = config.prepare_update_schema_test(tmp_dir) - config.set_logger(utils.log('test-bot', log_path=None)) + config.set_logger(utils.log('test-bot', log_path=None, log_level=logging.DEBUG)) self.assertEqual(True, config.update_schema()) self.assertEqual(True, os.path.exists(schema_file)) From 128048272e04ab012ff80f67f588e326d10859c3 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Fri, 25 Aug 2023 18:47:54 +0000 Subject: [PATCH 58/76] Refactored test_download_schema to utilize mocking. --- intelmq/bots/parsers/shadowserver/parser.py | 6 ++++ .../shadowserver/test_download_schema.py | 30 ++++++++++++------- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index fd9fa6b2cf..48cbba901a 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -23,6 +23,7 @@ import copy import re import os +import tempfile from intelmq.lib.bot import ParserBot from intelmq.lib.exceptions import InvalidKey, InvalidValue @@ -265,5 +266,10 @@ def run(cls, parsed_args=None): else: super().run(parsed_args=parsed_args) + def test_update_schema(cls): + with tempfile.TemporaryDirectory() as tmp_dir: + schema_file = config.prepare_update_schema_test(tmp_dir) + return config.update_schema() + BOT = ShadowserverParserBot diff --git a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py index abcd0ca2a4..abf27a5bd4 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py @@ -8,21 +8,29 @@ """ -import unittest -import os -import tempfile import logging -import intelmq.bots.parsers.shadowserver._config as config +import unittest +import unittest.mock as mock +from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot import intelmq.lib.utils as utils import intelmq.lib.test as test + @test.skip_internet() -class TestShadowserverSchemaDownload(unittest.TestCase): +class TestShadowserverSchemaDownload(test.BotTestCase, unittest.TestCase): + + @classmethod + def set_bot(cls): + cls.bot_reference = ShadowserverParserBot + cls.sysconfig = {"logging_level": "DEBUG"} def test_download(self): - if not os.environ.get('INTELMQ_SKIP_INTERNET'): - with tempfile.TemporaryDirectory() as tmp_dir: - schema_file = config.prepare_update_schema_test(tmp_dir) - config.set_logger(utils.log('test-bot', log_path=None, log_level=logging.DEBUG)) - self.assertEqual(True, config.update_schema()) - self.assertEqual(True, os.path.exists(schema_file)) + self.prepare_bot(prepare_source_queue=False, parameters={'test_mode': True}) + result = False + with mock.patch('intelmq.lib.utils.load_configuration', new=self.mocked_config): + with mock.patch('intelmq.lib.utils.log', self.get_mocked_logger(self.logger)): + self.log_stream.truncate(0) + result = self.bot.test_update_schema() + self.bot.stop(exitcode=0) + print(self.log_stream.getvalue()) + self.assertEqual(True, result) From 2a60d2e10a581c9332151da909f3e716d5a825c3 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Mon, 28 Aug 2023 14:18:22 +0000 Subject: [PATCH 59/76] Added docstring for test_update_schema(). --- intelmq/bots/parsers/shadowserver/parser.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index 48cbba901a..4485a26020 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -267,6 +267,13 @@ def run(cls, parsed_args=None): super().run(parsed_args=parsed_args) def test_update_schema(cls): + """ + Test schema download to a temporary directory. + + This is necessary as the request session requires mocking in order to function. + + Returns True on success. + """ with tempfile.TemporaryDirectory() as tmp_dir: schema_file = config.prepare_update_schema_test(tmp_dir) return config.update_schema() From e401e2c1950851092c6febc37d8739eef402a3b4 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 29 Aug 2023 14:09:33 +0000 Subject: [PATCH 60/76] Removed logging output. --- intelmq/tests/bots/parsers/shadowserver/test_download_schema.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py index abf27a5bd4..84922bf176 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py @@ -29,8 +29,6 @@ def test_download(self): result = False with mock.patch('intelmq.lib.utils.load_configuration', new=self.mocked_config): with mock.patch('intelmq.lib.utils.log', self.get_mocked_logger(self.logger)): - self.log_stream.truncate(0) result = self.bot.test_update_schema() self.bot.stop(exitcode=0) - print(self.log_stream.getvalue()) self.assertEqual(True, result) From 66ae9f5a10898dda15f3008656b18d44551b5b91 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Thu, 31 Aug 2023 20:52:17 +0000 Subject: [PATCH 61/76] Removed the assertion regarding report fields. --- docs/user/bots.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/user/bots.rst b/docs/user/bots.rst index a758ff8ad8..ae17cbf556 100644 --- a/docs/user/bots.rst +++ b/docs/user/bots.rst @@ -1637,8 +1637,6 @@ The parser will automatically reload the configuration when the file changes. Once set in the schema, the `classification.identifier`, `classification.taxonomy`, and `classification.type` fields will remain static for a specific report. -Report fields will not be removed from a report. - The schema revision history is maintained at https://github.com/The-Shadowserver-Foundation/report_schema/. **Sample configuration** From e04dfeee04cfa9308602f870a48af0b933616527 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Mon, 16 Oct 2023 17:57:46 +0000 Subject: [PATCH 62/76] Skip and log a warning message for fields not in the IDF. --- intelmq/bots/parsers/shadowserver/parser.py | 5 ++- .../parsers/shadowserver/schema.json.test | 37 +++++++++++++++++++ .../bots/parsers/shadowserver/test_broken.py | 15 ++++++++ 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index 4485a26020..cfa343138d 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -157,7 +157,10 @@ def parse_line(self, row, report): raise if value is not None: - event.add(intelmqkey, value) + try: + event.add(intelmqkey, value) + except InvalidKey: + self.logger.warning('Key not found in IDF %r.', intelmqkey) fields.remove(shadowkey) # Now add optional fields. diff --git a/intelmq/bots/parsers/shadowserver/schema.json.test b/intelmq/bots/parsers/shadowserver/schema.json.test index 2cfb8bb1d3..932b8df03b 100644 --- a/intelmq/bots/parsers/shadowserver/schema.json.test +++ b/intelmq/bots/parsers/shadowserver/schema.json.test @@ -176,5 +176,42 @@ "convert_int" ] ] + }, + "test_afs" : { + "constant_fields" : { + "classification.identifier" : "test-afs", + "classification.taxonomy" : "vulnerable", + "classification.type" : "vulnerable-system", + "protocol.application" : "afs" + }, + "feed_name" : "Test-Accessible-AFS", + "file_name" : "test_afs", + "required_fields" : [ + [ + "time.source", + "timestamp", + "add_UTC_to_timestamp" + ], + [ + "source.ip", + "ip", + "validate_ip" + ], + [ + "source.port", + "port", + "convert_int" + ], + [ + "not_in_idf", + "severity" + ] + ], + "optional_fields" : [ + [ + "protocol.transport", + "protocol" + ] + ] } } diff --git a/intelmq/tests/bots/parsers/shadowserver/test_broken.py b/intelmq/tests/bots/parsers/shadowserver/test_broken.py index 54a85e7802..f1af08e586 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_broken.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_broken.py @@ -30,6 +30,11 @@ "time.observation": "2015-01-01T00:00:00+00:00", "extra.file_name": "2020.wrong-filename.csv", } +REPORT5 = {"raw": utils.base64_encode('timestamp,ip,protocol,port,severity\n2018-08-01T00:00:00+00,127.0.0.1,tcp,7000,critical'), + "__type": "Report", + "time.observation": "2023-10-16T00:00:00+00:00", + "extra.file_name": "2023-10-16-test_afs-test-test.csv", + } class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): @@ -100,6 +105,16 @@ def test_no_report_name(self): "Ensure that at least one is given. " "Also have a look at the documentation of the bot.") + def test_field_not_in_idf(self): + """ + Test a report that contains a field mapping not in the IDF. + Error message should be verbose. + """ + self.prepare_bot(parameters={'test_mode': True}) + self.input_message = REPORT5 + self.run_bot(allowed_error_count=0, allowed_warning_count=1) + self.assertLogMatches(pattern="Key not found in IDF", levelname="WARNING") + if __name__ == '__main__': # pragma: no cover unittest.main() From 6f2388349c7f217e6ee35ecf1d7266e726db783b Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 31 Oct 2023 16:19:09 +0000 Subject: [PATCH 63/76] Updated convert_http_host_and_url and added category_or_detail. --- intelmq/bots/parsers/shadowserver/_config.py | 22 ++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index d573d12c61..178bd0869c 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -170,12 +170,15 @@ def convert_http_host_and_url(value: str, row: Dict[str, str]) -> str: Sinkhole-HTTP-Drone: http_host, url With some reports, url/http_url holds only the path, with others the full HTTP request. """ + hostname = '' if "cc_dns" in row: hostname = row.get('cc_dns', '') - elif "http_host" in row: - hostname = row.get('http_host', '') - else: - hostname = '' + if not hostname and "http_host" in row: + hostname = row.get("http_host") + if not hostname and "hostname" in row: + hostname = row.get("hostname") + if not hostname and "ip" in row: + hostname = row.get("ip") if "url" in row: path = row.get('url', '') @@ -275,6 +278,16 @@ def scan_exchange_identifier(field): return 'vulnerable-exchange-server' +def category_or_detail(value: str, row: Dict[str, str]) -> str: + """ + Returns the category or detail field from the row. + """ + category = row.get('category', '') + if category != "": + return category + return row.get('detail', '') + + functions = { 'add_UTC_to_timestamp': add_UTC_to_timestamp, 'convert_bool': convert_bool, @@ -292,6 +305,7 @@ def scan_exchange_identifier(field): 'scan_exchange_taxonomy': scan_exchange_taxonomy, 'scan_exchange_type': scan_exchange_type, 'scan_exchange_identifier': scan_exchange_identifier, + 'category_or_detail': category_or_detail, } From a0b34cbc4c7121dd25e0ec5cbec495224e1dc690 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 31 Oct 2023 18:19:01 +0000 Subject: [PATCH 64/76] Avoid exception when a conversion function is not available in the current version. --- intelmq/bots/parsers/shadowserver/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index cfa343138d..ec1908269a 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -182,7 +182,7 @@ def parse_line(self, row, report): raw_value = row.get(shadowkey) value = raw_value - if conv_func is not None and raw_value is not None: + if conv_func is not None and raw_value is not None and conv_func in config.functions: try: if len(item) == 4 and item[3]: value = config.functions[conv_func](raw_value, row) From 61c756d77322cdeb18beeae5b78fcc1c5f4cb64d Mon Sep 17 00:00:00 2001 From: elsif2 Date: Sat, 4 Nov 2023 00:43:26 +0000 Subject: [PATCH 65/76] Added exception for missing schema and added intelmq user to the crontab suggestion. --- docs/user/bots.rst | 2 +- intelmq/bots/parsers/shadowserver/_config.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/user/bots.rst b/docs/user/bots.rst index ae17cbf556..6c1e657594 100644 --- a/docs/user/bots.rst +++ b/docs/user/bots.rst @@ -1622,7 +1622,7 @@ The report configuration is stored in a `shadowserver-schema.json` file download The parser will attempt to download a schema update on startup when the *auto_update* option is enabled. -Schema downloads can also be scheduled as a cron job: +Schema downloads can also be scheduled as a cron job for the `intelmq` user: .. code-block:: bash diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index 178bd0869c..c84bc7e85b 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -102,6 +102,7 @@ class __Container: __config.schema_active = __config.schema_file __config.schema_mtime = 0.0 __config.auto_update = False +__config.test_mode = False __config.feedname_mapping = {} __config.filename_mapping = {} @@ -113,6 +114,7 @@ def set_logger(logger): def enable_test_mode(enable): """ Set which schema to load. """ + __config.test_mode = enable if enable: __config.schema_active = __config.schema_base else: @@ -318,7 +320,8 @@ def reload(): if __config.schema_mtime == mtime: return else: - __config.logger.info("The schema file does not exist.") + if not __config.test_mode: + raise ValueError("The schema file does not exist.") if __config.schema_mtime == 0.0 and mtime == 0.0 and __config.auto_update: update_schema() From 307386d7d80dad00199c6ee3f89b6530bdbd6867 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Mon, 13 Nov 2023 17:49:05 +0000 Subject: [PATCH 66/76] Documentation update. --- docs/user/bots.md | 174 ++++++++++++++++------------------------------ 1 file changed, 61 insertions(+), 113 deletions(-) diff --git a/docs/user/bots.md b/docs/user/bots.md index 29977f56ed..e32d874c1e 100644 --- a/docs/user/bots.md +++ b/docs/user/bots.md @@ -929,11 +929,6 @@ The resulting reports contain the following special field: **Parameters (also expects [feed parameters](#feed-parameters) and [cache parameters](#cache-parameters)):** -**`country`** - -(required, string) **Deprecated:** The country you want to download the reports for. Will be removed in IntelMQ version -4.0.0, use *reports* instead. - **`apikey`** (required, string) Your Shadowserver API key. @@ -948,7 +943,27 @@ The resulting reports contain the following special field: **`types`** -(optional, string/array of strings) An array of strings (or a list of comma-separated values) with the names of report types you want to process. If you leave this empty, all the available reports will be downloaded and processed (i.e. 'scan', 'drones', 'intel', 'sandbox_connection', 'sinkhole_combined'). The possible report types are equivalent to the file names given in the section Supported Reports of the [Shadowserver parser](#intelmq.bots.parsers.shadowserver.parser_json). +(optional, string/array of strings) An array of strings (or a list of comma-separated values) with the names of report types you want to process. If you leave this empty, all the available reports will be downloaded and processed (i.e. 'scan', 'drones', 'intel', 'sandbox_connection', 'sinkhole_combined'). The possible report types are equivalent to the file names given in the section Supported Reports of the [Shadowserver parser](#intelmq.bots.parsers.shadowserver.parser). + +**Sample configuration** + +```yaml + + shadowserver-collector: + description: Our bot responsible for getting reports from Shadowserver + enabled: true + group: Collector + module: intelmq.bots.collectors.shadowserver.collector_reports_api + name: Shadowserver_Collector + parameters: + destination_queues: + _default: [shadowserver-parser-queue] + file_format: csv + api_key: "$API_KEY_received_from_the_shadowserver_foundation" + secret: "$SECRET_received_from_the_shadowserver_foundation" + run_mode: continuous + +``` --- @@ -2071,12 +2086,10 @@ No additional parameters. --- -### Shadowserver
+### Shadowserver
-Parses various reports from Shadowserver. +The Shadowserver parser operates on CSV formatted data. -There are two Shadowserver parsers, one for data in `CSV` format and one for data in `JSON` format. The latter was added -in IntelMQ 2.3 and is meant to be used together with the Shadowserver API collector. **How this bot works?** @@ -2105,8 +2118,7 @@ correct mapping of the columns: **Module:** -`intelmq.bots.parsers.shadowserver.parser` (for CSV data) -`intelmq.bots.parsers.shadowserver.parser_json` (for JSON data) +`intelmq.bots.parsers.shadowserver.parser` **Parameters:** @@ -2120,108 +2132,44 @@ correct mapping of the columns: **Supported reports:** -These are the supported report types and their corresponding file name for automatic detection: - -| Report Type (`feedname`) | File Name | -|-----------|-----------| -| Accessible-ADB | `scan_adb` | -| Accessible-AFP | `scan_afp` | -| Accessible-AMQP | `scan_amqp` | -| Accessible-ARD | `scan_ard` | -| Accessible-Cisco-Smart-Install | `cisco_smart_install` | -| Accessible-CoAP | `scan_coap` | -| Accessible-CWMP | `scan_cwmp` | -| Accessible-MS-RDPEUDP | `scan_msrdpeudp` | -| Accessible-FTP | `scan_ftp` | -| Accessible-Hadoop | `scan_hadoop` | -| Accessible-HTTP | `scan_http` | -| Accessible-Radmin | `scan_radmin` | -| Accessible-RDP | `scan_rdp` | -| Accessible-Rsync | `scan_rsync` | -| Accessible-SMB | `scan_smb` | -| Accessible-Telnet | `scan_telnet` | -| Accessible-Ubiquiti-Discovery-Service | `scan_ubiquiti` | -| Accessible-VNC | `scan_vnc` | -| Blacklisted-IP (deprecated) | `blacklist` | -| Blocklist | `blocklist` | -| Compromised-Website| `compromised_website` | -| Device-Identification-IPv4 | `device_id` | -| Device-Identification-IPv6 | `device_id6` | -| DNS-Open-Resolvers | `scan_dns` | -| Honeypot-Amplification-DDoS-Events | `event4_honeypot_ddos_amp` | -| Honeypot-Brute-Force-Events | `event4_honeypot_brute_force` | -| Honeypot-Darknet | `event4_honeypot_darknet` | -| Honeypot-HTTP-Scan | `event4_honeypot_http_scan` | -| HTTP-Scanners | `hp_http_scan` | -| ICS-Scanners | `hp_ics_scan` | -| IP-Spoofer-Events | `event4_ip_spoofer` | -| Microsoft-Sinkhole-Events-IPv4 | `event4_microsoft_sinkhole` | -| Microsoft-Sinkhole-Events-HTTP | `event4_microsoft_sinkhole_http` | -| NTP-Monitor | `scan_ntpmonitor` | -| NTP-Version | `scan_ntp` | -| Open-Chargen | `scan_chargen` | -| Open-DB2-Discovery-Service | `scan_db2` | -| Open-Elasticsearch | `scan_elasticsearch` | -| Open-IPMI| `scan_ipmi` | -| Open-IPP | `scan_ipp` | -| Open-LDAP | `scan_ldap` | -| Open-LDAP-TCP | `scan_ldap_tcp` | -| Open-mDNS | `scan_mdns` | -| Open-Memcached | `scan_memcached` | -| Open-MongoDB | `scan_mongodb` | -| Open-MQTT | `scan_mqtt` | -| Open-MSSQL | `scan_mssql` | -| Open-NATPMP | `scan_nat_pmp` | -| Open-NetBIOS-Nameservice | `scan_netbios` | -| Open-Netis | `netis_router` | -| Open-Portmapper | `scan_portmapper` | -| Open-QOTD | `scan_qotd` | -| Open-Redis | `scan_redis` | -| Open-SNMP | `scan_snmp` | -| Open-SSDP | `scan_ssdp` | -| Open-TFTP | `scan_tftp` | -| Open-XDMCP | `scan_xdmcp` | -| Outdated-DNSSEC-Key| `outdated_dnssec_key` | -| Outdated-DNSSEC-Key-IPv6 | `outdated_dnssec_key_v6` | -| Sandbox-URL | `cwsandbox_url` | -| Sinkhole-DNS | `sinkhole_dns` | -| Sinkhole-Events | `event4_sinkhole` | -| Sinkhole-Events IPv4 | `event4_sinkhole` | -| Sinkhole-Events IPv6 | `event6_sinkhole` | -| Sinkhole-HTTP-Events | `event4_sinkhole_http`/`event6_sinkhole_http` | -| Sinkhole-HTTP-Events IPv4 | `event4_sinkhole_http` | -| Sinkhole-HTTP-Events IPv6 | `event6_sinkhole_http` | -| Sinkhole-Events-HTTP-Referer| `event4_sinkhole_http_referer`/`event6_sinkhole_http_referer` | -| Sinkhole-Events-HTTP-Referer IPv4 | `event4_sinkhole_http_referer` | -| Sinkhole-Events-HTTP-Referer IPv6 | `event6_sinkhole_http_referer` | -| Spam-URL | `spam_url` | -| SSL-FREAK-Vulnerable-Servers | `scan_ssl_freak` | -| SSL-POODLE-Vulnerable-Servers | `scan_ssl_poodle`/`scan6_ssl_poodle` | -| Vulnerable-Exchange-Server* | `scan_exchange` | -| Vulnerable-ISAKMP | `scan_isakmp` | -| Vulnerable-HTTP | `scan_http` | -| Vulnerable-SMTP | `scan_smtp_vulnerable` | - -\* This report can also contain data on active webshells (column `tag` is `exchange;webshell`), and are therefore not -only vulnerable but also actively infected. - -In addition, the following legacy reports are supported: - -| Legacy Report Type | Successor Report Type | File Name | -|--------------------|-----------------------|-----------| -| Amplification-DDoS-Victim | Honeypot-Amplification-DDoS-Events | `ddos_amplification` | -| CAIDA-IP-Spoofer | IP-Spoofer-Events | `caida_ip_spoofer` | -| Darknet | Honeypot-Darknet | `darknet` | -| Drone | Sinkhole-Events | `botnet_drone` | -| Drone-Brute-Force | Honeypot-Brute-Force-Events, Sinkhole-HTTP-Events | `drone_brute_force` | -| Microsoft-Sinkhole | Sinkhole-HTTP-Events | `microsoft_sinkhole` | -| Sinkhole-HTTP-Drone | Sinkhole-HTTP-Events | `sinkhole_http_drone` | -| IPv6-Sinkhole-HTTP-Drone | Sinkhole-HTTP-Events | `sinkhole6_http` | - -More information on these legacy reports can be found -in [Changes in Sinkhole and Honeypot Report Types and Formats](https://www.shadowserver.org/news/changes-in-sinkhole-and-honeypot-report-types-and-formats/) -. +The report configuration is stored in a `shadowserver-schema.json` file downloaded from https://interchange.shadowserver.org/intelmq/v1/schema. + +The parser will attempt to download a schema update on startup when the *auto_update* option is enabled. +Schema downloads can also be scheduled as a cron job for the `intelmq` user: + +```bash + 02 01 * * * intelmq.bots.parsers.shadowserver.parser --update-schema +``` + +For air-gapped systems automation will be required to download and copy the file to VAR_STATE_PATH/shadowserver-schema.json. + +The parser will automatically reload the configuration when the file changes. + + +**Schema contract** + +Once set in the schema, the `classification.identifier`, `classification.taxonomy`, and `classification.type` fields will remain static for a specific report. + +The schema revision history is maintained at https://github.com/The-Shadowserver-Foundation/report_schema/. + + +**Sample configuration** + +```yaml + shadowserver-parser: + bot_id: shadowserver-parser + name: Shadowserver Parser + enabled: true + group: Parser + groupname: parsers + module: intelmq.bots.parsers.shadowserver.parser + parameters: + destination_queues: + _default: [file-output-queue] + auto_update: true + run_mode: continuous +``` --- ### Shodan
From ac0447189f2f1afe679c7ec9bdc2a58b922decc2 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Thu, 16 Nov 2023 15:53:55 +0000 Subject: [PATCH 67/76] Removed old unsorted doc and updated the taxonomy functions for the scan_exchange report. --- docs/unsorted/shadowserver.md | 24 -------------------- intelmq/bots/parsers/shadowserver/_config.py | 10 ++++---- 2 files changed, 5 insertions(+), 29 deletions(-) delete mode 100644 docs/unsorted/shadowserver.md diff --git a/docs/unsorted/shadowserver.md b/docs/unsorted/shadowserver.md deleted file mode 100644 index 1c7c2918e2..0000000000 --- a/docs/unsorted/shadowserver.md +++ /dev/null @@ -1,24 +0,0 @@ - - - -# Shadowserver Parser - -**Structure of this Parser Bot** - -The parser consists of two files: - -: - `_config.py` - -- `parser.py` or `parser_json.py` - -Both files are required for the parser to work properly. - -**Add new Feedformats** - -Add a new feed format and conversions if required to the file -`_config.py`. Don't forget to update the `mapping` dict. It is required to look up the correct configuration. - -Look at the documentation in the bot's `_config.py` file for more information. diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index c84bc7e85b..6931e54109 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -263,19 +263,19 @@ def force_base64(value: Optional[str]) -> Optional[str]: def scan_exchange_taxonomy(field): - if field == 'exchange;webshell': + if 'webshell' in field: return 'intrusions' return 'vulnerable' def scan_exchange_type(field): - if field == 'exchange;webshell': + if 'webshell' in field: return 'system-compromise' - return 'infected-system' + return 'vulnerable-system' def scan_exchange_identifier(field): - if field == 'exchange;webshell': + if 'webshell' in field: return 'exchange-server-webshell' return 'vulnerable-exchange-server' @@ -321,7 +321,7 @@ def reload(): return else: if not __config.test_mode: - raise ValueError("The schema file does not exist.") + raise ValueError("The schema file does not exist: %r.", __config.schema_file) if __config.schema_mtime == 0.0 and mtime == 0.0 and __config.auto_update: update_schema() From a81572abab18aa3bcc5666ea177bb12917845b23 Mon Sep 17 00:00:00 2001 From: Kamil Mankowski Date: Tue, 14 Nov 2023 10:09:35 +0100 Subject: [PATCH 68/76] Handle connection error in STOMP output Sometimes the output bot gots disconnected, and stays so until restart or reload. This change adds a try to reconnect. It intentionally doesn't hide the error, but let's IntelMQ handle message retry. --- CHANGELOG.md | 2 ++ intelmq/bots/outputs/stomp/output.py | 8 ++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2aae7fd66f..92eebfa5d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -84,6 +84,8 @@ of necessary file(s)). - Add `stomp.py` version check (raise `MissingDependencyError` if not `>=4.1.12`). - Minor fixes/improvements and some refactoring (see also above: *Core*...). +- `intelmq.bots.outputs.stomp.output` (PR#2423 by Kamil Mankowski): + - Try to reconnect on `NotConnectedException`. ### Documentation - Add a readthedocs configuration file to fix the build fail (PR#2403 by Sebastian Wagner). diff --git a/intelmq/bots/outputs/stomp/output.py b/intelmq/bots/outputs/stomp/output.py index 50c9a1d5f3..6beb0fa567 100644 --- a/intelmq/bots/outputs/stomp/output.py +++ b/intelmq/bots/outputs/stomp/output.py @@ -76,8 +76,12 @@ def process(self): body = self.export_event(event) - self._conn.send(body=body, - destination=self.exchange) + try: + self._conn.send(body=body, destination=self.exchange) + except stomp.exception.NotConnectedException: + self.logger.warning("Detected connection error, trying to reestablish it.") + self.connect() + raise # Fallback to default retry self.acknowledge_message() @classmethod From b775fcdb65635068831f54618f13e6d442f598fe Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Wed, 22 Nov 2023 20:15:38 +0100 Subject: [PATCH 69/76] ENH: upgrade: replace url2fqn by url expert the url2fqn expert is deprecated and will be removed in version 4 remove the bot from the default runtime config add an upgrade function that replaces the bot in installations by the url expert and parameters so that the behavior will stay the same fixes certtools/intelmq#2430 --- CHANGELOG.md | 1 + intelmq/etc/runtime.yaml | 12 +++++------ intelmq/lib/upgrades.py | 20 ++++++++++++++++++ intelmq/tests/lib/test_upgrades.py | 33 ++++++++++++++++++++++++++++++ 4 files changed, 60 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2aae7fd66f..13d85d575c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ which provides certain common STOMP-bot-specific operations, factored out from `intelmq.bots.collectors.stomp.collector` and `intelmq.bots.outputs.stomp.output` (PR#2408 and PR#2414 by Jan Kaliszewski). +- `intelmq.lib.upgrades`: Replace deprecated instances of `url2fqdn` experts by the new `url` expert in runtime configuration (PR#2432 by Sebastian Wagner). ### Development - Makefile: Add codespell and test commands (PR#2425 by Sebastian Wagner). diff --git a/intelmq/etc/runtime.yaml b/intelmq/etc/runtime.yaml index 170f0cbfda..c39c7b2482 100644 --- a/intelmq/etc/runtime.yaml +++ b/intelmq/etc/runtime.yaml @@ -156,16 +156,16 @@ taxonomy-expert: parameters: destination_queues: _default: - - url2fqdn-expert-queue + - url-expert-queue run_mode: continuous -url2fqdn-expert: - bot_id: url2fqdn-expert - description: url2fqdn is the bot responsible to parsing the fqdn from the url. +url-expert: + bot_id: url-expert + description: Extract additional information for the URL enabled: true group: Expert groupname: experts - module: intelmq.bots.experts.url2fqdn.expert - name: URL2FQDN + module: intelmq.bots.experts.url.expert + name: url parameters: destination_queues: _default: diff --git a/intelmq/lib/upgrades.py b/intelmq/lib/upgrades.py index a920d4be87..1972a628a0 100644 --- a/intelmq/lib/upgrades.py +++ b/intelmq/lib/upgrades.py @@ -39,6 +39,7 @@ 'v310_feed_changes', 'v310_shadowserver_feednames', 'v320_update_turris_greylist_url', + 'v322_url_replacement', ] @@ -879,6 +880,24 @@ def v320_update_turris_greylist_url(configuration, harmonization, dry_run, **kwa return ' '.join(messages) if messages else None, configuration, harmonization +def v322_url_replacement(configuration, harmonization, dry_run, **kwargs): + """ + Replace deprecated url2fqdn expert with url expert. + """ + changed = False + for bot_id, bot in configuration.items(): + if bot_id == 'global': + continue + if bot["module"] == "intelmq.bots.experts.url2fqdn.expert": + configuration[bot_id]["module"] = "intelmq.bots.experts.url.expert" + if "parameters" not in configuration[bot_id]: + configuration[bot_id]["parameters"] = {} + # skip all fields except the fqdn field for backwards compatibility + configuration[bot_id]["parameters"]["skip_fields"] = ["source.ip", "source.port", "source.urlpath", "source.account", "destination.ip", "destination.port", "destination.urlpath", "destination.account", "protocol.application", "protocol.transport"] + changed = True + return changed, configuration, harmonization + + UPGRADES = OrderedDict([ ((1, 0, 0, 'dev7'), (v100_dev7_modify_syntax,)), ((1, 1, 0), (v110_shadowserver_feednames, v110_deprecations)), @@ -905,6 +924,7 @@ def v320_update_turris_greylist_url(configuration, harmonization, dry_run, **kwa ((3, 0, 2), ()), ((3, 1, 0), (v310_feed_changes, v310_shadowserver_feednames)), ((3, 2, 0), (v320_update_turris_greylist_url,)), + ((3, 2, 2), (v322_url_replacement, )), ]) ALWAYS = (harmonization,) diff --git a/intelmq/tests/lib/test_upgrades.py b/intelmq/tests/lib/test_upgrades.py index 151bfb161d..dfdfdca0a5 100644 --- a/intelmq/tests/lib/test_upgrades.py +++ b/intelmq/tests/lib/test_upgrades.py @@ -550,6 +550,29 @@ "parameters": {} } } +V322_URL2FQN_IN = { + "global": {}, + "url2fqdn-expert": { + "module": "intelmq.bots.experts.url2fqdn.expert", + "parameters": { + } + }, +} +V322_URL2FQN_IN_1 = { + "global": {}, + "url2fqdn-expert": { + "module": "intelmq.bots.experts.url2fqdn.expert", + }, +} +V322_URL2FQN_OUT = { + "global": {}, + "url2fqdn-expert": { + "module": "intelmq.bots.experts.url.expert", + "parameters": { + "skip_fields": ["source.ip", "source.port", "source.urlpath", "source.account", "destination.ip", "destination.port", "destination.urlpath", "destination.account", "protocol.application", "protocol.transport"] + }, + }, +} def generate_function(function): @@ -762,6 +785,16 @@ def test_v310_feed_changes(self): result[0]) self.assertEqual(V310_FEED_CHANGES, result[1]) + def test_v322_url_replacement(self): + """ Test v322_url_replacement """ + result = upgrades.v322_url_replacement(V322_URL2FQN_IN, {}, False) + self.assertTrue(result[0]) + self.assertEqual(V322_URL2FQN_OUT, result[1]) + + result = upgrades.v322_url_replacement(V322_URL2FQN_IN_1, {}, False) + self.assertTrue(result[0]) + self.assertEqual(V322_URL2FQN_OUT, result[1]) + for name in upgrades.__all__: setattr(TestUpgradeLib, 'test_function_%s' % name, From 9c6f867752c034ab6971a3083f9cc1a1ed1f82c9 Mon Sep 17 00:00:00 2001 From: Kamil Mankowski Date: Mon, 11 Dec 2023 12:55:45 +0100 Subject: [PATCH 70/76] FIX: Ensure closing log files During the reloading process, log handlers are not explicitly closed. This may cause long living open file handlers and keeping cached files in the memory. If log files are big, the RAM cache usage can be very high. --- CHANGELOG.md | 1 + intelmq/lib/bot.py | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2613d7e96b..e3b8d0e20e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ `intelmq.bots.collectors.stomp.collector` and `intelmq.bots.outputs.stomp.output` (PR#2408 and PR#2414 by Jan Kaliszewski). - `intelmq.lib.upgrades`: Replace deprecated instances of `url2fqdn` experts by the new `url` expert in runtime configuration (PR#2432 by Sebastian Wagner). +- `intelmq.lib.bot`: Ensure closing log files on reloading (PR by Kamil Mankowski). ### Development - Makefile: Add codespell and test commands (PR#2425 by Sebastian Wagner). diff --git a/intelmq/lib/bot.py b/intelmq/lib/bot.py index 31b6847c1e..d04b24b7cf 100644 --- a/intelmq/lib/bot.py +++ b/intelmq/lib/bot.py @@ -308,10 +308,25 @@ def __handle_sighup(self): self.shutdown() # disconnects, stops threads etc except Exception: self.logger.exception('Error during shutdown of bot.') - self.logger.handlers = [] # remove all existing handlers + self.__cleanup_logging_handlers() self.__sighup.clear() self.__init__(self.__bot_id_full, sighup_event=self.__sighup, standalone=self._standalone) + def __cleanup_logging_handlers(self): + # thread-safe removing of handlers and closing opened files + handlers_list = self.logger.handlers[:] + for handler in handlers_list: + try: + self.logger.removeHandler(handler) + # ensure all log files are closed to prevent caching them in RAM + if isinstance(handler, logging.FileHandler): + handler.close() + except: + # Logger should still be safe to use even without handlers + # In addition, we do not want any side issue to break execution + # - we are about to reinitialize logging. + self.logger.exception("Error while cleaning up logging handlers") + def init(self): pass From 66bef8f530412ed7fa8b85f21d168a59a4c77465 Mon Sep 17 00:00:00 2001 From: Kamil Mankowski Date: Mon, 11 Dec 2023 13:59:10 +0100 Subject: [PATCH 71/76] FIX: Return None when no changes in upgrade Fixes: #2433 --- intelmq/lib/upgrades.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/intelmq/lib/upgrades.py b/intelmq/lib/upgrades.py index 1972a628a0..7391e3c18e 100644 --- a/intelmq/lib/upgrades.py +++ b/intelmq/lib/upgrades.py @@ -884,7 +884,7 @@ def v322_url_replacement(configuration, harmonization, dry_run, **kwargs): """ Replace deprecated url2fqdn expert with url expert. """ - changed = False + changed = None for bot_id, bot in configuration.items(): if bot_id == 'global': continue From e8f1dc8d53f1173be59513c70bdec7a34ad54dc6 Mon Sep 17 00:00:00 2001 From: Kamil Mankowski Date: Mon, 11 Dec 2023 15:03:05 +0100 Subject: [PATCH 72/76] Add PR number --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e3b8d0e20e..25c1e64007 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,7 +33,7 @@ `intelmq.bots.collectors.stomp.collector` and `intelmq.bots.outputs.stomp.output` (PR#2408 and PR#2414 by Jan Kaliszewski). - `intelmq.lib.upgrades`: Replace deprecated instances of `url2fqdn` experts by the new `url` expert in runtime configuration (PR#2432 by Sebastian Wagner). -- `intelmq.lib.bot`: Ensure closing log files on reloading (PR by Kamil Mankowski). +- `intelmq.lib.bot`: Ensure closing log files on reloading (PR#2435 by Kamil Mankowski). ### Development - Makefile: Add codespell and test commands (PR#2425 by Sebastian Wagner). From 4728a322d91a8e1e3dcbd9ffab703e95dc81e170 Mon Sep 17 00:00:00 2001 From: Kamil Mankowski Date: Tue, 12 Dec 2023 11:21:03 +0100 Subject: [PATCH 73/76] ENH: add support for generating JSONB columns JSONB offers additional features, like indexing and quick data extraction. --- CHANGELOG.md | 4 +++- intelmq/bin/intelmq_psql_initdb.py | 8 ++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2613d7e96b..161bceec27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -104,7 +104,9 @@ ### Tests ### Tools - - `intelmq_psql_initdb` got support for providing custom harmonization file, generating view for storing `raw` fields separately, and adding `IF NOT EXISTS`/`OR REPLACE` clauses ([PR#2404](https://github.com/certtools/intelmq/pull/2404) by Kamil Mankowski). + - `intelmq_psql_initdb`: + - got support for providing custom harmonization file, generating view for storing `raw` fields separately, and adding `IF NOT EXISTS`/`OR REPLACE` clauses ([PR#2404](https://github.com/certtools/intelmq/pull/2404) by Kamil Mankowski). + - got support for generating JSONB fields for PostgreSQL schema (PR by Kamil Mankowski). ### Contrib diff --git a/intelmq/bin/intelmq_psql_initdb.py b/intelmq/bin/intelmq_psql_initdb.py index 34a55a380e..5b715a0e7a 100644 --- a/intelmq/bin/intelmq_psql_initdb.py +++ b/intelmq/bin/intelmq_psql_initdb.py @@ -132,7 +132,8 @@ def _generate_separated_raws_schema(fields: dict, partition_key: str) -> list: def generate(harmonization_file=HARMONIZATION_CONF_FILE, skip_events=False, - separate_raws=False, partition_key=None, skip_or_replace=False): + separate_raws=False, partition_key=None, skip_or_replace=False, + use_jsonb=False): FIELDS = {} sql_lines = [] @@ -170,7 +171,7 @@ def generate(harmonization_file=HARMONIZATION_CONF_FILE, skip_events=False, elif value['type'] == 'UUID': dbtype = 'UUID' elif value['type'] in ('JSON', 'JSONDict'): - dbtype = 'json' + dbtype = 'jsonb' if use_jsonb else 'json' else: raise ValueError('Unknown type %r.' % value['type']) @@ -212,6 +213,8 @@ def main(): help="Path to the harmonization file") parser.add_argument("--skip-or-replace", default=False, action="store_true", help="Add IF NOT EXISTS or REPLACE directive to created schemas") + parser.add_argument("--jsonb", default=False, action="store_true", + help="Use JSONB type to represent dictionary fields") args = parser.parse_args() OUTPUTFILE = args.outputfile @@ -229,6 +232,7 @@ def main(): separate_raws=args.separate_raws, partition_key=args.partition_key, skip_or_replace=args.skip_or_replace, + use_jsonb=args.jsonb, ) print("INFO - Writing %s file" % OUTPUTFILE) fp.write(psql) From 377b706dbcadfa574849aac657ef7eb8b89f42ce Mon Sep 17 00:00:00 2001 From: Sebastian Date: Wed, 13 Dec 2023 10:47:22 +0100 Subject: [PATCH 74/76] Update CHANGELOG.md: Add PR number for #2436 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 161bceec27..af4081552f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -106,7 +106,7 @@ ### Tools - `intelmq_psql_initdb`: - got support for providing custom harmonization file, generating view for storing `raw` fields separately, and adding `IF NOT EXISTS`/`OR REPLACE` clauses ([PR#2404](https://github.com/certtools/intelmq/pull/2404) by Kamil Mankowski). - - got support for generating JSONB fields for PostgreSQL schema (PR by Kamil Mankowski). + - got support for generating JSONB fields for PostgreSQL schema (PR#2436 by Kamil Mankowski). ### Contrib From 632fd7a702fe87bf63f00f47f75d9397fa68caa7 Mon Sep 17 00:00:00 2001 From: gethvi Date: Wed, 13 Dec 2023 18:00:07 +0100 Subject: [PATCH 75/76] FIX: Adds missing SMTP Batch Output bot to the docs. --- docs/user/bots.md | 126 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) diff --git a/docs/user/bots.md b/docs/user/bots.md index 57baf0a884..c97c8f5fe1 100644 --- a/docs/user/bots.md +++ b/docs/user/bots.md @@ -5008,6 +5008,132 @@ rpz.yourdomain.eu. *.secondmaliciousdomain.com CNAME rpz.yourdomain.eu. --- +### SMTP Batch
+ +Aggregate events by e-mail addresses in the `source.abuse_contact` field and batch send them at once as a zipped CSV file attachment in a GPG signed message. + +When the bot is run normally by IntelMQ, it just aggregates the events for later use into a custom Redis database. +If run through CLI (by a cron or manually), it shows e-mail messages that are ready to be sent and let you send them to the tester's e-mail OR to abuse contact e-mails. +E-mails are sent in a zipped CSV file, delimited by a comma, while keeping strings in double quotes. +Note: The field "raw" gets base64 decoded if possible. Bytes `\n` and `\r` are replaced with "\n" and "\r" strings in order to guarantee best CSV files readability both in Microsoft Office and LibreOffice. (A multiline string may be stored in "raw" which completely confused Microsoft Excel.) + +Launch it like this: +``` + cli [--tester tester's email] +``` +Example: +```bash +intelmq.bots.outputs.smtp_batch.output smtp-batch-output --cli --tester your-email@example.com +``` + +CLI flags: +``` +-h, --help show this help message and exit +--cli initiate CLI interface +--tester TESTING_TO tester's e-mail +--ignore-older-than-days IGNORE_OLDER_THAN_DAYS + 1..n skip all events with time.observation older than 1..n day; 0 disabled (allow all) +--gpg-key GPG_KEY fingerprint of gpg key to be used +--limit-results LIMIT_RESULTS + Just send first N mails. +--send Sends now, without dialog. +``` + +You can schedule the batch sending easily with a cron script, I.E. put this into `crontab -e` of the `intelmq` user: + +``` +# Send the e-mails every day at 6 AM +0 6 * * * /usr/local/bin/intelmq.bots.outputs.smtp_batch.output smtp-batch-output-cz cli --ignore-older-than-days 4 --send > /tmp/intelmq-send.log +``` + +**Module:** `intelmq.bots.outputs.smtp_batch.output` + +**Parameters:** + +**`alternative_mails`** + +(optional, string) Path to CSV in the form `original@email.com,alternative@email.com`. Needed when some of the recipients ask you to forward their e-mails to another address. + +**`attachment_name`** + +(optional, string) Attachment file name for the outgoing messages. May contain date formatting like this `%Y-%m-%d`. Example: "events_%Y-%m-%d" will appear as "events_2022-12-01.zip". Defaults to "intelmq_%Y-%m-%d". + +**`bcc`** + +(optional, array of strings) An array of e-mails to be put in the `Bcc` field for every mail. + +**`email_from`** + +(required, string) Sender's e-mail of the outgoing messages. + + +**`gpg_key`** + +(optional, string) The Key or the fingerprint of a GPG key stored in ~/.gnupg keyring folder. + + +**`gpg_pass`** + +(optional, string) Password for the GPG key if needed. + + +**`mail_template`** + +(required, string) Path to the file containing the body of the mail for the outgoing messages. + + +**`ignore_older_than_days`** + +(optional, integer) Skips events with time.observation older than now-N. (If your queue gets stuck for a reason, you do not want to send old and probably already solved events.) Defaults to 0 (allow all). + + +**`limit_results`** + +(optional, integer) Intended as a debugging option, allows loading just first N e-mails from the queue. + + +**`redis_cache_db`** + +(required, integer) Redis database used for event aggregation. As the databases < 10 are reserved for the IntelMQ core, recommended is a bigger number. + + +**`redis_cache_host`** + +(required, string) Hostname of the Redis database. + + +**`redis_cache_port`** + +(required, string) Port of the Redis database. + + +**`redis_cache_ttl`** + +(required, integer) TTL in seconds used for caching. Recommended 1728000 for 20 days. + + +**`smtp_server`** + +(required, string/array/object) SMTP server information and credentials. See [SMTP parameter](https://github.com/CZ-NIC/envelope#sending) of the envelope module. + +Examples: +```yaml +smtp_server: "mailer" +smtp_server: {"host": "mailer", "port": 587, "user": "john", "password": "123"} +smtp_server: ["mailer", 587, "john", "password"] +``` + +**`subject`** + +(required, string) Subject for the outgoing messages. May contain date formatting like this `%Y-%m-%d`. Example: "IntelMQ weekly warning (%d.%m.%Y)". + + +**`testing_to`** + +(optional, string) Tester's e-mail. + +--- + ### SMTP
Sends a MIME Multipart message containing the text and the event as CSV for every single event. From be25ed264306af4da04ce9e28df91b73e0c1b568 Mon Sep 17 00:00:00 2001 From: Edvard Rejthar Date: Wed, 13 Dec 2023 11:43:16 +0100 Subject: [PATCH 76/76] Fix smtp_batch bcc #2437 --- CHANGELOG.md | 2 ++ intelmq/bots/outputs/smtp_batch/output.py | 7 +++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 38a140f6ee..2c9fc22620 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -88,6 +88,8 @@ - Minor fixes/improvements and some refactoring (see also above: *Core*...). - `intelmq.bots.outputs.stomp.output` (PR#2423 by Kamil Mankowski): - Try to reconnect on `NotConnectedException`. +- `intelmq.bots.outputs.smtp_batch.output` (PR #2439 by Edvard Rejthar): + - Fix ability to send with the default `bcc` ### Documentation - Add a readthedocs configuration file to fix the build fail (PR#2403 by Sebastian Wagner). diff --git a/intelmq/bots/outputs/smtp_batch/output.py b/intelmq/bots/outputs/smtp_batch/output.py index 5d2779f47d..2a01b1ed27 100644 --- a/intelmq/bots/outputs/smtp_batch/output.py +++ b/intelmq/bots/outputs/smtp_batch/output.py @@ -246,9 +246,8 @@ def prepare_mails(self): lines.extend(json.loads(str(message, encoding="utf-8")) for message in messages) # prepare rows for csv attachment - threshold = datetime.datetime.now() - datetime.timedelta( - days=self.ignore_older_than_days) if getattr(self, 'ignore_older_than_days', - False) else False + threshold = self.ignore_older_than_days and \ + datetime.datetime.now() - datetime.timedelta(days=self.ignore_older_than_days) # TODO: worthy to generate on the fly https://github.com/certtools/intelmq/pull/2253#discussion_r1172779620 fieldnames = set() @@ -338,7 +337,7 @@ def build_mail(self, mail, send=False, override_to=None): return (Envelope(text) .attach(path=mail.path, name=attachment_name + '.zip') .from_(email_from).to(email_to) - .bcc([] if intended_to else getattr(self, 'bcc', [])) + .bcc(not intended_to and self.bcc or []) .subject(subject) .smtp(self.smtp_server) .signature(self.gpg_key, self.gpg_pass)