From 6ce6769d6775c75f3ea870edff90edf1ec6cf681 Mon Sep 17 00:00:00 2001 From: Ryan Hays Date: Fri, 17 May 2019 10:31:07 -0800 Subject: [PATCH 1/7] Added new Binary Edge module to query IP Address and populate ports table. --- modules/recon/hosts-ports/binaryedge_ports.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 modules/recon/hosts-ports/binaryedge_ports.py diff --git a/modules/recon/hosts-ports/binaryedge_ports.py b/modules/recon/hosts-ports/binaryedge_ports.py new file mode 100644 index 0000000..d841800 --- /dev/null +++ b/modules/recon/hosts-ports/binaryedge_ports.py @@ -0,0 +1,23 @@ +from recon.core.module import BaseModule + +class Module(BaseModule): + + meta = { + 'name': 'BinaryEdge.io Ports lookup', + 'author': 'Ryan Hays', + 'version': '1.0', + 'description': 'Uses the BinaryEdge API to discover open services for IP Addresses.', + 'required_keys': ['binaryedge_api'], + 'query': 'SELECT DISTINCT ip_address FROM hosts WHERE ip_address IS NOT NULL', + } + + def module_run(self, ipaddrs): + key = self.get_key('binaryedge_api') + for ipaddr in ipaddrs: + self.heading(ipaddr, level=0) + resp = self.request(url=f"https://api.binaryedge.io/v2/query/ip/{ipaddr}", headers={'X-Key': key}) + if resp.status_code == 200: + for event in resp.json().get('events'): + for result in event['results']: + self.insert_ports(ip_address=ipaddr, port=result['target']['port'], + protocol=result['target']['protocol']) From d4dc2bf27413be696d17a5d067f38f93d8f80528 Mon Sep 17 00:00:00 2001 From: Ryan Hays Date: Fri, 17 May 2019 10:33:21 -0800 Subject: [PATCH 2/7] Revert "Added new Binary Edge module to query IP Address and populate ports table." This reverts commit 6ce6769d --- modules/recon/hosts-ports/binaryedge_ports.py | 23 ------------------- 1 file changed, 23 deletions(-) delete mode 100644 modules/recon/hosts-ports/binaryedge_ports.py diff --git a/modules/recon/hosts-ports/binaryedge_ports.py b/modules/recon/hosts-ports/binaryedge_ports.py deleted file mode 100644 index d841800..0000000 --- a/modules/recon/hosts-ports/binaryedge_ports.py +++ /dev/null @@ -1,23 +0,0 @@ -from recon.core.module import BaseModule - -class Module(BaseModule): - - meta = { - 'name': 'BinaryEdge.io Ports lookup', - 'author': 'Ryan Hays', - 'version': '1.0', - 'description': 'Uses the BinaryEdge API to discover open services for IP Addresses.', - 'required_keys': ['binaryedge_api'], - 'query': 'SELECT DISTINCT ip_address FROM hosts WHERE ip_address IS NOT NULL', - } - - def module_run(self, ipaddrs): - key = self.get_key('binaryedge_api') - for ipaddr in ipaddrs: - self.heading(ipaddr, level=0) - resp = self.request(url=f"https://api.binaryedge.io/v2/query/ip/{ipaddr}", headers={'X-Key': key}) - if resp.status_code == 200: - for event in resp.json().get('events'): - for result in event['results']: - self.insert_ports(ip_address=ipaddr, port=result['target']['port'], - protocol=result['target']['protocol']) From 3778ce203c65a01082afc7d7fdd64b0754c3ce69 Mon Sep 17 00:00:00 2001 From: Ryan Hays Date: Fri, 31 May 2019 08:25:14 -0400 Subject: [PATCH 3/7] Added a new module that allowsd users to validate scope of hosts within the hosts table. --- modules.yml | 11 +++++++++ modules/scope/scope_checker.py | 41 ++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 modules/scope/scope_checker.py diff --git a/modules.yml b/modules.yml index c5e9360..646bc83 100644 --- a/modules.yml +++ b/modules.yml @@ -1050,3 +1050,14 @@ path: import/nmap required_keys: [] version: '1.0' +- author: Ryan Hays + dependencies: [] + description: Compares the hosts table and removes hosts that are not found within + the provided scope document. Scope should be defined within the document either + by single IP or in CIDR notation (eg. 192.168.1.0/24) + files: [] + last_updated: '2019-05-31' + name: Scope Checker + path: scope/scope_checker + required_keys: [] + version: '1.0' \ No newline at end of file diff --git a/modules/scope/scope_checker.py b/modules/scope/scope_checker.py new file mode 100644 index 0000000..10789df --- /dev/null +++ b/modules/scope/scope_checker.py @@ -0,0 +1,41 @@ +from recon.core.module import BaseModule +import ipaddress + + +class Module(BaseModule): + meta = { + 'name': 'Scope Checker', + 'author': 'Ryan Hays', + 'version': '1.0', + 'description': 'Compares the hosts table and removes hosts that are not found within the provided scope document. ' + 'Scope should be defined within the document either by single IP or in CIDR notation (eg. 192.168.1.0/24)', + 'query': 'SELECT DISTINCT ip_address FROM hosts WHERE ip_address IS NOT NULL', + 'options': ( + ('filename', None, True, 'path and filename for scope document'), + ), + } + + def module_run(self, hosts): + cidr_dict = {} + with open(self.options['filename'], 'rt') as fp: + for cidr in fp: + try: + cidr_dict[ipaddress.ip_network(cidr.rstrip())] = cidr.rstrip() + except ValueError: + self.error(f"Invalid CIDR notation for: {cidr.rstrip()}") + + cnt = 0 + for host in hosts: + in_scope = False + ip = ipaddress.ip_address(host) + for net in cidr_dict.keys(): + if ip in net: + in_scope = True + break + + if not in_scope: + cnt += 1 + self.alert(f"{str(ip)} not found in scope document removing from hosts table.") + self.query('DELETE FROM hosts WHERE ip_address=?', (str(ip),)) + + self.output(f"Remove {cnt} out of scope hosts from the hosts table.") From c5a252b9392a1c94f0789fe17765c82d57e2287b Mon Sep 17 00:00:00 2001 From: Ryan Hays Date: Mon, 3 Jun 2019 21:22:40 -0400 Subject: [PATCH 4/7] Added functionality for checking domains within the hosts table as well as IPs and CIDR. --- data/tld_list.txt | 1532 ++++++++++++++++++++++++++++++++ modules/scope/scope_checker.py | 31 +- 2 files changed, 1559 insertions(+), 4 deletions(-) create mode 100644 data/tld_list.txt diff --git a/data/tld_list.txt b/data/tld_list.txt new file mode 100644 index 0000000..c4a899c --- /dev/null +++ b/data/tld_list.txt @@ -0,0 +1,1532 @@ +# Version 2019060300, Last Updated Mon Jun 3 07:07:02 2019 UTC +AAA +AARP +ABARTH +ABB +ABBOTT +ABBVIE +ABC +ABLE +ABOGADO +ABUDHABI +AC +ACADEMY +ACCENTURE +ACCOUNTANT +ACCOUNTANTS +ACO +ACTOR +AD +ADAC +ADS +ADULT +AE +AEG +AERO +AETNA +AF +AFAMILYCOMPANY +AFL +AFRICA +AG +AGAKHAN +AGENCY +AI +AIG +AIGO +AIRBUS +AIRFORCE +AIRTEL +AKDN +AL +ALFAROMEO +ALIBABA +ALIPAY +ALLFINANZ +ALLSTATE +ALLY +ALSACE +ALSTOM +AM +AMERICANEXPRESS +AMERICANFAMILY +AMEX +AMFAM +AMICA +AMSTERDAM +ANALYTICS +ANDROID +ANQUAN +ANZ +AO +AOL +APARTMENTS +APP +APPLE +AQ +AQUARELLE +AR +ARAB +ARAMCO +ARCHI +ARMY +ARPA +ART +ARTE +AS +ASDA +ASIA +ASSOCIATES +AT +ATHLETA +ATTORNEY +AU +AUCTION +AUDI +AUDIBLE +AUDIO +AUSPOST +AUTHOR +AUTO +AUTOS +AVIANCA +AW +AWS +AX +AXA +AZ +AZURE +BA +BABY +BAIDU +BANAMEX +BANANAREPUBLIC +BAND +BANK +BAR +BARCELONA +BARCLAYCARD +BARCLAYS +BAREFOOT +BARGAINS +BASEBALL +BASKETBALL +BAUHAUS +BAYERN +BB +BBC +BBT +BBVA +BCG +BCN +BD +BE +BEATS +BEAUTY +BEER +BENTLEY +BERLIN +BEST +BESTBUY +BET +BF +BG +BH +BHARTI +BI +BIBLE +BID +BIKE +BING +BINGO +BIO +BIZ +BJ +BLACK +BLACKFRIDAY +BLOCKBUSTER +BLOG +BLOOMBERG +BLUE +BM +BMS +BMW +BN +BNL +BNPPARIBAS +BO +BOATS +BOEHRINGER +BOFA +BOM +BOND +BOO +BOOK +BOOKING +BOSCH +BOSTIK +BOSTON +BOT +BOUTIQUE +BOX +BR +BRADESCO +BRIDGESTONE +BROADWAY +BROKER +BROTHER +BRUSSELS +BS +BT +BUDAPEST +BUGATTI +BUILD +BUILDERS +BUSINESS +BUY +BUZZ +BV +BW +BY +BZ +BZH +CA +CAB +CAFE +CAL +CALL +CALVINKLEIN +CAM +CAMERA +CAMP +CANCERRESEARCH +CANON +CAPETOWN +CAPITAL +CAPITALONE +CAR +CARAVAN +CARDS +CARE +CAREER +CAREERS +CARS +CARTIER +CASA +CASE +CASEIH +CASH +CASINO +CAT +CATERING +CATHOLIC +CBA +CBN +CBRE +CBS +CC +CD +CEB +CENTER +CEO +CERN +CF +CFA +CFD +CG +CH +CHANEL +CHANNEL +CHARITY +CHASE +CHAT +CHEAP +CHINTAI +CHRISTMAS +CHROME +CHRYSLER +CHURCH +CI +CIPRIANI +CIRCLE +CISCO +CITADEL +CITI +CITIC +CITY +CITYEATS +CK +CL +CLAIMS +CLEANING +CLICK +CLINIC +CLINIQUE +CLOTHING +CLOUD +CLUB +CLUBMED +CM +CN +CO +COACH +CODES +COFFEE +COLLEGE +COLOGNE +COM +COMCAST +COMMBANK +COMMUNITY +COMPANY +COMPARE +COMPUTER +COMSEC +CONDOS +CONSTRUCTION +CONSULTING +CONTACT +CONTRACTORS +COOKING +COOKINGCHANNEL +COOL +COOP +CORSICA +COUNTRY +COUPON +COUPONS +COURSES +CR +CREDIT +CREDITCARD +CREDITUNION +CRICKET +CROWN +CRS +CRUISE +CRUISES +CSC +CU +CUISINELLA +CV +CW +CX +CY +CYMRU +CYOU +CZ +DABUR +DAD +DANCE +DATA +DATE +DATING +DATSUN +DAY +DCLK +DDS +DE +DEAL +DEALER +DEALS +DEGREE +DELIVERY +DELL +DELOITTE +DELTA +DEMOCRAT +DENTAL +DENTIST +DESI +DESIGN +DEV +DHL +DIAMONDS +DIET +DIGITAL +DIRECT +DIRECTORY +DISCOUNT +DISCOVER +DISH +DIY +DJ +DK +DM +DNP +DO +DOCS +DOCTOR +DODGE +DOG +DOMAINS +DOT +DOWNLOAD +DRIVE +DTV +DUBAI +DUCK +DUNLOP +DUNS +DUPONT +DURBAN +DVAG +DVR +DZ +EARTH +EAT +EC +ECO +EDEKA +EDU +EDUCATION +EE +EG +EMAIL +EMERCK +ENERGY +ENGINEER +ENGINEERING +ENTERPRISES +EPSON +EQUIPMENT +ER +ERICSSON +ERNI +ES +ESQ +ESTATE +ESURANCE +ET +ETISALAT +EU +EUROVISION +EUS +EVENTS +EVERBANK +EXCHANGE +EXPERT +EXPOSED +EXPRESS +EXTRASPACE +FAGE +FAIL +FAIRWINDS +FAITH +FAMILY +FAN +FANS +FARM +FARMERS +FASHION +FAST +FEDEX +FEEDBACK +FERRARI +FERRERO +FI +FIAT +FIDELITY +FIDO +FILM +FINAL +FINANCE +FINANCIAL +FIRE +FIRESTONE +FIRMDALE +FISH +FISHING +FIT +FITNESS +FJ +FK +FLICKR +FLIGHTS +FLIR +FLORIST +FLOWERS +FLY +FM +FO +FOO +FOOD +FOODNETWORK +FOOTBALL +FORD +FOREX +FORSALE +FORUM +FOUNDATION +FOX +FR +FREE +FRESENIUS +FRL +FROGANS +FRONTDOOR +FRONTIER +FTR +FUJITSU +FUJIXEROX +FUN +FUND +FURNITURE +FUTBOL +FYI +GA +GAL +GALLERY +GALLO +GALLUP +GAME +GAMES +GAP +GARDEN +GB +GBIZ +GD +GDN +GE +GEA +GENT +GENTING +GEORGE +GF +GG +GGEE +GH +GI +GIFT +GIFTS +GIVES +GIVING +GL +GLADE +GLASS +GLE +GLOBAL +GLOBO +GM +GMAIL +GMBH +GMO +GMX +GN +GODADDY +GOLD +GOLDPOINT +GOLF +GOO +GOODYEAR +GOOG +GOOGLE +GOP +GOT +GOV +GP +GQ +GR +GRAINGER +GRAPHICS +GRATIS +GREEN +GRIPE +GROCERY +GROUP +GS +GT +GU +GUARDIAN +GUCCI +GUGE +GUIDE +GUITARS +GURU +GW +GY +HAIR +HAMBURG +HANGOUT +HAUS +HBO +HDFC +HDFCBANK +HEALTH +HEALTHCARE +HELP +HELSINKI +HERE +HERMES +HGTV +HIPHOP +HISAMITSU +HITACHI +HIV +HK +HKT +HM +HN +HOCKEY +HOLDINGS +HOLIDAY +HOMEDEPOT +HOMEGOODS +HOMES +HOMESENSE +HONDA +HONEYWELL +HORSE +HOSPITAL +HOST +HOSTING +HOT +HOTELES +HOTELS +HOTMAIL +HOUSE +HOW +HR +HSBC +HT +HU +HUGHES +HYATT +HYUNDAI +IBM +ICBC +ICE +ICU +ID +IE +IEEE +IFM +IKANO +IL +IM +IMAMAT +IMDB +IMMO +IMMOBILIEN +IN +INC +INDUSTRIES +INFINITI +INFO +ING +INK +INSTITUTE +INSURANCE +INSURE +INT +INTEL +INTERNATIONAL +INTUIT +INVESTMENTS +IO +IPIRANGA +IQ +IR +IRISH +IS +ISELECT +ISMAILI +IST +ISTANBUL +IT +ITAU +ITV +IVECO +JAGUAR +JAVA +JCB +JCP +JE +JEEP +JETZT +JEWELRY +JIO +JLL +JM +JMP +JNJ +JO +JOBS +JOBURG +JOT +JOY +JP +JPMORGAN +JPRS +JUEGOS +JUNIPER +KAUFEN +KDDI +KE +KERRYHOTELS +KERRYLOGISTICS +KERRYPROPERTIES +KFH +KG +KH +KI +KIA +KIM +KINDER +KINDLE +KITCHEN +KIWI +KM +KN +KOELN +KOMATSU +KOSHER +KP +KPMG +KPN +KR +KRD +KRED +KUOKGROUP +KW +KY +KYOTO +KZ +LA +LACAIXA +LADBROKES +LAMBORGHINI +LAMER +LANCASTER +LANCIA +LANCOME +LAND +LANDROVER +LANXESS +LASALLE +LAT +LATINO +LATROBE +LAW +LAWYER +LB +LC +LDS +LEASE +LECLERC +LEFRAK +LEGAL +LEGO +LEXUS +LGBT +LI +LIAISON +LIDL +LIFE +LIFEINSURANCE +LIFESTYLE +LIGHTING +LIKE +LILLY +LIMITED +LIMO +LINCOLN +LINDE +LINK +LIPSY +LIVE +LIVING +LIXIL +LK +LLC +LOAN +LOANS +LOCKER +LOCUS +LOFT +LOL +LONDON +LOTTE +LOTTO +LOVE +LPL +LPLFINANCIAL +LR +LS +LT +LTD +LTDA +LU +LUNDBECK +LUPIN +LUXE +LUXURY +LV +LY +MA +MACYS +MADRID +MAIF +MAISON +MAKEUP +MAN +MANAGEMENT +MANGO +MAP +MARKET +MARKETING +MARKETS +MARRIOTT +MARSHALLS +MASERATI +MATTEL +MBA +MC +MCKINSEY +MD +ME +MED +MEDIA +MEET +MELBOURNE +MEME +MEMORIAL +MEN +MENU +MERCKMSD +METLIFE +MG +MH +MIAMI +MICROSOFT +MIL +MINI +MINT +MIT +MITSUBISHI +MK +ML +MLB +MLS +MM +MMA +MN +MO +MOBI +MOBILE +MOBILY +MODA +MOE +MOI +MOM +MONASH +MONEY +MONSTER +MOPAR +MORMON +MORTGAGE +MOSCOW +MOTO +MOTORCYCLES +MOV +MOVIE +MOVISTAR +MP +MQ +MR +MS +MSD +MT +MTN +MTR +MU +MUSEUM +MUTUAL +MV +MW +MX +MY +MZ +NA +NAB +NADEX +NAGOYA +NAME +NATIONWIDE +NATURA +NAVY +NBA +NC +NE +NEC +NET +NETBANK +NETFLIX +NETWORK +NEUSTAR +NEW +NEWHOLLAND +NEWS +NEXT +NEXTDIRECT +NEXUS +NF +NFL +NG +NGO +NHK +NI +NICO +NIKE +NIKON +NINJA +NISSAN +NISSAY +NL +NO +NOKIA +NORTHWESTERNMUTUAL +NORTON +NOW +NOWRUZ +NOWTV +NP +NR +NRA +NRW +NTT +NU +NYC +NZ +OBI +OBSERVER +OFF +OFFICE +OKINAWA +OLAYAN +OLAYANGROUP +OLDNAVY +OLLO +OM +OMEGA +ONE +ONG +ONL +ONLINE +ONYOURSIDE +OOO +OPEN +ORACLE +ORANGE +ORG +ORGANIC +ORIGINS +OSAKA +OTSUKA +OTT +OVH +PA +PAGE +PANASONIC +PARIS +PARS +PARTNERS +PARTS +PARTY +PASSAGENS +PAY +PCCW +PE +PET +PF +PFIZER +PG +PH +PHARMACY +PHD +PHILIPS +PHONE +PHOTO +PHOTOGRAPHY +PHOTOS +PHYSIO +PIAGET +PICS +PICTET +PICTURES +PID +PIN +PING +PINK +PIONEER +PIZZA +PK +PL +PLACE +PLAY +PLAYSTATION +PLUMBING +PLUS +PM +PN +PNC +POHL +POKER +POLITIE +PORN +POST +PR +PRAMERICA +PRAXI +PRESS +PRIME +PRO +PROD +PRODUCTIONS +PROF +PROGRESSIVE +PROMO +PROPERTIES +PROPERTY +PROTECTION +PRU +PRUDENTIAL +PS +PT +PUB +PW +PWC +PY +QA +QPON +QUEBEC +QUEST +QVC +RACING +RADIO +RAID +RE +READ +REALESTATE +REALTOR +REALTY +RECIPES +RED +REDSTONE +REDUMBRELLA +REHAB +REISE +REISEN +REIT +RELIANCE +REN +RENT +RENTALS +REPAIR +REPORT +REPUBLICAN +REST +RESTAURANT +REVIEW +REVIEWS +REXROTH +RICH +RICHARDLI +RICOH +RIGHTATHOME +RIL +RIO +RIP +RMIT +RO +ROCHER +ROCKS +RODEO +ROGERS +ROOM +RS +RSVP +RU +RUGBY +RUHR +RUN +RW +RWE +RYUKYU +SA +SAARLAND +SAFE +SAFETY +SAKURA +SALE +SALON +SAMSCLUB +SAMSUNG +SANDVIK +SANDVIKCOROMANT +SANOFI +SAP +SARL +SAS +SAVE +SAXO +SB +SBI +SBS +SC +SCA +SCB +SCHAEFFLER +SCHMIDT +SCHOLARSHIPS +SCHOOL +SCHULE +SCHWARZ +SCIENCE +SCJOHNSON +SCOR +SCOT +SD +SE +SEARCH +SEAT +SECURE +SECURITY +SEEK +SELECT +SENER +SERVICES +SES +SEVEN +SEW +SEX +SEXY +SFR +SG +SH +SHANGRILA +SHARP +SHAW +SHELL +SHIA +SHIKSHA +SHOES +SHOP +SHOPPING +SHOUJI +SHOW +SHOWTIME +SHRIRAM +SI +SILK +SINA +SINGLES +SITE +SJ +SK +SKI +SKIN +SKY +SKYPE +SL +SLING +SM +SMART +SMILE +SN +SNCF +SO +SOCCER +SOCIAL +SOFTBANK +SOFTWARE +SOHU +SOLAR +SOLUTIONS +SONG +SONY +SOY +SPACE +SPORT +SPOT +SPREADBETTING +SR +SRL +SRT +SS +ST +STADA +STAPLES +STAR +STARHUB +STATEBANK +STATEFARM +STC +STCGROUP +STOCKHOLM +STORAGE +STORE +STREAM +STUDIO +STUDY +STYLE +SU +SUCKS +SUPPLIES +SUPPLY +SUPPORT +SURF +SURGERY +SUZUKI +SV +SWATCH +SWIFTCOVER +SWISS +SX +SY +SYDNEY +SYMANTEC +SYSTEMS +SZ +TAB +TAIPEI +TALK +TAOBAO +TARGET +TATAMOTORS +TATAR +TATTOO +TAX +TAXI +TC +TCI +TD +TDK +TEAM +TECH +TECHNOLOGY +TEL +TELEFONICA +TEMASEK +TENNIS +TEVA +TF +TG +TH +THD +THEATER +THEATRE +TIAA +TICKETS +TIENDA +TIFFANY +TIPS +TIRES +TIROL +TJ +TJMAXX +TJX +TK +TKMAXX +TL +TM +TMALL +TN +TO +TODAY +TOKYO +TOOLS +TOP +TORAY +TOSHIBA +TOTAL +TOURS +TOWN +TOYOTA +TOYS +TR +TRADE +TRADING +TRAINING +TRAVEL +TRAVELCHANNEL +TRAVELERS +TRAVELERSINSURANCE +TRUST +TRV +TT +TUBE +TUI +TUNES +TUSHU +TV +TVS +TW +TZ +UA +UBANK +UBS +UCONNECT +UG +UK +UNICOM +UNIVERSITY +UNO +UOL +UPS +US +UY +UZ +VA +VACATIONS +VANA +VANGUARD +VC +VE +VEGAS +VENTURES +VERISIGN +VERSICHERUNG +VET +VG +VI +VIAJES +VIDEO +VIG +VIKING +VILLAS +VIN +VIP +VIRGIN +VISA +VISION +VISTAPRINT +VIVA +VIVO +VLAANDEREN +VN +VODKA +VOLKSWAGEN +VOLVO +VOTE +VOTING +VOTO +VOYAGE +VU +VUELOS +WALES +WALMART +WALTER +WANG +WANGGOU +WARMAN +WATCH +WATCHES +WEATHER +WEATHERCHANNEL +WEBCAM +WEBER +WEBSITE +WED +WEDDING +WEIBO +WEIR +WF +WHOSWHO +WIEN +WIKI +WILLIAMHILL +WIN +WINDOWS +WINE +WINNERS +WME +WOLTERSKLUWER +WOODSIDE +WORK +WORKS +WORLD +WOW +WS +WTC +WTF +XBOX +XEROX +XFINITY +XIHUAN +XIN +XN--11B4C3D +XN--1CK2E1B +XN--1QQW23A +XN--2SCRJ9C +XN--30RR7Y +XN--3BST00M +XN--3DS443G +XN--3E0B707E +XN--3HCRJ9C +XN--3OQ18VL8PN36A +XN--3PXU8K +XN--42C2D9A +XN--45BR5CYL +XN--45BRJ9C +XN--45Q11C +XN--4GBRIM +XN--54B7FTA0CC +XN--55QW42G +XN--55QX5D +XN--5SU34J936BGSG +XN--5TZM5G +XN--6FRZ82G +XN--6QQ986B3XL +XN--80ADXHKS +XN--80AO21A +XN--80AQECDR1A +XN--80ASEHDB +XN--80ASWG +XN--8Y0A063A +XN--90A3AC +XN--90AE +XN--90AIS +XN--9DBQ2A +XN--9ET52U +XN--9KRT00A +XN--B4W605FERD +XN--BCK1B9A5DRE4C +XN--C1AVG +XN--C2BR7G +XN--CCK2B3B +XN--CG4BKI +XN--CLCHC0EA0B2G2A9GCD +XN--CZR694B +XN--CZRS0T +XN--CZRU2D +XN--D1ACJ3B +XN--D1ALF +XN--E1A4C +XN--ECKVDTC9D +XN--EFVY88H +XN--ESTV75G +XN--FCT429K +XN--FHBEI +XN--FIQ228C5HS +XN--FIQ64B +XN--FIQS8S +XN--FIQZ9S +XN--FJQ720A +XN--FLW351E +XN--FPCRJ9C3D +XN--FZC2C9E2C +XN--FZYS8D69UVGM +XN--G2XX48C +XN--GCKR3F0F +XN--GECRJ9C +XN--GK3AT1E +XN--H2BREG3EVE +XN--H2BRJ9C +XN--H2BRJ9C8C +XN--HXT814E +XN--I1B6B1A6A2E +XN--IMR513N +XN--IO0A7I +XN--J1AEF +XN--J1AMH +XN--J6W193G +XN--JLQ61U9W7B +XN--JVR189M +XN--KCRX77D1X4A +XN--KPRW13D +XN--KPRY57D +XN--KPU716F +XN--KPUT3I +XN--L1ACC +XN--LGBBAT1AD8J +XN--MGB9AWBF +XN--MGBA3A3EJT +XN--MGBA3A4F16A +XN--MGBA7C0BBN0A +XN--MGBAAKC7DVF +XN--MGBAAM7A8H +XN--MGBAB2BD +XN--MGBAH1A3HJKRD +XN--MGBAI9AZGQP6J +XN--MGBAYH7GPA +XN--MGBB9FBPOB +XN--MGBBH1A +XN--MGBBH1A71E +XN--MGBC0A9AZCG +XN--MGBCA7DZDO +XN--MGBERP4A5D4AR +XN--MGBGU82A +XN--MGBI4ECEXP +XN--MGBPL2FH +XN--MGBT3DHD +XN--MGBTX2B +XN--MGBX4CD0AB +XN--MIX891F +XN--MK1BU44C +XN--MXTQ1M +XN--NGBC5AZD +XN--NGBE9E0A +XN--NGBRX +XN--NODE +XN--NQV7F +XN--NQV7FS00EMA +XN--NYQY26A +XN--O3CW4H +XN--OGBPF8FL +XN--OTU796D +XN--P1ACF +XN--P1AI +XN--PBT977C +XN--PGBS0DH +XN--PSSY2U +XN--Q9JYB4C +XN--QCKA1PMC +XN--QXAM +XN--RHQV96G +XN--ROVU88B +XN--RVC1E0AM3E +XN--S9BRJ9C +XN--SES554G +XN--T60B56A +XN--TCKWE +XN--TIQ49XQYJ +XN--UNUP4Y +XN--VERMGENSBERATER-CTB +XN--VERMGENSBERATUNG-PWB +XN--VHQUV +XN--VUQ861B +XN--W4R85EL8FHU5DNRA +XN--W4RS40L +XN--WGBH1C +XN--WGBL6A +XN--XHQ521B +XN--XKC2AL3HYE2A +XN--XKC2DL3A5EE0H +XN--Y9A3AQ +XN--YFRO4I67O +XN--YGBI2AMMX +XN--ZFR164B +XXX +XYZ +YACHTS +YAHOO +YAMAXUN +YANDEX +YE +YODOBASHI +YOGA +YOKOHAMA +YOU +YOUTUBE +YT +YUN +ZA +ZAPPOS +ZARA +ZERO +ZIP +ZM +ZONE +ZUERICH +ZW \ No newline at end of file diff --git a/modules/scope/scope_checker.py b/modules/scope/scope_checker.py index 10789df..30503e2 100644 --- a/modules/scope/scope_checker.py +++ b/modules/scope/scope_checker.py @@ -1,5 +1,7 @@ from recon.core.module import BaseModule import ipaddress +import os +import socket class Module(BaseModule): @@ -7,22 +9,43 @@ class Module(BaseModule): 'name': 'Scope Checker', 'author': 'Ryan Hays', 'version': '1.0', - 'description': 'Compares the hosts table and removes hosts that are not found within the provided scope document. ' - 'Scope should be defined within the document either by single IP or in CIDR notation (eg. 192.168.1.0/24)', + 'description': 'Compares the hosts table and removes hosts that are not found within the provided scope ' + 'document. Scope should be defined within the document by one entry per line either by single ' + 'domain, IP, or CIDR notation (eg. google.com, 192.168.1.1, or 192.168.1.0/24)', 'query': 'SELECT DISTINCT ip_address FROM hosts WHERE ip_address IS NOT NULL', 'options': ( ('filename', None, True, 'path and filename for scope document'), ), + 'files': ( + 'tld_list.txt', + ), } def module_run(self, hosts): + tld_list = [] + with open(os.path.join(self.data_path, 'tld_list.txt'), 'r') as fp: + for tld in fp: + if '#' != tld[0]: + tld_list.append(tld.rstrip().lower()) + cidr_dict = {} with open(self.options['filename'], 'rt') as fp: for cidr in fp: + cidr = cidr.rstrip() + try: - cidr_dict[ipaddress.ip_network(cidr.rstrip())] = cidr.rstrip() + cidr_dict[ipaddress.ip_network(cidr)] = cidr except ValueError: - self.error(f"Invalid CIDR notation for: {cidr.rstrip()}") + if cidr.split('.')[-1].lower() in tld_list: + try: + ips = socket.gethostbyname_ex(cidr)[2] + for ip in ips: + cidr_dict[ipaddress.ip_network(ip)] = cidr + except socket.gaierror: + self.error(f"Couldn't lookup IP Address for: {cidr}") + + else: + self.error(f"Invalid scope item: {cidr}") cnt = 0 for host in hosts: From 54a34d6c204793eecf3fd9275af2f6664e4c35bf Mon Sep 17 00:00:00 2001 From: Ryan Hays Date: Tue, 4 Jun 2019 19:09:46 -0400 Subject: [PATCH 5/7] Updated the SQL Query and added a removal from the ports table. --- modules/scope/scope_checker.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/modules/scope/scope_checker.py b/modules/scope/scope_checker.py index 30503e2..a07c3ac 100644 --- a/modules/scope/scope_checker.py +++ b/modules/scope/scope_checker.py @@ -9,10 +9,11 @@ class Module(BaseModule): 'name': 'Scope Checker', 'author': 'Ryan Hays', 'version': '1.0', - 'description': 'Compares the hosts table and removes hosts that are not found within the provided scope ' - 'document. Scope should be defined within the document by one entry per line either by single ' - 'domain, IP, or CIDR notation (eg. google.com, 192.168.1.1, or 192.168.1.0/24)', - 'query': 'SELECT DISTINCT ip_address FROM hosts WHERE ip_address IS NOT NULL', + 'description': 'Compares the hosts and ports table and removes hosts that are not found within the provided ' + 'scope document. Scope should be defined within the document by one entry per line either by ' + 'single domain, IP, or CIDR notation (eg. google.com, 192.168.1.1, or 192.168.1.0/24)', + 'query': 'SELECT DISTINCT ip_address FROM hosts WHERE ip_address IS NOT NULL ' + 'UNION SELECT ip_address FROM ports WHERE ip_address IS NOT NULL', 'options': ( ('filename', None, True, 'path and filename for scope document'), ), @@ -60,5 +61,6 @@ def module_run(self, hosts): cnt += 1 self.alert(f"{str(ip)} not found in scope document removing from hosts table.") self.query('DELETE FROM hosts WHERE ip_address=?', (str(ip),)) + self.query('DELETE FROM ports WHERE ip_address=?', (str(ip),)) self.output(f"Remove {cnt} out of scope hosts from the hosts table.") From 3fe1c4c03303c52ea44615a71630bff3bd168d23 Mon Sep 17 00:00:00 2001 From: Ryan Hays Date: Tue, 4 Jun 2019 20:28:50 -0400 Subject: [PATCH 6/7] Changed final output message --- modules/scope/scope_checker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/scope/scope_checker.py b/modules/scope/scope_checker.py index a07c3ac..7865db3 100644 --- a/modules/scope/scope_checker.py +++ b/modules/scope/scope_checker.py @@ -63,4 +63,4 @@ def module_run(self, hosts): self.query('DELETE FROM hosts WHERE ip_address=?', (str(ip),)) self.query('DELETE FROM ports WHERE ip_address=?', (str(ip),)) - self.output(f"Remove {cnt} out of scope hosts from the hosts table.") + self.output(f"Remove {cnt} out of scope hosts.") From f209f7c1c36558ef881ae39af5bb4f002241b5fc Mon Sep 17 00:00:00 2001 From: Ryan Hays Date: Wed, 5 Jun 2019 15:52:09 -0400 Subject: [PATCH 7/7] Added the strict=False to the ip_network call to catch scoped items that may not be written to standard byusing a host when labeling the CIDR. --- modules/scope/scope_checker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/scope/scope_checker.py b/modules/scope/scope_checker.py index 7865db3..525fa68 100644 --- a/modules/scope/scope_checker.py +++ b/modules/scope/scope_checker.py @@ -35,7 +35,7 @@ def module_run(self, hosts): cidr = cidr.rstrip() try: - cidr_dict[ipaddress.ip_network(cidr)] = cidr + cidr_dict[ipaddress.ip_network(cidr, False)] = cidr except ValueError: if cidr.split('.')[-1].lower() in tld_list: try: