From 4d1e0610b1313ecea8e6ff983133ef52d94079b5 Mon Sep 17 00:00:00 2001 From: lovelyjuice Date: Tue, 10 Sep 2024 12:17:13 +0800 Subject: [PATCH 1/2] =?UTF-8?q?1.=20=E5=A2=9E=E5=8A=A0=E7=A9=BA=E9=97=B4?= =?UTF-8?q?=E6=B5=8B=E7=BB=98=E5=BC=95=E6=93=8E=E6=9C=80=E5=A4=A7=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=95=B0=E9=87=8F=E9=99=90=E5=88=B6=EF=BC=8C=E9=98=B2?= =?UTF-8?q?=E6=AD=A2=E6=B3=9B=E8=A7=A3=E6=9E=90=E5=92=8CCDN=E6=B5=AA?= =?UTF-8?q?=E8=B4=B9=E7=A7=AF=E5=88=86=E3=80=82=E5=AF=B9=20fofa,=20hunter,?= =?UTF-8?q?=20quake,=20zoomeye=20=E7=94=9F=E6=95=88=202.=20=E5=85=81?= =?UTF-8?q?=E8=AE=B8=E7=94=A8=E6=88=B7=E4=B8=8D=E4=BD=BF=E7=94=A8=20tldext?= =?UTF-8?q?ract=20=E6=8F=90=E5=8F=96=E4=B8=BB=E5=9F=9F=E5=90=8D=EF=BC=8C?= =?UTF-8?q?=E9=98=B2=E6=AD=A2=E6=94=B6=E9=9B=86=E5=88=86=E5=85=AC=E5=8F=B8?= =?UTF-8?q?=E5=9F=9F=E5=90=8D=E6=97=B6=E9=94=99=E8=AF=AF=E6=94=B6=E9=9B=86?= =?UTF-8?q?=E5=88=B0=E4=B8=BB=E5=85=AC=E5=8F=B8=E7=9A=84=E5=9F=9F=E5=90=8D?= =?UTF-8?q?=203.=20=E5=8D=87=E7=BA=A7exrex=E7=89=88=E6=9C=AC=EF=BC=8C?= =?UTF-8?q?=E5=85=BC=E5=AE=B9python3.12=204.=20=E4=BF=AE=E5=A4=8D`No=20mod?= =?UTF-8?q?ule=20named=20'distutils.util'`=E7=9A=84bug=EF=BC=8C=E5=85=B3?= =?UTF-8?q?=E8=81=94issue:=20https://github.com/shmilylty/OneForAll/issues?= =?UTF-8?q?/394?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/domain.py | 2 ++ config/setting.py | 8 +++++++- modules/search/fofa_api.py | 6 +++--- modules/search/hunter_api.py | 2 +- modules/search/quake_api.py | 2 +- modules/search/zoomeye_api.py | 2 +- requirements.txt | 3 ++- 7 files changed, 17 insertions(+), 8 deletions(-) diff --git a/common/domain.py b/common/domain.py index cd1a68b4..3e57cea3 100644 --- a/common/domain.py +++ b/common/domain.py @@ -55,6 +55,8 @@ def registered(self): :return: registered domain result """ + if not settings.use_tld_extract: + return self.string result = self.extract() if result: return result.registered_domain diff --git a/config/setting.py b/config/setting.py index 58967016..643ffa00 100644 --- a/config/setting.py +++ b/config/setting.py @@ -40,7 +40,7 @@ # 爆破所使用的字典路径(默认None则使用data/subdomains.txt，自定义字典请使用绝对路径) brute_wordlist_path = None use_china_nameservers = True # 使用中国域名服务器 如果你所在网络不在中国则建议设置False -enable_recursive_brute = False # 是否使用递归爆破(默认False) +enable_recursive_brute = True # 是否使用递归爆破(默认False) brute_recursive_depth = 2 # 递归爆破深度(默认2层) # 爆破下一层子域所使用的字典路径(默认None则使用data/subnames_next.txt，自定义字典请使用绝对路径) recursive_nextlist_path = None @@ -100,3 +100,9 @@ # 搜索模块设置 enable_recursive_search = False # 递归搜索子域 search_recursive_times = 2 # 递归搜索层数 + +# 网络空间测绘引擎设置 +cam_records_maximum_per_domain = 1000 # 对于单个主域名，在测绘引擎中的最多查询多少条记录，防止泛解析和CDN浪费积分，对 fofa, hunter, quake, zoomeye 生效，最低为100 + +# 是否从输入的数据中使用tldextract提取主域名。若设为 False，OneForAll会直接将输入的域名作为主域名，比如北京分公司的域名 beijing.10086.com 就不会被解析成母公司的域名 10086.com +use_tld_extract = True \ No newline at end of file diff --git a/modules/search/fofa_api.py b/modules/search/fofa_api.py index f2dc2086..86b59b81 100644 --- a/modules/search/fofa_api.py +++ b/modules/search/fofa_api.py @@ -23,7 +23,7 @@ def search(self): self.page_num = 1 subdomain_encode = f'domain="{self.domain}"'.encode('utf-8') query_data = base64.b64encode(subdomain_encode) - while True: + while 100 * self.page_num < settings.cam_records_maximum_per_domain: time.sleep(self.delay) self.header = self.get_header() self.proxy = self.get_proxy(self.source) @@ -32,7 +32,7 @@ def search(self): 'qbase64': query_data, 'page': self.page_num, 'full': 'true', - 'size': 1000} + 'size': min(1000, settings.cam_records_maximum_per_domain)} resp = self.get(self.addr, query) if not resp: return @@ -42,7 +42,7 @@ def search(self): break self.subdomains.update(subdomains) size = resp_json.get('size') - if size < 1000: + if size < min(1000, settings.cam_records_maximum_per_domain): break self.page_num += 1 diff --git a/modules/search/hunter_api.py b/modules/search/hunter_api.py index cc1d237a..39436512 100644 --- a/modules/search/hunter_api.py +++ b/modules/search/hunter_api.py @@ -22,7 +22,7 @@ def search(self): self.page_num = 1 subdomain_encode = f'domain_suffix="{self.domain}"'.encode('utf-8') query_data = base64.b64encode(subdomain_encode) - while True: + while 100 * self.page_num < settings.cam_records_maximum_per_domain: time.sleep(self.delay) self.header = self.get_header() self.proxy = self.get_proxy(self.source) diff --git a/modules/search/quake_api.py b/modules/search/quake_api.py index 2e42b8d4..a3c554a1 100644 --- a/modules/search/quake_api.py +++ b/modules/search/quake_api.py @@ -20,7 +20,7 @@ def search(self): """ self.per_page_num = 100 self.page_num = 0 - while True: + while self.per_page_num * self.page_num < settings.cam_records_maximum_per_domain: time.sleep(self.delay) self.header = self.get_header() self.header.update({'Content-Type': 'application/json'}) diff --git a/modules/search/zoomeye_api.py b/modules/search/zoomeye_api.py index 9afff49b..e27bba04 100644 --- a/modules/search/zoomeye_api.py +++ b/modules/search/zoomeye_api.py @@ -19,7 +19,7 @@ def search(self): """ self.per_page_num = 30 self.page_num = 1 - while True: + while self.per_page_num * self.page_num < settings.cam_records_maximum_per_domain: time.sleep(self.delay) self.header = self.get_header() self.header.update({'API-KEY': self.key}) diff --git a/requirements.txt b/requirements.txt index 42460b78..8f195ea6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ certifi==2022.06.15 chardet==5.0.0 colorama==0.4.4 dnspython==2.2.1 -exrex==0.10.5 +exrex==0.11.0 fire==0.4.0 future==0.18.2 idna==3.3 @@ -20,3 +20,4 @@ tqdm==4.64.0 treelib==1.6.1 urllib3==1.26.9 win32-setctime==1.1.0 +setuptools From 202f0c3aedf3aca8df36422b73cd4e04ca574d12 Mon Sep 17 00:00:00 2001 From: lovelyjuice Date: Tue, 10 Sep 2024 13:14:20 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E4=B8=8A=E4=B8=AAcommit=E8=AF=AF=E5=BC=80?= =?UTF-8?q?=E5=90=AF=E4=BA=86=E9=80=92=E5=BD=92=E7=88=86=E7=A0=B4=EF=BC=8C?= =?UTF-8?q?=E7=8E=B0=E5=9C=A8=E6=81=A2=E5=A4=8D=E9=BB=98=E8=AE=A4=E9=85=8D?= =?UTF-8?q?=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/setting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/setting.py b/config/setting.py index 643ffa00..4f8070d6 100644 --- a/config/setting.py +++ b/config/setting.py @@ -40,7 +40,7 @@ # 爆破所使用的字典路径(默认None则使用data/subdomains.txt，自定义字典请使用绝对路径) brute_wordlist_path = None use_china_nameservers = True # 使用中国域名服务器 如果你所在网络不在中国则建议设置False -enable_recursive_brute = True # 是否使用递归爆破(默认False) +enable_recursive_brute = False # 是否使用递归爆破(默认False) brute_recursive_depth = 2 # 递归爆破深度(默认2层) # 爆破下一层子域所使用的字典路径(默认None则使用data/subnames_next.txt，自定义字典请使用绝对路径) recursive_nextlist_path = None