From e5bf0c4b7831bb4d4d0c94926bd2c845bda727b6 Mon Sep 17 00:00:00 2001 From: Raynald Date: Wed, 22 Nov 2023 03:32:20 +0100 Subject: [PATCH 1/9] Add check & exception catch while saving endpoint --- web/reNgine/tasks.py | 79 ++++++++++++++++++++++++-------------------- 1 file changed, 43 insertions(+), 36 deletions(-) diff --git a/web/reNgine/tasks.py b/web/reNgine/tasks.py index 6e6e67d04..50d64d9c2 100644 --- a/web/reNgine/tasks.py +++ b/web/reNgine/tasks.py @@ -1666,37 +1666,38 @@ def dir_file_fuzz(self, ctx={}, description=None): logger.error(f'FUZZ not found for "{url}"') continue endpoint, created = save_endpoint(url, crawl=False, ctx=ctx) - # endpoint.is_default = False - endpoint.http_status = status - endpoint.content_length = length - endpoint.response_time = duration / 1000000000 - endpoint.save() - if created: - urls.append(endpoint.http_url) - endpoint.status = status - endpoint.content_type = content_type - endpoint.content_length = length - dfile, created = DirectoryFile.objects.get_or_create( - name=name, - length=length, - words=words, - lines=lines, - content_type=content_type, - url=url) - dfile.http_status = status - dfile.save() - # if created: - # logger.warning(f'Found new directory or file {url}') - dirscan.directory_files.add(dfile) - dirscan.save() - - if self.subscan: - dirscan.dir_subscan_ids.add(self.subscan) - - subdomain_name = get_subdomain_from_url(endpoint.http_url) - subdomain = Subdomain.objects.get(name=subdomain_name, scan_history=self.scan) - subdomain.directories.add(dirscan) - subdomain.save() + if endpoint: + # endpoint.is_default = False + endpoint.http_status = status + endpoint.content_length = length + endpoint.response_time = duration / 1000000000 + endpoint.save() + if created: + urls.append(endpoint.http_url) + endpoint.status = status + endpoint.content_type = content_type + endpoint.content_length = length + dfile, created = DirectoryFile.objects.get_or_create( + name=name, + length=length, + words=words, + lines=lines, + content_type=content_type, + url=url) + dfile.http_status = status + dfile.save() + # if created: + # logger.warning(f'Found new directory or file {url}') + dirscan.directory_files.add(dfile) + dirscan.save() + + if self.subscan: + dirscan.dir_subscan_ids.add(self.subscan) + + subdomain_name = get_subdomain_from_url(endpoint.http_url) + subdomain = Subdomain.objects.get(name=subdomain_name, scan_history=self.scan) + subdomain.directories.add(dirscan) + subdomain.save() # Crawl discovered URLs if enable_http_crawl: @@ -4495,11 +4496,17 @@ def save_endpoint( if not validators.url(http_url): return None, False http_url = sanitize_url(http_url) - endpoint, created = EndPoint.objects.get_or_create( - scan_history=scan, - target_domain=domain, - http_url=http_url, - **endpoint_data) + if os.environ.get('DEBUG') + logger.warning(f'================== URL : '+http_url+' ==================') + try: + endpoint, created = EndPoint.objects.get_or_create( + scan_history=scan, + target_domain=domain, + http_url=http_url, + **endpoint_data) + except Exception as e: + logger.error(e) + return None, False if created: endpoint.is_default = is_default From 7b4ee42cbb5a20498bdcf52fe78e61a8e657aaf6 Mon Sep 17 00:00:00 2001 From: Raynald Date: Wed, 22 Nov 2023 03:35:40 +0100 Subject: [PATCH 2/9] Resolve syntax error --- web/reNgine/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/reNgine/tasks.py b/web/reNgine/tasks.py index 50d64d9c2..9b3925fb6 100644 --- a/web/reNgine/tasks.py +++ b/web/reNgine/tasks.py @@ -4496,7 +4496,7 @@ def save_endpoint( if not validators.url(http_url): return None, False http_url = sanitize_url(http_url) - if os.environ.get('DEBUG') + if os.environ.get('DEBUG'): logger.warning(f'================== URL : '+http_url+' ==================') try: endpoint, created = EndPoint.objects.get_or_create( From fb28da7d48978ccf027fb2413ec90b52ef8ea9db Mon Sep 17 00:00:00 2001 From: Raynald Date: Wed, 22 Nov 2023 03:38:34 +0100 Subject: [PATCH 3/9] Reduce debug equal sign --- web/reNgine/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/reNgine/tasks.py b/web/reNgine/tasks.py index 9b3925fb6..f0fff9908 100644 --- a/web/reNgine/tasks.py +++ b/web/reNgine/tasks.py @@ -4497,7 +4497,7 @@ def save_endpoint( return None, False http_url = sanitize_url(http_url) if os.environ.get('DEBUG'): - logger.warning(f'================== URL : '+http_url+' ==================') + logger.warning(f'================== URL : '+http_url) try: endpoint, created = EndPoint.objects.get_or_create( scan_history=scan, From 39c68718058a65e27caf364fc89ee0e4e18a476f Mon Sep 17 00:00:00 2001 From: Raynald Date: Wed, 22 Nov 2023 03:42:36 +0100 Subject: [PATCH 4/9] Move debug log on error only --- web/reNgine/tasks.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/web/reNgine/tasks.py b/web/reNgine/tasks.py index f0fff9908..5d850f6e3 100644 --- a/web/reNgine/tasks.py +++ b/web/reNgine/tasks.py @@ -4496,8 +4496,6 @@ def save_endpoint( if not validators.url(http_url): return None, False http_url = sanitize_url(http_url) - if os.environ.get('DEBUG'): - logger.warning(f'================== URL : '+http_url) try: endpoint, created = EndPoint.objects.get_or_create( scan_history=scan, @@ -4505,7 +4503,7 @@ def save_endpoint( http_url=http_url, **endpoint_data) except Exception as e: - logger.error(e) + logger.error(f'/!\ - URL : '+http_url+', exception: '+e) return None, False if created: From 914929feac27d306a570c4bd05e5d19e0bea4734 Mon Sep 17 00:00:00 2001 From: Raynald Date: Wed, 22 Nov 2023 03:56:21 +0100 Subject: [PATCH 5/9] cast exception text to string --- web/reNgine/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/reNgine/tasks.py b/web/reNgine/tasks.py index 5d850f6e3..881e8c11b 100644 --- a/web/reNgine/tasks.py +++ b/web/reNgine/tasks.py @@ -4503,7 +4503,7 @@ def save_endpoint( http_url=http_url, **endpoint_data) except Exception as e: - logger.error(f'/!\ - URL : '+http_url+', exception: '+e) + logger.error(f'/!\ - URL : '+http_url+', exception: '+str(e)) return None, False if created: From 5c7bb8d07a3780d810edcc61d6ec7271bf547bba Mon Sep 17 00:00:00 2001 From: Raynald Date: Wed, 22 Nov 2023 18:09:19 +0100 Subject: [PATCH 6/9] Prevent empty endpoint by getting first endpoint --- web/reNgine/tasks.py | 84 +++++++++++++++++++++++++------------------- 1 file changed, 47 insertions(+), 37 deletions(-) diff --git a/web/reNgine/tasks.py b/web/reNgine/tasks.py index 881e8c11b..1c767c2c7 100644 --- a/web/reNgine/tasks.py +++ b/web/reNgine/tasks.py @@ -1666,38 +1666,37 @@ def dir_file_fuzz(self, ctx={}, description=None): logger.error(f'FUZZ not found for "{url}"') continue endpoint, created = save_endpoint(url, crawl=False, ctx=ctx) - if endpoint: - # endpoint.is_default = False - endpoint.http_status = status - endpoint.content_length = length - endpoint.response_time = duration / 1000000000 - endpoint.save() - if created: - urls.append(endpoint.http_url) - endpoint.status = status - endpoint.content_type = content_type - endpoint.content_length = length - dfile, created = DirectoryFile.objects.get_or_create( - name=name, - length=length, - words=words, - lines=lines, - content_type=content_type, - url=url) - dfile.http_status = status - dfile.save() - # if created: - # logger.warning(f'Found new directory or file {url}') - dirscan.directory_files.add(dfile) - dirscan.save() - - if self.subscan: - dirscan.dir_subscan_ids.add(self.subscan) - - subdomain_name = get_subdomain_from_url(endpoint.http_url) - subdomain = Subdomain.objects.get(name=subdomain_name, scan_history=self.scan) - subdomain.directories.add(dirscan) - subdomain.save() + # endpoint.is_default = False + endpoint.http_status = status + endpoint.content_length = length + endpoint.response_time = duration / 1000000000 + endpoint.save() + if created: + urls.append(endpoint.http_url) + endpoint.status = status + endpoint.content_type = content_type + endpoint.content_length = length + dfile, created = DirectoryFile.objects.get_or_create( + name=name, + length=length, + words=words, + lines=lines, + content_type=content_type, + url=url) + dfile.http_status = status + dfile.save() + # if created: + # logger.warning(f'Found new directory or file {url}') + dirscan.directory_files.add(dfile) + dirscan.save() + + if self.subscan: + dirscan.dir_subscan_ids.add(self.subscan) + + subdomain_name = get_subdomain_from_url(endpoint.http_url) + subdomain = Subdomain.objects.get(name=subdomain_name, scan_history=self.scan) + subdomain.directories.add(dirscan) + subdomain.save() # Crawl discovered URLs if enable_http_crawl: @@ -4496,15 +4495,26 @@ def save_endpoint( if not validators.url(http_url): return None, False http_url = sanitize_url(http_url) - try: + + # Try to get the first matching record (prevent duplicate error) + endpoints = EndPoint.objects.filter( + scan_history=scan, + target_domain=domain, + http_url=http_url, + **endpoint_data + ) + + if endpoints.exists(): + endpoint = endpoints.first() + created = False + else: + # No existing record, create a new one endpoint, created = EndPoint.objects.get_or_create( scan_history=scan, target_domain=domain, http_url=http_url, - **endpoint_data) - except Exception as e: - logger.error(f'/!\ - URL : '+http_url+', exception: '+str(e)) - return None, False + **endpoint_data + ) if created: endpoint.is_default = is_default From e91ca6b974a37e4c77d54ed1a6c548f18fefe9a1 Mon Sep 17 00:00:00 2001 From: Raynald Date: Wed, 22 Nov 2023 18:14:51 +0100 Subject: [PATCH 7/9] Change method to create only to reduce query --- web/reNgine/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/reNgine/tasks.py b/web/reNgine/tasks.py index 1c767c2c7..6994ad299 100644 --- a/web/reNgine/tasks.py +++ b/web/reNgine/tasks.py @@ -4509,7 +4509,7 @@ def save_endpoint( created = False else: # No existing record, create a new one - endpoint, created = EndPoint.objects.get_or_create( + endpoint, created = EndPoint.objects.create( scan_history=scan, target_domain=domain, http_url=http_url, From d12d69ba2afea2e5ca6a7db9b7501b7b843cf4c2 Mon Sep 17 00:00:00 2001 From: Raynald Date: Thu, 23 Nov 2023 13:49:57 +0100 Subject: [PATCH 8/9] Remove bad tuple var and set create flag to true --- web/reNgine/tasks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/web/reNgine/tasks.py b/web/reNgine/tasks.py index 6994ad299..e7b66ee95 100644 --- a/web/reNgine/tasks.py +++ b/web/reNgine/tasks.py @@ -4509,12 +4509,13 @@ def save_endpoint( created = False else: # No existing record, create a new one - endpoint, created = EndPoint.objects.create( + endpoint = EndPoint.objects.create( scan_history=scan, target_domain=domain, http_url=http_url, **endpoint_data ) + created = True if created: endpoint.is_default = is_default From caa0f2d2907106294616d2a72791f369ec210399 Mon Sep 17 00:00:00 2001 From: Raynald Date: Sat, 25 Nov 2023 17:33:27 +0100 Subject: [PATCH 9/9] Remove comment --- web/reNgine/tasks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/web/reNgine/tasks.py b/web/reNgine/tasks.py index e7b66ee95..3a17493fe 100644 --- a/web/reNgine/tasks.py +++ b/web/reNgine/tasks.py @@ -1666,7 +1666,6 @@ def dir_file_fuzz(self, ctx={}, description=None): logger.error(f'FUZZ not found for "{url}"') continue endpoint, created = save_endpoint(url, crawl=False, ctx=ctx) - # endpoint.is_default = False endpoint.http_status = status endpoint.content_length = length endpoint.response_time = duration / 1000000000