Merge pull request #1058 from ocervell/fix-1006

fix: ffuf ANSI code processing preventing task to finish

web/reNgine/definitions.py

@@ -398,27 +398,27 @@
 
 # Default Dir File Fuzz Params
 DEFAULT_DIR_FILE_FUZZ_EXTENSIONS =  [
-    'html',
-    'php',
-    'git',
-    'yaml',
-    'conf',
-    'cnf',
-    'config',
-    'gz',
-    'env',
-    'log',
-    'db',
-    'mysql',
-    'bak',
-    'asp',
-    'aspx',
-    'txt',
-    'conf',
-    'sql',
-    'json',
-    'yml',
-    'pdf',
+    '.html',
+    '.php',
+    '.git',
+    '.yaml',
+    '.conf',
+    '.cnf',
+    '.config',
+    '.gz',
+    '.env',
+    '.log',
+    '.db',
+    '.mysql',
+    '.bak',
+    '.asp',
+    '.aspx',
+    '.txt',
+    '.conf',
+    '.sql',
+    '.json',
+    '.yml',
+    '.pdf',
 ]
 
 # Roles and Permissions

web/reNgine/tasks.py

@@ -1583,6 +1583,8 @@ def dir_file_fuzz(self, ctx={}, description=None):
 	enable_http_crawl = config.get(ENABLE_HTTP_CRAWL, DEFAULT_ENABLE_HTTP_CRAWL)
 	rate_limit = config.get(RATE_LIMIT) or self.yaml_configuration.get(RATE_LIMIT, DEFAULT_RATE_LIMIT)
 	extensions = config.get(EXTENSIONS, DEFAULT_DIR_FILE_FUZZ_EXTENSIONS)
+	# prepend . on extensions
+	extensions = [ext if ext.startswith('.') else '.' + ext for ext in extensions]
 	extensions_str = ','.join(map(str, extensions))
 	follow_redirect = config.get(FOLLOW_REDIRECT, FFUF_DEFAULT_FOLLOW_REDIRECT)
 	max_time = config.get(MAX_TIME, 0)
@@ -3184,7 +3186,7 @@ def parse_nmap_results(xml_file, output_file=None):
 		if hostnames_dict:
 			# Ensure that hostnames['hostname'] is a list for consistency
 			hostnames_list = hostnames_dict['hostname'] if isinstance(hostnames_dict['hostname'], list) else [hostnames_dict['hostname']]
-			
+
 			# Extract all the @name values from the list of dictionaries
 			hostnames = [entry.get('@name') for entry in hostnames_list]
 		else:
@@ -3519,7 +3521,7 @@ def record_exists(model, data, exclude_keys=[]):
 	Returns:
 		bool: True if the record exists, False otherwise.
 	"""
-	
+
 	# Extract the keys that will be used for the lookup
 	lookup_fields = {key: data[key] for key in data if key not in exclude_keys}
 
@@ -4131,15 +4133,21 @@ def stream_command(cmd, cwd=None, shell=False, history_file=None, encoding='utf-
 	process = subprocess.Popen(
 		command,
 		stdout=subprocess.PIPE,
-		stderr=subprocess.PIPE,
+		stderr=subprocess.STDOUT,
+		universal_newlines=True,
 		shell=shell)
 
 	# Log the output in real-time to the database
 	output = ""
 
 	# Process the output
-	for line in iter(lambda: process.stdout.readline() or process.stderr.readline(), b''):
-		line = re.sub(r'\x1b[^m]*m', '', line.decode('utf-8').strip())
+	for line in iter(lambda: process.stdout.readline(), b''):
+		if not line:
+			break
+		line = line.strip()
+		ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])')
+		line = ansi_escape.sub('', line)
+		line = line.replace('\\x0d\\x0a', '\n')
 		if trunc_char and line.endswith(trunc_char):
 			line = line[:-1]
 		item = line