From 618f749bf9405639647e311d82a9f1af3673fdd3 Mon Sep 17 00:00:00 2001 From: Sahil Date: Mon, 22 Apr 2024 11:22:43 +0530 Subject: [PATCH 01/24] added exception condition for /matching_record --- modal2.py | 121 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 63 insertions(+), 58 deletions(-) diff --git a/modal2.py b/modal2.py index 4481af8..4b77c15 100644 --- a/modal2.py +++ b/modal2.py @@ -182,68 +182,73 @@ async def get_data(request: ModalRequestSchema, tenant: str = Header(...)): @app.post("/reconsrvc/matching_records") async def get_matching_records(request: MatchingRecordRequestSchema, tenant: str = Header(...)): # Log the start of the operation with tenant information - logging.info("matching_record API satrted") + logging.info("matching_record API started") logging.info(f"Starting 'get_matching_records' for tenant: {tenant} at {datetime.datetime.now()}") - # Initialize the collection references - reconciliationPull_collection = reconciliationPull(tenant) # Presumably initializes the collection based on tenant - shReconReport_collection = get_collection(tenant, 'shReconReport') # Gets the 'shReconReport' collection for the tenant - - # Step 2: Create a pipeline to aggregate data for the specified appId and calculate the total number of unique reconciliationIds - pipeline_for_count = [ - {"$match": {"appId": request.appId, "createdAt": {"$exists": True}}}, - {"$group": {"_id": "$reconciliationId"}}, - {"$group": {"_id": None, "count": {"$sum": 1}}} - ] - - # Run the aggregation pipeline for counting documents - count_Result = shReconReport_collection.aggregate(pipeline_for_count) - count = 0 - for doc in count_Result: - count = doc['count'] # Extract the total count of unique documents - logging.debug(f"Total records found: {count}") - - # Step 3: Define the pipeline to retrieve the latest document per reconciliationId and paginate the results - pipeline = [ - {"$match": {"appId": request.appId, "createdAt": {"$exists": True}}}, - {"$group": {"_id": "$reconciliationId", "maxStartTime": {"$max": "$createdAt"}, "latestDocument": {"$last": "$$ROOT"}}}, - {"$sort": {"_id": 1}}, - {"$skip": request.pageNumber * request.pageSize}, - {"$limit": request.pageSize}, - {"$replaceRoot": {"newRoot": "$latestDocument"}} - ] - - # Execute the main data retrieval pipeline - result = shReconReport_collection.aggregate(pipeline) - response = [] + try: + # Initialize the collection references + reconciliationPull_collection = reconciliationPull(tenant) # Presumably initializes the collection based on tenant + shReconReport_collection = get_collection(tenant, 'shReconReport') # Gets the 'shReconReport' collection for the tenant + logging.info("Collections successfully initialized.") + + # Step 2: Create a pipeline to aggregate data for the specified appId and calculate the total number of unique reconciliationIds + pipeline_for_count = [ + {"$match": {"appId": request.appId, "createdAt": {"$exists": True}}}, + {"$group": {"_id": "$reconciliationId"}}, + {"$group": {"_id": None, "count": {"$sum": 1}}} + ] + + # Run the aggregation pipeline for counting documents + count_Result = shReconReport_collection.aggregate(pipeline_for_count) + count = 0 + for doc in count_Result: + count = doc['count'] # Extract the total count of unique documents + logging.debug(f"Total records found: {count}") + + # Step 3: Define the pipeline to retrieve the latest document per reconciliationId and paginate the results + pipeline = [ + {"$match": {"appId": request.appId, "createdAt": {"$exists": True}}}, + {"$group": {"_id": "$reconciliationId", "maxStartTime": {"$max": "$createdAt"}, "latestDocument": {"$last": "$$ROOT"}}}, + {"$sort": {"_id": 1}}, + {"$skip": request.pageNumber * request.pageSize}, + {"$limit": request.pageSize}, + {"$replaceRoot": {"newRoot": "$latestDocument"}} + ] + + # Execute the main data retrieval pipeline + result = shReconReport_collection.aggregate(pipeline) + response = [] + + # Iterate through the results, fetch additional data, and prepare the response + for doc in result: + recon_id = doc['reconciliationId'] + last_date = doc['createdAt'] + datetime.timedelta(hours=5, minutes=30) # Adjust time based on timezone + logging.debug(f"Processing document for reconciliationId: {recon_id}") + + recon_data = reconciliationPull_collection.find_one({'_id': ObjectId(recon_id)}) + if recon_data: + response.append({ + "Id": str(doc['reconReportMetadata']), + "name": recon_data.get('name'), + "mode": recon_data.get('reconMode'), + "type": recon_data.get('type'), + "recon_type": recon_data.get('reconType'), + "last_run_datetime": last_date + }) + + # Construct the return value + return_val = { + 'success': True, + 'data': {'content': response}, + 'totalRecords': count, + 'totalPages': math.ceil(count / request.pageSize) + } - # Iterate through the results, fetch additional data, and prepare the response - for doc in result: - recon_id = doc['reconciliationId'] - last_date = doc['createdAt'] + datetime.timedelta(hours=5, minutes=30) # Adjust time based on timezone - logging.debug(f"Processing document for reconciliationId: {recon_id}") - - recon_data = reconciliationPull_collection.find_one({'_id': ObjectId(recon_id)}) - if recon_data: - response.append({ - "Id": str(doc['reconReportMetadata']), - "name": recon_data.get('name'), - "mode": recon_data.get('reconMode'), - "type": recon_data.get('type'), - "recon_type": recon_data.get('reconType'), - "last_run_datetime": last_date - }) - - # Construct the return value - return_val = { - 'success': True, - 'data': {'content': response}, - 'totalRecords': count, - 'totalPages': math.ceil(count / request.pageSize) - } + # Log the successful completion of the operation + logging.info(f"Ending 'get_matching_records' for tenant: {tenant} at {datetime.datetime.now()}") - # Log the successful completion of the operation - logging.info(f"Ending 'get_matching_records' for tenant: {tenant} at {datetime.datetime.now()}") + except Exception as e: + logging.error(f"An error occurred: {str(e)}") return return_val From ced830fe8203032d17a22a68ab16e83e8344e4a5 Mon Sep 17 00:00:00 2001 From: Sahil Date: Mon, 22 Apr 2024 12:05:13 +0530 Subject: [PATCH 02/24] added logging statements in connection.py --- database/connection.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/database/connection.py b/database/connection.py index 04e5688..4b8f8f6 100644 --- a/database/connection.py +++ b/database/connection.py @@ -2,6 +2,7 @@ from pymongo import MongoClient from config.loader import Configuration import platform +import logging def load_configuration(): if platform.system() == 'Windows': @@ -21,4 +22,6 @@ def get_collection(tenant_name: str, collection_name: str): mongo_tenant_details = BASE_TENANT_STRING % tenant_name database = mongo_client[mongo_tenant_details] generic_collection = database[collection_name] + logging.debug(f"got database {database}") + logging.debug(f"info of database {mongo_tenant_details}") return generic_collection \ No newline at end of file From 088eb59eee23b7de04dbc72ed9ccb6851fa7bfc6 Mon Sep 17 00:00:00 2001 From: Sahil Date: Mon, 22 Apr 2024 15:10:53 +0530 Subject: [PATCH 03/24] removed unwanted logging statements --- modal2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modal2.py b/modal2.py index 4b77c15..93e2611 100644 --- a/modal2.py +++ b/modal2.py @@ -223,7 +223,7 @@ async def get_matching_records(request: MatchingRecordRequestSchema, tenant: str for doc in result: recon_id = doc['reconciliationId'] last_date = doc['createdAt'] + datetime.timedelta(hours=5, minutes=30) # Adjust time based on timezone - logging.debug(f"Processing document for reconciliationId: {recon_id}") + #logging.debug(f"Processing document for reconciliationId: {recon_id}") recon_data = reconciliationPull_collection.find_one({'_id': ObjectId(recon_id)}) if recon_data: From 43146543f6b3278829548fd2bb88847336973a61 Mon Sep 17 00:00:00 2001 From: Sahil Date: Mon, 22 Apr 2024 15:11:42 +0530 Subject: [PATCH 04/24] removed unwanted logging statements for connection.py --- database/connection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/database/connection.py b/database/connection.py index 4b8f8f6..c529aa4 100644 --- a/database/connection.py +++ b/database/connection.py @@ -22,6 +22,6 @@ def get_collection(tenant_name: str, collection_name: str): mongo_tenant_details = BASE_TENANT_STRING % tenant_name database = mongo_client[mongo_tenant_details] generic_collection = database[collection_name] - logging.debug(f"got database {database}") - logging.debug(f"info of database {mongo_tenant_details}") + # logging.debug(f"got database {database}") + # logging.debug(f"info of database {mongo_tenant_details}") return generic_collection \ No newline at end of file From f8e243d9061cfee637215475ff41e3dd7590391f Mon Sep 17 00:00:00 2001 From: Shreyas Date: Mon, 22 Apr 2024 16:26:12 +0530 Subject: [PATCH 05/24] added logging statements for debugging --- database/watcher/watcher.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/database/watcher/watcher.py b/database/watcher/watcher.py index 889dce8..45af745 100644 --- a/database/watcher/watcher.py +++ b/database/watcher/watcher.py @@ -328,7 +328,6 @@ def execute_final_code(tenant, reconciliationId): logging.debug(f"Retrieved from Redis - appId: {appId}, appName: {appName}, reconReportMetadataId: {rrMetaId}") - for i in full_set_of_target_logins: try: full_set_of_cymmetri_logins.remove(i) @@ -341,8 +340,10 @@ def execute_final_code(tenant, reconciliationId): #print(full_set_of_cymmetri_logins) for login in full_set_of_cymmetri_logins: + logging.debug(f"printing logins: {login}") #print(f'{login} only found in cymmetri') login_break_count = getBreakCount(tenant, reconciliationId,login) + logging.debug(f" printing the count of logins in full_set_of_cymmetri_logins: {login_break_count} ") record = { "reconciliationId": reconciliationId, "AppName": appName, @@ -355,7 +356,7 @@ def execute_final_code(tenant, reconciliationId): "break_count": login_break_count } shReconBreakRecords.insert_one(record) - #logging.debug(f"Inserted break record for login {login}: {record.inserted_id}") + logging.debug(f"Inserted break record for login {login}: {record.inserted_id}") break_types = ["present_in_target_only", "present_in_Cymmetri_only", "app_overdue"] From e1a45d36b0b4aaf454b387e554d1e71d073ccc39 Mon Sep 17 00:00:00 2001 From: Shreyas Date: Mon, 22 Apr 2024 16:56:44 +0530 Subject: [PATCH 06/24] fix the issues for watcher inserted id for login --- database/watcher/watcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/watcher/watcher.py b/database/watcher/watcher.py index 45af745..feca593 100644 --- a/database/watcher/watcher.py +++ b/database/watcher/watcher.py @@ -356,7 +356,7 @@ def execute_final_code(tenant, reconciliationId): "break_count": login_break_count } shReconBreakRecords.insert_one(record) - logging.debug(f"Inserted break record for login {login}: {record.inserted_id}") + logging.debug(f"Inserted break record for login {login}") break_types = ["present_in_target_only", "present_in_Cymmetri_only", "app_overdue"] From ca993b6a77fa36e8656a7a2ff8194b30541c0762 Mon Sep 17 00:00:00 2001 From: Shreyas Date: Tue, 23 Apr 2024 11:28:51 +0530 Subject: [PATCH 07/24] added exeception for debugging --- modal2.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/modal2.py b/modal2.py index 93e2611..51d8752 100644 --- a/modal2.py +++ b/modal2.py @@ -4,7 +4,7 @@ from pymongo import MongoClient,DESCENDING import logging from logging.config import dictConfig -from fastapi import FastAPI,HTTPException,Header, Request +from fastapi import FastAPI,HTTPException,Header, Request, status from database.connection import get_collection from bson import ObjectId from pydantic import BaseModel @@ -40,6 +40,13 @@ def load_configuration(): format='[%(asctime)s] %(levelname)s in %(filename)s on %(lineno)d: %(message)s', ) +def create_bad_request_response(response_val): + return Response( + content=json.dumps(response_val), + status_code=status.HTTP_400_BAD_REQUEST, + headers={'Content-Type': 'application/json'} + ) + #print(f'Log level is {log_level_from_data}') # dictConfig({ # "version": 1, @@ -205,6 +212,15 @@ async def get_matching_records(request: MatchingRecordRequestSchema, tenant: str count = doc['count'] # Extract the total count of unique documents logging.debug(f"Total records found: {count}") + if not count: + response_val = { + "data": None, + "success": False, + "errorCode": "DATA_MISSING_ERROR", + "message": "Missing data for requested appId" + } + return create_bad_request_response(response_val) + # Step 3: Define the pipeline to retrieve the latest document per reconciliationId and paginate the results pipeline = [ {"$match": {"appId": request.appId, "createdAt": {"$exists": True}}}, From 7c8b318dfdf17a39ef1267c5a793effb8d2e3dcc Mon Sep 17 00:00:00 2001 From: Sahil Date: Tue, 23 Apr 2024 13:34:52 +0530 Subject: [PATCH 08/24] added exception handlings and error codes for all API --- modal2.py | 393 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 233 insertions(+), 160 deletions(-) diff --git a/modal2.py b/modal2.py index 51d8752..d10a98e 100644 --- a/modal2.py +++ b/modal2.py @@ -47,6 +47,25 @@ def create_bad_request_response(response_val): headers={'Content-Type': 'application/json'} ) +def ResponseModel(data, message, code=200, errorCode=None): + return { + "success": True, + "data": data, + "code": code, + "message": message, + "errorCode": errorCode + } + +def ErrorResponseModel(error, code, message, errorCode): + return { + "data": None, + "success": False, + "code": code, + "message": message, + "errorCode": errorCode, + "error": error + } + #print(f'Log level is {log_level_from_data}') # dictConfig({ # "version": 1, @@ -85,7 +104,7 @@ class Config: # Pydantic model for request body validation class ModalRequestSchema(BaseModel): - Id: str + reconReportMetadataID: str class Config: schema_extra = { "example": { @@ -120,7 +139,7 @@ class Config: # } class BreakRecordWithSearchAndSortRequestSchema(BaseModel): - Id: str + reconReportMetadataID: str pageNumber: int pageSize: int filters: Dict = {} # Allow both string and integer values for filters @@ -159,12 +178,22 @@ def breakReportMetadata(tenant: str): @app.post('/reconsrvc/get_data') async def get_data(request: ModalRequestSchema, tenant: str = Header(...)): logging.info("get_data API Started") - logging.info(f"Fetching data for tenant: {tenant} and request ID: {request.Id}") - + logging.info(f"Fetching data for tenant: {tenant} and request ID: {request.reconReportMetadataID}") + reconReportMetadataID = request.reconReportMetadataID + if not reconReportMetadataID: + response_val = { + "data": None, + "success": False, + "errorCode": "RECONMETADATAID_MISSING_ERROR", + "message": "Missing reconReportMetadataID in request" + } + return create_bad_request_response(response_val) + + breakReportMetadata_collection = breakReportMetadata(tenant) logging.debug("Attempting to retrieve document from the database.") - data = breakReportMetadata_collection.find_one({'reconReportMetadata': request.Id}) + data = breakReportMetadata_collection.find_one({'reconReportMetadata': reconReportMetadataID}) if data: logging.info("Document found, processing data.") @@ -176,14 +205,12 @@ async def get_data(request: ModalRequestSchema, tenant: str = Header(...)): data['startTime'] = start_date data['endTime'] = last_date logging.debug(f"Data processed with updated times: Start - {start_date}, End - {last_date}") - return data + return ResponseModel(data=data, message="get data excecuted succesfully") + else: - error_message = f"No records found matching Id: {request.Id}" + error_message = f"No records found matching Id: {reconReportMetadataID}" logging.error(error_message) - raise HTTPException(status_code=404, detail=error_message) - -# Ensure that appropriate logging configuration is set up to capture and store logs. -logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s') + @app.post("/reconsrvc/matching_records") @@ -263,8 +290,13 @@ async def get_matching_records(request: MatchingRecordRequestSchema, tenant: str # Log the successful completion of the operation logging.info(f"Ending 'get_matching_records' for tenant: {tenant} at {datetime.datetime.now()}") + return ResponseModel(data=return_val, message="Matching Record executed successfully") + + + except Exception as e: logging.error(f"An error occurred: {str(e)}") + return ErrorResponseModel(error=str(e), code=500, message="Exception Occured.", errorCode= "Invalid") return return_val @@ -275,10 +307,22 @@ async def populating_data(request: ModalRequestSchema, tenant: str = Header(...) syncDataForRecon = get_collection(tenant, "syncDataForRecon") user = get_collection(tenant, "user") + + try: # Convert _id from string to ObjectId - object_id = bson.ObjectId(request.Id) - logging.debug(f"Converted {request.Id} to ObjectId: {object_id}") + object_id = bson.ObjectId(request.reconReportMetadata) + logging.debug(f"Converted {request.reconReportMetadata} to ObjectId: {object_id}") + + if not object_id: + response_val = { + "data": None, + "success": False, + "errorCode": "RECONREPORTMETADATAID_MISSING_ERROR", + "message": "Missing reconreportmetadataID in request" + } + return create_bad_request_response(response_val) + # Retrieve the break record breakReconRecord = shReconBreakRecords.find_one({"_id": object_id}) @@ -340,179 +384,208 @@ async def populating_data(request: ModalRequestSchema, tenant: str = Header(...) } logging.debug(f"Return value prepared: {return_val}") - return return_val - + data_response = return_val + return ResponseModel(data=data_response, message="Policy mapping generated successfully") + except Exception as e: logging.error(f"An error occurred: {str(e)}") - raise HTTPException(status_code=500, detail=str(e)) - + return ErrorResponseModel(error=str(e), code=500, message="Exception Occured.", errorCode= "Invalid") + @app.post('/reconsrvc/break_data') async def break_data_table_records(request: BreakRecordWithSearchAndSortRequestSchema, tenant: str = Header(...)) -> Dict: logging.info(f"Processing break data for tenant: {tenant} with request: {request}") + try: - shReconBreakRecords_collection = get_collection(tenant, 'shReconBreakRecords') + shReconBreakRecords_collection = get_collection(tenant, 'shReconBreakRecords') - search_keyword = request.keyword - sortOn_field = "" - sortDirection_value = "" - if request.sortOn == "": - sortOn_field = request.sort - else: - sortOn_field = request.sortOn + search_keyword = request.keyword + sortOn_field = "" + sortDirection_value = "" + if request.sortOn == "": + sortOn_field = request.sort + else: + sortOn_field = request.sortOn - if request.sortDirection == "": - sortDirection_value = request.direction - else: - sortDirection_value = request.sortDirection + if request.sortDirection == "": + sortDirection_value = request.direction + else: + sortDirection_value = request.sortDirection - - #logging.debug(f'found keyword {search_keyword}') - reconReportMetadataId = request.Id - #logging.debug(f"reconReportMetadataId: {reconReportMetadataId}") - search_filters_dict = request.filters - - '''if search_filters_dict['break_count_min']: - search_filters_dict['break_count_min'] = {'$gte': search_filters_dict['break_count_min']} - del search_filters_dict['break_count_min'] - if search_filters_dict['break_count_max']: - search_filters_dict['break_count_max'] = {'$lte': search_filters_dict['break_count_max']} - del search_filters_dict['break_count_max'] - ''' - # Constructing the $and conditions to match reconReportMetadataId, login, and break_type - and_conditions = [{"reconReportMetadataId": reconReportMetadataId}] - - if "login" in search_filters_dict and search_filters_dict["login"]: - and_conditions.append({"$or": [ - {"cymmetri_login": search_filters_dict["login"]}, - {"target_app_login": search_filters_dict["login"]}, - {"break_type": search_filters_dict["login"]}, - ]}) - - if "break_type" in search_filters_dict and search_filters_dict["break_type"]: - and_conditions.append({"break_type": search_filters_dict["break_type"]}) - - if "break_count_min" in search_filters_dict and search_filters_dict["break_count_min"]: - and_conditions.append({"break_count": {"$gte": search_filters_dict["break_count_min"]}}) - - if "break_count_max" in search_filters_dict and search_filters_dict["break_count_max"]: - and_conditions.append({"break_count": {"$lte": search_filters_dict["break_count_max"]}}) - - if search_keyword: + #logging.debug(f'found keyword {search_keyword}') - # Construct a regex pattern to match the provided keyword in a continuous manner as a whole word - #keyword_regex ="" - # Add a new condition to match any field that contains the keyword - or_conditions = [] - for field in ["cymmetri_login", "target_app_login", "break_type"]: - value = f'.*{search_keyword}.*' - logging.debug(f'value is {value}') - or_conditions.append({field: {"$regex": value}}) - and_conditions.append({"$or": or_conditions}) - - #logging.debug(f'And conditions are {and_conditions}') - - if sortDirection_value == "ASC": - sortDirection_value = 1 - else: - sortDirection_value = -1 - - pipeline = [ - {"$match": {"$and": and_conditions}}, - {"$sort": {sortOn_field: sortDirection_value}}, - {"$skip": request.pageNumber * request.pageSize}, - {"$limit": request.pageSize} - ] - - logging.debug(f"Constructed MongoDB query pipeline: {pipeline}") - - result = shReconBreakRecords_collection.aggregate(pipeline) - #logging.debug(f"Query executed successfully") - response = [] - - #counts = {} - - for doc in result: - id_str = str(doc.get('_id')) - #counts[id_str] = await get_count_of_doc(shReconBreakRecords_collection, id_str) - - #count_a = doc.get('count') - #doc = doc.get('latestRecord') - response.append({ - "Id": id_str, - "target_app_login": doc.get('target_app_login'), - "source_app_login": doc.get('cymmetri_login'), - "cymmetri_login": doc.get('cymmetri_login'), - "break_type": doc.get('break_type'), - "break_count": doc.get('break_count') - }) + reconReportMetadataId = request.reconReportMetadataID - count_pipeline = pipeline.copy() # Creating a copy to avoid modifying the original pipeline - count_pipeline[-2] = {"$count": "count"} # Replacing the last stage to get count directly - count_pipeline.pop() - count_result = list(shReconBreakRecords_collection.aggregate(count_pipeline)) - total_records = count_result[0]["count"] if count_result else 0 - logging.debug(f"Total records found: {total_records}") + if not reconReportMetadataId: + response_val = { + "data": None, + "success": False, + "errorCode": "RECONMETADATAID_MISSING_ERROR", + "message": "Missing reconReportMetadataId in request" + } + return create_bad_request_response(response_val) - total_pages = math.ceil(total_records / request.pageSize) - logging.debug("Returning processed data to client.") - return { - "success": True, - "data": {"content": response}, - "totalRecords": total_records, - "totalPages": total_pages - } + #logging.debug(f"reconReportMetadataId: {reconReportMetadataId}") + search_filters_dict = request.filters + + '''if search_filters_dict['break_count_min']: + search_filters_dict['break_count_min'] = {'$gte': search_filters_dict['break_count_min']} + del search_filters_dict['break_count_min'] + if search_filters_dict['break_count_max']: + search_filters_dict['break_count_max'] = {'$lte': search_filters_dict['break_count_max']} + del search_filters_dict['break_count_max'] + ''' + # Constructing the $and conditions to match reconReportMetadataId, login, and break_type + and_conditions = [{"reconReportMetadataId": reconReportMetadataId}] + + if "login" in search_filters_dict and search_filters_dict["login"]: + and_conditions.append({"$or": [ + {"cymmetri_login": search_filters_dict["login"]}, + {"target_app_login": search_filters_dict["login"]}, + {"break_type": search_filters_dict["login"]}, + ]}) + + if "break_type" in search_filters_dict and search_filters_dict["break_type"]: + and_conditions.append({"break_type": search_filters_dict["break_type"]}) + + if "break_count_min" in search_filters_dict and search_filters_dict["break_count_min"]: + and_conditions.append({"break_count": {"$gte": search_filters_dict["break_count_min"]}}) + + if "break_count_max" in search_filters_dict and search_filters_dict["break_count_max"]: + and_conditions.append({"break_count": {"$lte": search_filters_dict["break_count_max"]}}) + + if search_keyword: + #logging.debug(f'found keyword {search_keyword}') + # Construct a regex pattern to match the provided keyword in a continuous manner as a whole word + #keyword_regex ="" + # Add a new condition to match any field that contains the keyword + or_conditions = [] + for field in ["cymmetri_login", "target_app_login", "break_type"]: + value = f'.*{search_keyword}.*' + logging.debug(f'value is {value}') + or_conditions.append({field: {"$regex": value}}) + and_conditions.append({"$or": or_conditions}) + + #logging.debug(f'And conditions are {and_conditions}') + + if sortDirection_value == "ASC": + sortDirection_value = 1 + else: + sortDirection_value = -1 + + pipeline = [ + {"$match": {"$and": and_conditions}}, + {"$sort": {sortOn_field: sortDirection_value}}, + {"$skip": request.pageNumber * request.pageSize}, + {"$limit": request.pageSize} + ] + logging.debug(f"Constructed MongoDB query pipeline: {pipeline}") + + result = shReconBreakRecords_collection.aggregate(pipeline) + #logging.debug(f"Query executed successfully") + response = [] + + #counts = {} + + for doc in result: + id_str = str(doc.get('_id')) + #counts[id_str] = await get_count_of_doc(shReconBreakRecords_collection, id_str) + + #count_a = doc.get('count') + #doc = doc.get('latestRecord') + response.append({ + "Id": id_str, + "target_app_login": doc.get('target_app_login'), + "source_app_login": doc.get('cymmetri_login'), + "cymmetri_login": doc.get('cymmetri_login'), + "break_type": doc.get('break_type'), + "break_count": doc.get('break_count') + }) + + count_pipeline = pipeline.copy() # Creating a copy to avoid modifying the original pipeline + count_pipeline[-2] = {"$count": "count"} # Replacing the last stage to get count directly + count_pipeline.pop() + count_result = list(shReconBreakRecords_collection.aggregate(count_pipeline)) + total_records = count_result[0]["count"] if count_result else 0 + logging.debug(f"Total records found: {total_records}") + + total_pages = math.ceil(total_records / request.pageSize) + logging.debug("Returning processed data to client.") + + + return { + "success": True, + "data": {"content": response}, + "totalRecords": total_records, + "totalPages": total_pages + } + + except Exception as e: + return ErrorResponseModel(error=str(e), code=500, message="Exception Occured.", errorCode= "Invalid") @app.post('/reconsrvc/start_recon') async def start_recon(request: StartReconRequestSchema, fullRequest: Request) -> Response: logging.debug("Start of 360 degree recon for reconciliationId: %s", request.reconciliationId) - - # Fetch job history collection - job_history_collection = get_collection(fullRequest.headers.get('tenant'), 'recon_360_job_history') - - # Check for existing jobs - count = job_history_collection.count_documents({'reconciliationId': request.reconciliationId, 'completed': False}) - logging.debug("Existing job count for reconciliationId %s: %d", request.reconciliationId, count) - - if count > 0: - response_val = {} - response_val['success'] = 'false' - response_val['error'] = 'JOB_RUNNING' - response_val['errorMessage'] = '360 degree reconciliation is already running for given reconciliationId' - raise HTTPException(status_code=400, detail=response_val) - - # Call to external service - baseUrl_for_java_srvc = data['RECON_PROV_SRVC'] - url = baseUrl_for_java_srvc + '/reconciliation/userReconciliation' - payload = json.dumps({ - "reconciliationId": request.reconciliationId, - "reconType": request.reconType - }) - - logging.info("Calling external service at URL %s with payload %s", url, payload) - response = requests.post(url, headers=fullRequest.headers, data=payload) - logging.info("Received response status %s from external service for reconciliationId %s", response.status_code, request.reconciliationId) - - if response.status_code == 200: - job_history_collection.insert_one({ + try: + + reconciliationId = request.reconciliationId + + # Fetch job history collection + job_history_collection = get_collection(fullRequest.headers.get('tenant'), 'recon_360_job_history') + + if not reconciliationId: + response_val = { + "data": None, + "success": False, + "errorCode": "RECONCILIATIONID_MISSING_ERROR", + "message": "Missing reconciliationID in request" + } + return create_bad_request_response(response_val) + + # Check for existing jobs + count = job_history_collection.count_documents({'reconciliationId': reconciliationId, 'completed': False}) + logging.debug("Existing job count for reconciliationId %s: %d", reconciliationId, count) + + if count > 0: + response_val = {} + response_val['success'] = 'false' + response_val['error'] = 'JOB_RUNNING' + response_val['errorMessage'] = '360 degree reconciliation is already running for given reconciliationId' + raise HTTPException(status_code=400, detail=response_val) + + # Call to external service + baseUrl_for_java_srvc = data['RECON_PROV_SRVC'] + url = baseUrl_for_java_srvc + '/reconciliation/userReconciliation' + payload = json.dumps({ "reconciliationId": request.reconciliationId, - "completed": False, - "startTime": datetime.datetime.now() + "reconType": request.reconType }) - logging.info("Inserted new job history record for reconciliationId %s", request.reconciliationId) - # Start background task - logging.debug("Starting background task for tenant %s and reconciliationId %s", fullRequest.headers.get('tenant'), request.reconciliationId) - create_task(printTenant(tenant=fullRequest.headers.get('tenant'), reconciliationId=request.reconciliationId)) + logging.info("Calling external service at URL %s with payload %s", url, payload) + response = requests.post(url, headers=fullRequest.headers, data=payload) + logging.info("Received response status %s from external service for reconciliationId %s", response.status_code, request.reconciliationId) - print("res",response) - - return response.json() + if response.status_code == 200: + job_history_collection.insert_one({ + "reconciliationId": request.reconciliationId, + "completed": False, + "startTime": datetime.datetime.now() + }) + logging.info("Inserted new job history record for reconciliationId %s", request.reconciliationId) + + # Start background task + logging.debug("Starting background task for tenant %s and reconciliationId %s", fullRequest.headers.get('tenant'), request.reconciliationId) + create_task(printTenant(tenant=fullRequest.headers.get('tenant'), reconciliationId=request.reconciliationId)) + + data_response = response.json() + return ResponseModel(data=data_response, message="360 Recon Started successfully") + except Exception as e: + return ErrorResponseModel(error=str(e), code=500, message="Exception while starting 360 Recon.", errorCode= "Invalid") if __name__ == "__main__": import uvicorn uvicorn.run(app, host="0.0.0.0", port=8000) \ No newline at end of file From 163bb896c1e722931bf67c0919a4dc6255433bea Mon Sep 17 00:00:00 2001 From: Sahil Date: Tue, 23 Apr 2024 13:35:38 +0530 Subject: [PATCH 09/24] removed unwanted printing statements --- database/watcher/watcher.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/database/watcher/watcher.py b/database/watcher/watcher.py index feca593..2f9f8ff 100644 --- a/database/watcher/watcher.py +++ b/database/watcher/watcher.py @@ -340,10 +340,8 @@ def execute_final_code(tenant, reconciliationId): #print(full_set_of_cymmetri_logins) for login in full_set_of_cymmetri_logins: - logging.debug(f"printing logins: {login}") #print(f'{login} only found in cymmetri') login_break_count = getBreakCount(tenant, reconciliationId,login) - logging.debug(f" printing the count of logins in full_set_of_cymmetri_logins: {login_break_count} ") record = { "reconciliationId": reconciliationId, "AppName": appName, @@ -356,7 +354,7 @@ def execute_final_code(tenant, reconciliationId): "break_count": login_break_count } shReconBreakRecords.insert_one(record) - logging.debug(f"Inserted break record for login {login}") + logging.debug("Data Inserted into shReconBreakRecords succesfully.") break_types = ["present_in_target_only", "present_in_Cymmetri_only", "app_overdue"] From eaaca283e60ecd3c0a6d7294e91627c7355b8bc6 Mon Sep 17 00:00:00 2001 From: Sahil Date: Tue, 23 Apr 2024 13:54:38 +0530 Subject: [PATCH 10/24] added exception for breakdata --- modal2.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modal2.py b/modal2.py index d10a98e..b91a1a6 100644 --- a/modal2.py +++ b/modal2.py @@ -516,13 +516,15 @@ async def break_data_table_records(request: BreakRecordWithSearchAndSortRequestS logging.debug("Returning processed data to client.") - return { + data = { "success": True, "data": {"content": response}, "totalRecords": total_records, "totalPages": total_pages } + return ResponseModel(data=data, message="Break Data Executed succesfully.") + except Exception as e: return ErrorResponseModel(error=str(e), code=500, message="Exception Occured.", errorCode= "Invalid") From 6ea03cefc927decde32cdb1d2ebeffd9aa3509e2 Mon Sep 17 00:00:00 2001 From: Sahil Date: Tue, 23 Apr 2024 16:11:42 +0530 Subject: [PATCH 11/24] revert changes for modal_APi --- modal2.py | 187 ++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 125 insertions(+), 62 deletions(-) diff --git a/modal2.py b/modal2.py index b91a1a6..b67f3d9 100644 --- a/modal2.py +++ b/modal2.py @@ -104,11 +104,11 @@ class Config: # Pydantic model for request body validation class ModalRequestSchema(BaseModel): - reconReportMetadataID: str + Id: str class Config: schema_extra = { "example": { - "reconReportMetadata": "6126126126166af11", + "Id": "6126126126166af11", } } @@ -300,97 +300,160 @@ async def get_matching_records(request: MatchingRecordRequestSchema, tenant: str return return_val +# @app.post('/reconsrvc/modal') +# async def populating_data(request: ModalRequestSchema, tenant: str = Header(...)) -> Response: +# # Connect to MongoDB +# shReconBreakRecords = get_collection(tenant, "shReconBreakRecords") +# syncDataForRecon = get_collection(tenant, "syncDataForRecon") +# user = get_collection(tenant, "user") + + + +# try: +# # Convert _id from string to ObjectId +# object_id = bson.ObjectId(request.reconReportMetadataID) +# logging.debug(f"Converted {request.reconReportMetadataID} to ObjectId: {object_id}") +# print("object_id",object_id) +# if not object_id: +# response_val = { +# "data": None, +# "success": False, +# "errorCode": "RECONREPORTMETADATAID_MISSING_ERROR", +# "message": "Missing reconreportmetadataID in request" +# } +# return create_bad_request_response(response_val) + + +# # Retrieve the break record +# breakReconRecord = shReconBreakRecords.find_one({"_id": object_id}) +# logging.debug(f"Retrieved break record: {breakReconRecord}") + +# if not breakReconRecord: +# logging.error("No break record found") +# raise HTTPException(status_code=404, detail="Break record not found") + +# if breakReconRecord['break_type'] in ['present_in_target_only', 'app_overdue']: +# app_id = breakReconRecord['target_app_login'] +# logging.debug(f"Processing type {breakReconRecord['break_type']} for app ID {app_id}") + +# query = {"reconciliationId": breakReconRecord['reconciliationId'], "data.login": app_id} +# query_breakCount = {"reconciliationId": breakReconRecord['reconciliationId'], "target_app_login": app_id} + +# records = syncDataForRecon.find(query).sort([("updatedDateTime", DESCENDING)]).limit(1) +# return_val = {} +# for single_record in records: +# return_val['userInfo'] = single_record['data'] +# break_info = {'breakCount': shReconBreakRecords.count_documents(query_breakCount)} + +# if breakReconRecord['break_type'] == 'app_overdue': +# break_info['message'] = 'User overdue to be disabled but present in Application.' +# else: +# break_info['message'] = 'User only present in application.' + +# return_val['break_info'] = break_info + +# logging.debug(f"Return value prepared: {return_val}") +# return return_val + +# else: +# cymmetri_login = breakReconRecord['cymmetri_login'] +# logging.debug(f"Processing type {breakReconRecord['break_type']} for Cymmetri login {cymmetri_login}") + +# query_breakCount_Cymm = {"reconciliationId": breakReconRecord['reconciliationId'], "cymmetri_login": cymmetri_login} +# query = {"provisionedApps.CYMMETRI.login.login": cymmetri_login} + +# userRecord = user.find_one(query) +# if not userRecord: +# logging.error("User record not found") +# raise HTTPException(status_code=404, detail="User record not found") + +# return_val = { +# 'userInfo': { +# 'login': userRecord['provisionedApps']['CYMMETRI']['login']['login'], +# 'displayName': userRecord['displayName'], +# 'firstName': userRecord['firstName'], +# 'lastName': userRecord['lastName'], +# 'status': userRecord['status'], +# 'email': userRecord['email'], +# 'createdDate': userRecord['created'] +# }, +# 'break_info': { +# 'message': 'User is present in Cymmetri but not in Application.', +# 'breakCount': shReconBreakRecords.count_documents(query_breakCount_Cymm) +# } +# } +# print("return_val",return_val) +# logging.debug(f"Return value prepared: {return_val}") +# data_response = return_val +# return ResponseModel(data=data_response, message="Policy mapping generated successfully") + +# except Exception as e: +# logging.error(f"An error occurred: {str(e)}") +# return ErrorResponseModel(error=str(e), code=500, message="Exception Occured.", errorCode= "Invalid") + @app.post('/reconsrvc/modal') async def populating_data(request: ModalRequestSchema, tenant: str = Header(...)) -> Response: # Connect to MongoDB shReconBreakRecords = get_collection(tenant, "shReconBreakRecords") syncDataForRecon = get_collection(tenant, "syncDataForRecon") user = get_collection(tenant, "user") - - try: - # Convert _id from string to ObjectId - object_id = bson.ObjectId(request.reconReportMetadata) - logging.debug(f"Converted {request.reconReportMetadata} to ObjectId: {object_id}") - - if not object_id: - response_val = { - "data": None, - "success": False, - "errorCode": "RECONREPORTMETADATAID_MISSING_ERROR", - "message": "Missing reconreportmetadataID in request" - } - return create_bad_request_response(response_val) + # Convert _id from string to ObjectId + object_id = bson.ObjectId(request.Id) + - # Retrieve the break record breakReconRecord = shReconBreakRecords.find_one({"_id": object_id}) - logging.debug(f"Retrieved break record: {breakReconRecord}") - - if not breakReconRecord: - logging.error("No break record found") - raise HTTPException(status_code=404, detail="Break record not found") - - if breakReconRecord['break_type'] in ['present_in_target_only', 'app_overdue']: + ##print(breakReconRecord) + + if breakReconRecord['break_type'] == 'present_in_target_only' or breakReconRecord['break_type'] == 'app_overdue': + # Fetch appId from the record app_id = breakReconRecord['target_app_login'] - logging.debug(f"Processing type {breakReconRecord['break_type']} for app ID {app_id}") - query = {"reconciliationId": breakReconRecord['reconciliationId'], "data.login": app_id} query_breakCount = {"reconciliationId": breakReconRecord['reconciliationId'], "target_app_login": app_id} - records = syncDataForRecon.find(query).sort([("updatedDateTime", DESCENDING)]).limit(1) + sort_criteria = [("updatedDateTime", DESCENDING)] + records = syncDataForRecon.find(query).sort(sort_criteria).limit(1) return_val = {} for single_record in records: return_val['userInfo'] = single_record['data'] - break_info = {'breakCount': shReconBreakRecords.count_documents(query_breakCount)} - + break_info = {} if breakReconRecord['break_type'] == 'app_overdue': break_info['message'] = 'User overdue to be disabled but present in Application.' else: break_info['message'] = 'User only present in application.' - + break_info['breakCount'] = shReconBreakRecords.count_documents(query_breakCount) return_val['break_info'] = break_info - - logging.debug(f"Return value prepared: {return_val}") return return_val - else: cymmetri_login = breakReconRecord['cymmetri_login'] - logging.debug(f"Processing type {breakReconRecord['break_type']} for Cymmetri login {cymmetri_login}") - query_breakCount_Cymm = {"reconciliationId": breakReconRecord['reconciliationId'], "cymmetri_login": cymmetri_login} query = {"provisionedApps.CYMMETRI.login.login": cymmetri_login} userRecord = user.find_one(query) - if not userRecord: - logging.error("User record not found") - raise HTTPException(status_code=404, detail="User record not found") - - return_val = { - 'userInfo': { - 'login': userRecord['provisionedApps']['CYMMETRI']['login']['login'], - 'displayName': userRecord['displayName'], - 'firstName': userRecord['firstName'], - 'lastName': userRecord['lastName'], - 'status': userRecord['status'], - 'email': userRecord['email'], - 'createdDate': userRecord['created'] - }, - 'break_info': { - 'message': 'User is present in Cymmetri but not in Application.', - 'breakCount': shReconBreakRecords.count_documents(query_breakCount_Cymm) - } - } + return_val = {} + cymmetri_info = {} + cymmetri_info['login'] = userRecord['provisionedApps']['CYMMETRI']['login']['login'] + cymmetri_info['displayName'] = userRecord['displayName'] + cymmetri_info['firstName'] = userRecord['firstName'] + cymmetri_info['lastName'] = userRecord['lastName'] + cymmetri_info['status'] = userRecord['status'] + cymmetri_info['email'] = userRecord['email'] + cymmetri_info['createdDate'] = userRecord['created'] + return_val['userInfo'] = cymmetri_info + + break_info = {} + break_info['message'] = 'User is present in Cymmetri but not in Application.' + break_info['breakCount'] = shReconBreakRecords.count_documents(query_breakCount_Cymm) + return_val['break_info'] = break_info + return return_val - logging.debug(f"Return value prepared: {return_val}") - data_response = return_val - return ResponseModel(data=data_response, message="Policy mapping generated successfully") - - except Exception as e: - logging.error(f"An error occurred: {str(e)}") - return ErrorResponseModel(error=str(e), code=500, message="Exception Occured.", errorCode= "Invalid") + + # If no matching record found + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) @app.post('/reconsrvc/break_data') async def break_data_table_records(request: BreakRecordWithSearchAndSortRequestSchema, tenant: str = Header(...)) -> Dict: @@ -590,4 +653,4 @@ async def start_recon(request: StartReconRequestSchema, fullRequest: Request) -> return ErrorResponseModel(error=str(e), code=500, message="Exception while starting 360 Recon.", errorCode= "Invalid") if __name__ == "__main__": import uvicorn - uvicorn.run(app, host="0.0.0.0", port=8000) \ No newline at end of file + uvicorn.run(app, host="127.0.0.1", port=8000) \ No newline at end of file From c1c2c79eab9d0740be3d7520356e15ae578f2eba Mon Sep 17 00:00:00 2001 From: Sahil Date: Tue, 23 Apr 2024 17:20:19 +0530 Subject: [PATCH 12/24] done changes in get_data --- modal2.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/modal2.py b/modal2.py index b67f3d9..c32caee 100644 --- a/modal2.py +++ b/modal2.py @@ -104,11 +104,11 @@ class Config: # Pydantic model for request body validation class ModalRequestSchema(BaseModel): - Id: str + reconReportMetadataID: str class Config: schema_extra = { "example": { - "Id": "6126126126166af11", + "reconReportMetadataID": "6126126126166af11", } } @@ -298,7 +298,6 @@ async def get_matching_records(request: MatchingRecordRequestSchema, tenant: str logging.error(f"An error occurred: {str(e)}") return ErrorResponseModel(error=str(e), code=500, message="Exception Occured.", errorCode= "Invalid") - return return_val # @app.post('/reconsrvc/modal') # async def populating_data(request: ModalRequestSchema, tenant: str = Header(...)) -> Response: From 90083f49e706d40cb82abf3953cc52c752daddbc Mon Sep 17 00:00:00 2001 From: Sahil Date: Tue, 23 Apr 2024 17:31:21 +0530 Subject: [PATCH 13/24] fixed code for modal not working --- modal2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modal2.py b/modal2.py index c32caee..472458f 100644 --- a/modal2.py +++ b/modal2.py @@ -104,7 +104,7 @@ class Config: # Pydantic model for request body validation class ModalRequestSchema(BaseModel): - reconReportMetadataID: str + Id: str class Config: schema_extra = { "example": { From b1b9d3d5247e46087fdc66296aee45a1d9e78508 Mon Sep 17 00:00:00 2001 From: Sahil Date: Tue, 23 Apr 2024 17:42:36 +0530 Subject: [PATCH 14/24] did changes for get_data pydantic model is same for get_data and modal --- modal2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modal2.py b/modal2.py index 472458f..d944c1b 100644 --- a/modal2.py +++ b/modal2.py @@ -178,8 +178,8 @@ def breakReportMetadata(tenant: str): @app.post('/reconsrvc/get_data') async def get_data(request: ModalRequestSchema, tenant: str = Header(...)): logging.info("get_data API Started") - logging.info(f"Fetching data for tenant: {tenant} and request ID: {request.reconReportMetadataID}") - reconReportMetadataID = request.reconReportMetadataID + logging.info(f"Fetching data for tenant: {tenant} and request ID: {request.Id}") + reconReportMetadataID = request.Id if not reconReportMetadataID: response_val = { "data": None, From 83518939170284e07e263177640be7449787b361 Mon Sep 17 00:00:00 2001 From: Sahil Date: Tue, 23 Apr 2024 17:46:51 +0530 Subject: [PATCH 15/24] add seprate pydantic model for get_data --- modal2.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/modal2.py b/modal2.py index d944c1b..e580322 100644 --- a/modal2.py +++ b/modal2.py @@ -112,6 +112,15 @@ class Config: } } +class ModalRequestSchema_get_data(BaseModel): + reconReportMetadataID: str + class Config: + schema_extra = { + "example": { + "reconReportMetadataID": "6126126126166af11", + } + } + class MatchingRecordRequestSchema(BaseModel): appId: str pageNumber: int @@ -176,10 +185,10 @@ def breakReportMetadata(tenant: str): # API endpoint to fetch data by ObjectId # API endpoint to fetch data by ObjectId @app.post('/reconsrvc/get_data') -async def get_data(request: ModalRequestSchema, tenant: str = Header(...)): +async def get_data(request: ModalRequestSchema_get_data, tenant: str = Header(...)): logging.info("get_data API Started") - logging.info(f"Fetching data for tenant: {tenant} and request ID: {request.Id}") - reconReportMetadataID = request.Id + logging.info(f"Fetching data for tenant: {tenant} and request ID: {request.reconReportMetadataID}") + reconReportMetadataID = request.reconReportMetadataID if not reconReportMetadataID: response_val = { "data": None, From 7e267f3f14bd10134d74c222a1ea0f2628097d80 Mon Sep 17 00:00:00 2001 From: Sahil Date: Tue, 23 Apr 2024 17:57:16 +0530 Subject: [PATCH 16/24] added error code for modal api as well which was previously removed --- modal2.py | 244 +++++++++++++++++++++++++++--------------------------- 1 file changed, 122 insertions(+), 122 deletions(-) diff --git a/modal2.py b/modal2.py index e580322..c4fd194 100644 --- a/modal2.py +++ b/modal2.py @@ -104,7 +104,7 @@ class Config: # Pydantic model for request body validation class ModalRequestSchema(BaseModel): - Id: str + reconReportMetadataID: str class Config: schema_extra = { "example": { @@ -308,160 +308,160 @@ async def get_matching_records(request: MatchingRecordRequestSchema, tenant: str return ErrorResponseModel(error=str(e), code=500, message="Exception Occured.", errorCode= "Invalid") -# @app.post('/reconsrvc/modal') -# async def populating_data(request: ModalRequestSchema, tenant: str = Header(...)) -> Response: -# # Connect to MongoDB -# shReconBreakRecords = get_collection(tenant, "shReconBreakRecords") -# syncDataForRecon = get_collection(tenant, "syncDataForRecon") -# user = get_collection(tenant, "user") - - - -# try: -# # Convert _id from string to ObjectId -# object_id = bson.ObjectId(request.reconReportMetadataID) -# logging.debug(f"Converted {request.reconReportMetadataID} to ObjectId: {object_id}") -# print("object_id",object_id) -# if not object_id: -# response_val = { -# "data": None, -# "success": False, -# "errorCode": "RECONREPORTMETADATAID_MISSING_ERROR", -# "message": "Missing reconreportmetadataID in request" -# } -# return create_bad_request_response(response_val) - - -# # Retrieve the break record -# breakReconRecord = shReconBreakRecords.find_one({"_id": object_id}) -# logging.debug(f"Retrieved break record: {breakReconRecord}") - -# if not breakReconRecord: -# logging.error("No break record found") -# raise HTTPException(status_code=404, detail="Break record not found") - -# if breakReconRecord['break_type'] in ['present_in_target_only', 'app_overdue']: -# app_id = breakReconRecord['target_app_login'] -# logging.debug(f"Processing type {breakReconRecord['break_type']} for app ID {app_id}") - -# query = {"reconciliationId": breakReconRecord['reconciliationId'], "data.login": app_id} -# query_breakCount = {"reconciliationId": breakReconRecord['reconciliationId'], "target_app_login": app_id} - -# records = syncDataForRecon.find(query).sort([("updatedDateTime", DESCENDING)]).limit(1) -# return_val = {} -# for single_record in records: -# return_val['userInfo'] = single_record['data'] -# break_info = {'breakCount': shReconBreakRecords.count_documents(query_breakCount)} - -# if breakReconRecord['break_type'] == 'app_overdue': -# break_info['message'] = 'User overdue to be disabled but present in Application.' -# else: -# break_info['message'] = 'User only present in application.' - -# return_val['break_info'] = break_info - -# logging.debug(f"Return value prepared: {return_val}") -# return return_val - -# else: -# cymmetri_login = breakReconRecord['cymmetri_login'] -# logging.debug(f"Processing type {breakReconRecord['break_type']} for Cymmetri login {cymmetri_login}") - -# query_breakCount_Cymm = {"reconciliationId": breakReconRecord['reconciliationId'], "cymmetri_login": cymmetri_login} -# query = {"provisionedApps.CYMMETRI.login.login": cymmetri_login} - -# userRecord = user.find_one(query) -# if not userRecord: -# logging.error("User record not found") -# raise HTTPException(status_code=404, detail="User record not found") - -# return_val = { -# 'userInfo': { -# 'login': userRecord['provisionedApps']['CYMMETRI']['login']['login'], -# 'displayName': userRecord['displayName'], -# 'firstName': userRecord['firstName'], -# 'lastName': userRecord['lastName'], -# 'status': userRecord['status'], -# 'email': userRecord['email'], -# 'createdDate': userRecord['created'] -# }, -# 'break_info': { -# 'message': 'User is present in Cymmetri but not in Application.', -# 'breakCount': shReconBreakRecords.count_documents(query_breakCount_Cymm) -# } -# } -# print("return_val",return_val) -# logging.debug(f"Return value prepared: {return_val}") -# data_response = return_val -# return ResponseModel(data=data_response, message="Policy mapping generated successfully") - -# except Exception as e: -# logging.error(f"An error occurred: {str(e)}") -# return ErrorResponseModel(error=str(e), code=500, message="Exception Occured.", errorCode= "Invalid") - @app.post('/reconsrvc/modal') async def populating_data(request: ModalRequestSchema, tenant: str = Header(...)) -> Response: # Connect to MongoDB shReconBreakRecords = get_collection(tenant, "shReconBreakRecords") syncDataForRecon = get_collection(tenant, "syncDataForRecon") user = get_collection(tenant, "user") + - try: + try: # Convert _id from string to ObjectId - object_id = bson.ObjectId(request.Id) - + object_id = bson.ObjectId(request.reconReportMetadataID) + logging.debug(f"Converted {request.reconReportMetadataID} to ObjectId: {object_id}") + print("object_id",object_id) + if not object_id: + response_val = { + "data": None, + "success": False, + "errorCode": "RECONREPORTMETADATAID_MISSING_ERROR", + "message": "Missing reconreportmetadataID in request" + } + return create_bad_request_response(response_val) + + # Retrieve the break record breakReconRecord = shReconBreakRecords.find_one({"_id": object_id}) - ##print(breakReconRecord) - - if breakReconRecord['break_type'] == 'present_in_target_only' or breakReconRecord['break_type'] == 'app_overdue': - # Fetch appId from the record + logging.debug(f"Retrieved break record: {breakReconRecord}") + + if not breakReconRecord: + logging.error("No break record found") + raise HTTPException(status_code=404, detail="Break record not found") + + if breakReconRecord['break_type'] in ['present_in_target_only', 'app_overdue']: app_id = breakReconRecord['target_app_login'] + logging.debug(f"Processing type {breakReconRecord['break_type']} for app ID {app_id}") + query = {"reconciliationId": breakReconRecord['reconciliationId'], "data.login": app_id} query_breakCount = {"reconciliationId": breakReconRecord['reconciliationId'], "target_app_login": app_id} - sort_criteria = [("updatedDateTime", DESCENDING)] - records = syncDataForRecon.find(query).sort(sort_criteria).limit(1) + records = syncDataForRecon.find(query).sort([("updatedDateTime", DESCENDING)]).limit(1) return_val = {} for single_record in records: return_val['userInfo'] = single_record['data'] - break_info = {} + break_info = {'breakCount': shReconBreakRecords.count_documents(query_breakCount)} + if breakReconRecord['break_type'] == 'app_overdue': break_info['message'] = 'User overdue to be disabled but present in Application.' else: break_info['message'] = 'User only present in application.' - break_info['breakCount'] = shReconBreakRecords.count_documents(query_breakCount) + return_val['break_info'] = break_info + + logging.debug(f"Return value prepared: {return_val}") return return_val + else: cymmetri_login = breakReconRecord['cymmetri_login'] + logging.debug(f"Processing type {breakReconRecord['break_type']} for Cymmetri login {cymmetri_login}") + query_breakCount_Cymm = {"reconciliationId": breakReconRecord['reconciliationId'], "cymmetri_login": cymmetri_login} query = {"provisionedApps.CYMMETRI.login.login": cymmetri_login} userRecord = user.find_one(query) - return_val = {} - cymmetri_info = {} - cymmetri_info['login'] = userRecord['provisionedApps']['CYMMETRI']['login']['login'] - cymmetri_info['displayName'] = userRecord['displayName'] - cymmetri_info['firstName'] = userRecord['firstName'] - cymmetri_info['lastName'] = userRecord['lastName'] - cymmetri_info['status'] = userRecord['status'] - cymmetri_info['email'] = userRecord['email'] - cymmetri_info['createdDate'] = userRecord['created'] - return_val['userInfo'] = cymmetri_info - - break_info = {} - break_info['message'] = 'User is present in Cymmetri but not in Application.' - break_info['breakCount'] = shReconBreakRecords.count_documents(query_breakCount_Cymm) - return_val['break_info'] = break_info - return return_val + if not userRecord: + logging.error("User record not found") + raise HTTPException(status_code=404, detail="User record not found") + + return_val = { + 'userInfo': { + 'login': userRecord['provisionedApps']['CYMMETRI']['login']['login'], + 'displayName': userRecord['displayName'], + 'firstName': userRecord['firstName'], + 'lastName': userRecord['lastName'], + 'status': userRecord['status'], + 'email': userRecord['email'], + 'createdDate': userRecord['created'] + }, + 'break_info': { + 'message': 'User is present in Cymmetri but not in Application.', + 'breakCount': shReconBreakRecords.count_documents(query_breakCount_Cymm) + } + } + print("return_val",return_val) + logging.debug(f"Return value prepared: {return_val}") + data_response = return_val + return ResponseModel(data=data_response, message="Policy mapping generated successfully") + + except Exception as e: + logging.error(f"An error occurred: {str(e)}") + return ErrorResponseModel(error=str(e), code=500, message="Exception Occured.", errorCode= "Invalid") + +# @app.post('/reconsrvc/modal') +# async def populating_data(request: ModalRequestSchema, tenant: str = Header(...)) -> Response: +# # Connect to MongoDB +# shReconBreakRecords = get_collection(tenant, "shReconBreakRecords") +# syncDataForRecon = get_collection(tenant, "syncDataForRecon") +# user = get_collection(tenant, "user") + +# try: +# # Convert _id from string to ObjectId +# object_id = bson.ObjectId(request.Id) + +# breakReconRecord = shReconBreakRecords.find_one({"_id": object_id}) +# ##print(breakReconRecord) - # If no matching record found - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) +# if breakReconRecord['break_type'] == 'present_in_target_only' or breakReconRecord['break_type'] == 'app_overdue': +# # Fetch appId from the record +# app_id = breakReconRecord['target_app_login'] +# query = {"reconciliationId": breakReconRecord['reconciliationId'], "data.login": app_id} +# query_breakCount = {"reconciliationId": breakReconRecord['reconciliationId'], "target_app_login": app_id} + +# sort_criteria = [("updatedDateTime", DESCENDING)] +# records = syncDataForRecon.find(query).sort(sort_criteria).limit(1) +# return_val = {} +# for single_record in records: +# return_val['userInfo'] = single_record['data'] +# break_info = {} +# if breakReconRecord['break_type'] == 'app_overdue': +# break_info['message'] = 'User overdue to be disabled but present in Application.' +# else: +# break_info['message'] = 'User only present in application.' +# break_info['breakCount'] = shReconBreakRecords.count_documents(query_breakCount) +# return_val['break_info'] = break_info +# return return_val +# else: +# cymmetri_login = breakReconRecord['cymmetri_login'] +# query_breakCount_Cymm = {"reconciliationId": breakReconRecord['reconciliationId'], "cymmetri_login": cymmetri_login} +# query = {"provisionedApps.CYMMETRI.login.login": cymmetri_login} + +# userRecord = user.find_one(query) +# return_val = {} +# cymmetri_info = {} +# cymmetri_info['login'] = userRecord['provisionedApps']['CYMMETRI']['login']['login'] +# cymmetri_info['displayName'] = userRecord['displayName'] +# cymmetri_info['firstName'] = userRecord['firstName'] +# cymmetri_info['lastName'] = userRecord['lastName'] +# cymmetri_info['status'] = userRecord['status'] +# cymmetri_info['email'] = userRecord['email'] +# cymmetri_info['createdDate'] = userRecord['created'] +# return_val['userInfo'] = cymmetri_info + +# break_info = {} +# break_info['message'] = 'User is present in Cymmetri but not in Application.' +# break_info['breakCount'] = shReconBreakRecords.count_documents(query_breakCount_Cymm) +# return_val['break_info'] = break_info +# return return_val + + + +# # If no matching record found +# except Exception as e: +# raise HTTPException(status_code=500, detail=str(e)) @app.post('/reconsrvc/break_data') async def break_data_table_records(request: BreakRecordWithSearchAndSortRequestSchema, tenant: str = Header(...)) -> Dict: From f4960c78a618c180a6466974538bd6a0c13bb132 Mon Sep 17 00:00:00 2001 From: Sahil Date: Wed, 24 Apr 2024 12:48:10 +0530 Subject: [PATCH 17/24] changed pipeline to display record in proper manner for matching_record API --- modal2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modal2.py b/modal2.py index c4fd194..6967c57 100644 --- a/modal2.py +++ b/modal2.py @@ -261,7 +261,7 @@ async def get_matching_records(request: MatchingRecordRequestSchema, tenant: str pipeline = [ {"$match": {"appId": request.appId, "createdAt": {"$exists": True}}}, {"$group": {"_id": "$reconciliationId", "maxStartTime": {"$max": "$createdAt"}, "latestDocument": {"$last": "$$ROOT"}}}, - {"$sort": {"_id": 1}}, + {"$sort": {"maxStartTime": -1}}, {"$skip": request.pageNumber * request.pageSize}, {"$limit": request.pageSize}, {"$replaceRoot": {"newRoot": "$latestDocument"}} From bacc5a8cc9a3a3ca0202af2a2962388cfb1d8b8a Mon Sep 17 00:00:00 2001 From: Sahil Date: Wed, 24 Apr 2024 13:10:07 +0530 Subject: [PATCH 18/24] did changes for bulk insertion in shReconbreakrecords --- database/watcher/watcher.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/database/watcher/watcher.py b/database/watcher/watcher.py index 2f9f8ff..6370826 100644 --- a/database/watcher/watcher.py +++ b/database/watcher/watcher.py @@ -302,6 +302,8 @@ def getBreakCount(tenant, reconciliationId, login): def execute_final_code(tenant, reconciliationId): + login_data = [] + logging.info(f"Starting final execution for tenant: {tenant}, reconciliationId: {reconciliationId}") shReconBreakRecords = get_collection(tenant, 'shReconBreakRecords') @@ -353,7 +355,11 @@ def execute_final_code(tenant, reconciliationId): "login":login, "break_count": login_break_count } - shReconBreakRecords.insert_one(record) + #shReconBreakRecords.insert_one(record) + + login_data.append(record) + + shReconBreakRecords.insert_many(login_data) logging.debug("Data Inserted into shReconBreakRecords succesfully.") break_types = ["present_in_target_only", "present_in_Cymmetri_only", "app_overdue"] From 68501a344e6adb41a7e7acfe86f0f34d13dd3d4f Mon Sep 17 00:00:00 2001 From: Sahil Date: Wed, 24 Apr 2024 15:08:23 +0530 Subject: [PATCH 19/24] added code for exception handeling where data is inserted into shReconBreakRecord collection --- database/watcher/watcher.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/database/watcher/watcher.py b/database/watcher/watcher.py index 6370826..3152d53 100644 --- a/database/watcher/watcher.py +++ b/database/watcher/watcher.py @@ -260,6 +260,7 @@ def process_new_sync_data_for_recon(tenant, new_document): login_break_count = getBreakCount(tenant,reconciliationId,login) + if login not in all_cymmetri_logins_redis_data: logging.debug(f"User login {login} found in target app only.") record = { @@ -340,7 +341,7 @@ def execute_final_code(tenant, reconciliationId): #only_present_in_Cymmetri = full_set_of_cymmetri_logins - full_set_of_target_logins #print(full_set_of_cymmetri_logins) - + for login in full_set_of_cymmetri_logins: #print(f'{login} only found in cymmetri') login_break_count = getBreakCount(tenant, reconciliationId,login) @@ -358,9 +359,14 @@ def execute_final_code(tenant, reconciliationId): #shReconBreakRecords.insert_one(record) login_data.append(record) + try: + shReconBreakRecords.insert_many(login_data) + logging.debug("Data Inserted into shReconBreakRecords succesfully.") + except Exception as e: + logging.error(f"Failed to insert data into shReconBreakRecords collection: {e}") - shReconBreakRecords.insert_many(login_data) - logging.debug("Data Inserted into shReconBreakRecords succesfully.") + + break_types = ["present_in_target_only", "present_in_Cymmetri_only", "app_overdue"] From 25c228c9c0ab48af77565b51999e81f6081de4ca Mon Sep 17 00:00:00 2001 From: Sahil Date: Wed, 24 Apr 2024 15:49:52 +0530 Subject: [PATCH 20/24] added code for bulk insertion for app_overdue and present_in_target_only and also added exception handeling for both --- database/watcher/watcher.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/database/watcher/watcher.py b/database/watcher/watcher.py index 3152d53..d6775b8 100644 --- a/database/watcher/watcher.py +++ b/database/watcher/watcher.py @@ -196,7 +196,6 @@ def process_new_sync_data_for_recon(tenant, new_document): redis_key = f"recon_{tenant}_{rcId}" logging.debug(f"Redis key found: {redis_key}") - recon_report_metadata_id = redis_client.hget(redis_key, 'reconReportMetadataId') reconciliationId = redis_client.hget(redis_key, "reconciliationId") #batch_id = redis_client.hget(key, 'batchId') appId = redis_client.hget(redis_key, 'appId') @@ -259,8 +258,8 @@ def process_new_sync_data_for_recon(tenant, new_document): returnval = True login_break_count = getBreakCount(tenant,reconciliationId,login) - - + + present_in_target_only = [] if login not in all_cymmetri_logins_redis_data: logging.debug(f"User login {login} found in target app only.") record = { @@ -274,10 +273,18 @@ def process_new_sync_data_for_recon(tenant, new_document): "login":login, "break_count":login_break_count } - shReconBreakRecords.insert_one(record) - + #shReconBreakRecords.insert_one(record) + present_in_target_only.append(record) + try: + shReconBreakRecords.insert_many(present_in_target_only) + logging.debug("Data Inserted for present_in_target_only into shReconBreakRecords succesfully.") + except Exception as e: + logging.error(f"Failed to insert data for present_in_target_only into shReconBreakRecords collection: {e}") + + + app_overdue = [] if login in final_app_overdue_redis_data: logging.debug(f'user {login} should have been disabled but is still in target app') record = { @@ -291,7 +298,14 @@ def process_new_sync_data_for_recon(tenant, new_document): "login":login, "break_count": login_break_count } - shReconBreakRecords.insert_one(record) + #shReconBreakRecords.insert_one(record) + app_overdue.append(record) + try: + shReconBreakRecords.insert_many(app_overdue) + logging.debug("Data Inserted for app_overdue into shReconBreakRecords succesfully.") + except Exception as e: + logging.error(f"Failed to insert data for app_overdue into shReconBreakRecords collection: {e}") + logging.debug("Process for syncdata for recon insert data is completed.") return returnval From e4c930e395da2c446a25a4fa8938cd8153fedaba Mon Sep 17 00:00:00 2001 From: Sahil Date: Wed, 24 Apr 2024 15:59:24 +0530 Subject: [PATCH 21/24] remove printing stament from modal api which wad added previously for testing purpose --- modal2.py | 1 - 1 file changed, 1 deletion(-) diff --git a/modal2.py b/modal2.py index 6967c57..ea89a0a 100644 --- a/modal2.py +++ b/modal2.py @@ -390,7 +390,6 @@ async def populating_data(request: ModalRequestSchema, tenant: str = Header(...) 'breakCount': shReconBreakRecords.count_documents(query_breakCount_Cymm) } } - print("return_val",return_val) logging.debug(f"Return value prepared: {return_val}") data_response = return_val return ResponseModel(data=data_response, message="Policy mapping generated successfully") From 1678aaa97b9a4203b5bb564d43f015c8479328a9 Mon Sep 17 00:00:00 2001 From: Sahil Date: Wed, 24 Apr 2024 16:04:07 +0530 Subject: [PATCH 22/24] removed commneted code --- modal2.py | 62 ------------------------------------------------------- 1 file changed, 62 deletions(-) diff --git a/modal2.py b/modal2.py index ea89a0a..c0efbc2 100644 --- a/modal2.py +++ b/modal2.py @@ -398,69 +398,7 @@ async def populating_data(request: ModalRequestSchema, tenant: str = Header(...) logging.error(f"An error occurred: {str(e)}") return ErrorResponseModel(error=str(e), code=500, message="Exception Occured.", errorCode= "Invalid") -# @app.post('/reconsrvc/modal') -# async def populating_data(request: ModalRequestSchema, tenant: str = Header(...)) -> Response: -# # Connect to MongoDB -# shReconBreakRecords = get_collection(tenant, "shReconBreakRecords") -# syncDataForRecon = get_collection(tenant, "syncDataForRecon") -# user = get_collection(tenant, "user") - -# try: - -# # Convert _id from string to ObjectId -# object_id = bson.ObjectId(request.Id) - -# breakReconRecord = shReconBreakRecords.find_one({"_id": object_id}) -# ##print(breakReconRecord) - -# if breakReconRecord['break_type'] == 'present_in_target_only' or breakReconRecord['break_type'] == 'app_overdue': -# # Fetch appId from the record -# app_id = breakReconRecord['target_app_login'] -# query = {"reconciliationId": breakReconRecord['reconciliationId'], "data.login": app_id} -# query_breakCount = {"reconciliationId": breakReconRecord['reconciliationId'], "target_app_login": app_id} - -# sort_criteria = [("updatedDateTime", DESCENDING)] -# records = syncDataForRecon.find(query).sort(sort_criteria).limit(1) -# return_val = {} -# for single_record in records: -# return_val['userInfo'] = single_record['data'] -# break_info = {} -# if breakReconRecord['break_type'] == 'app_overdue': -# break_info['message'] = 'User overdue to be disabled but present in Application.' -# else: -# break_info['message'] = 'User only present in application.' -# break_info['breakCount'] = shReconBreakRecords.count_documents(query_breakCount) -# return_val['break_info'] = break_info -# return return_val -# else: -# cymmetri_login = breakReconRecord['cymmetri_login'] -# query_breakCount_Cymm = {"reconciliationId": breakReconRecord['reconciliationId'], "cymmetri_login": cymmetri_login} -# query = {"provisionedApps.CYMMETRI.login.login": cymmetri_login} - -# userRecord = user.find_one(query) -# return_val = {} -# cymmetri_info = {} -# cymmetri_info['login'] = userRecord['provisionedApps']['CYMMETRI']['login']['login'] -# cymmetri_info['displayName'] = userRecord['displayName'] -# cymmetri_info['firstName'] = userRecord['firstName'] -# cymmetri_info['lastName'] = userRecord['lastName'] -# cymmetri_info['status'] = userRecord['status'] -# cymmetri_info['email'] = userRecord['email'] -# cymmetri_info['createdDate'] = userRecord['created'] -# return_val['userInfo'] = cymmetri_info - -# break_info = {} -# break_info['message'] = 'User is present in Cymmetri but not in Application.' -# break_info['breakCount'] = shReconBreakRecords.count_documents(query_breakCount_Cymm) -# return_val['break_info'] = break_info -# return return_val - - - -# # If no matching record found -# except Exception as e: -# raise HTTPException(status_code=500, detail=str(e)) @app.post('/reconsrvc/break_data') async def break_data_table_records(request: BreakRecordWithSearchAndSortRequestSchema, tenant: str = Header(...)) -> Dict: From b17ef6a61d00f11268e0a7ac1c0160da03eec1d4 Mon Sep 17 00:00:00 2001 From: Sahil Date: Wed, 24 Apr 2024 17:25:17 +0530 Subject: [PATCH 23/24] added few logging staments to check the len of break types --- database/watcher/watcher.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/database/watcher/watcher.py b/database/watcher/watcher.py index d6775b8..4df2d29 100644 --- a/database/watcher/watcher.py +++ b/database/watcher/watcher.py @@ -275,6 +275,7 @@ def process_new_sync_data_for_recon(tenant, new_document): } #shReconBreakRecords.insert_one(record) present_in_target_only.append(record) + logging.debug("len of record for present_in_target_only: ",len(present_in_target_only)) try: shReconBreakRecords.insert_many(present_in_target_only) logging.debug("Data Inserted for present_in_target_only into shReconBreakRecords succesfully.") @@ -300,6 +301,7 @@ def process_new_sync_data_for_recon(tenant, new_document): } #shReconBreakRecords.insert_one(record) app_overdue.append(record) + logging.debug("len of records for app_overdue ",len(app_overdue)) try: shReconBreakRecords.insert_many(app_overdue) logging.debug("Data Inserted for app_overdue into shReconBreakRecords succesfully.") @@ -373,6 +375,7 @@ def execute_final_code(tenant, reconciliationId): #shReconBreakRecords.insert_one(record) login_data.append(record) + logging.debug("len of records for present_in_Cymmetri_only: ",len(login_data)) try: shReconBreakRecords.insert_many(login_data) logging.debug("Data Inserted into shReconBreakRecords succesfully.") From 7f35cfa9d28a7e234edc62c8f569b68ab9e86cca Mon Sep 17 00:00:00 2001 From: Sahil Date: Wed, 24 Apr 2024 18:23:02 +0530 Subject: [PATCH 24/24] removed unwanted logging statements from break_types --- database/watcher/watcher.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/database/watcher/watcher.py b/database/watcher/watcher.py index 4df2d29..d6775b8 100644 --- a/database/watcher/watcher.py +++ b/database/watcher/watcher.py @@ -275,7 +275,6 @@ def process_new_sync_data_for_recon(tenant, new_document): } #shReconBreakRecords.insert_one(record) present_in_target_only.append(record) - logging.debug("len of record for present_in_target_only: ",len(present_in_target_only)) try: shReconBreakRecords.insert_many(present_in_target_only) logging.debug("Data Inserted for present_in_target_only into shReconBreakRecords succesfully.") @@ -301,7 +300,6 @@ def process_new_sync_data_for_recon(tenant, new_document): } #shReconBreakRecords.insert_one(record) app_overdue.append(record) - logging.debug("len of records for app_overdue ",len(app_overdue)) try: shReconBreakRecords.insert_many(app_overdue) logging.debug("Data Inserted for app_overdue into shReconBreakRecords succesfully.") @@ -375,7 +373,6 @@ def execute_final_code(tenant, reconciliationId): #shReconBreakRecords.insert_one(record) login_data.append(record) - logging.debug("len of records for present_in_Cymmetri_only: ",len(login_data)) try: shReconBreakRecords.insert_many(login_data) logging.debug("Data Inserted into shReconBreakRecords succesfully.")