From 90a5eea79c1e50c98f692242b2c61764fe6764b4 Mon Sep 17 00:00:00 2001 From: pjpatil12 Date: Tue, 23 Dec 2025 12:20:38 +0530 Subject: [PATCH] client models modification and client import file with RA bill no changes commits --- app/models/mh_dc_client_model.py | 2 +- app/models/mh_ex_client_model.py | 2 +- app/models/tr_ex_client_model.py | 10 +- app/services/file_service.py | 1142 +++++++++++++++++++----------- 4 files changed, 738 insertions(+), 418 deletions(-) diff --git a/app/models/mh_dc_client_model.py b/app/models/mh_dc_client_model.py index b8b37a8..dd200b8 100644 --- a/app/models/mh_dc_client_model.py +++ b/app/models/mh_dc_client_model.py @@ -13,7 +13,7 @@ class ManholeDomesticChamberClient(db.Model): # Basic Fields RA_Bill_No=db.Column(db.String(500)) Location = db.Column(db.String(500)) - Node_No = db.Column(db.String(100)) + MH_NO = db.Column(db.String(100)) MH_TOP_LEVEL = db.Column(db.Float) MH_IL_LEVEL = db.Column(db.Float) Depth_of_MH = db.Column(db.Float) diff --git a/app/models/mh_ex_client_model.py b/app/models/mh_ex_client_model.py index 172b0ac..603cacb 100644 --- a/app/models/mh_ex_client_model.py +++ b/app/models/mh_ex_client_model.py @@ -14,7 +14,7 @@ class ManholeExcavationClient(db.Model): RA_Bill_No=db.Column(db.String(500)) Location = db.Column(db.String(500)) MH_NO = db.Column(db.String(100)) - GL_m = db.Column(db.Float) + Ground_Level = db.Column(db.Float) MH_Invert_Level = db.Column(db.Float) MH_Top_Level = db.Column(db.Float) Ex_Level = db.Column(db.Float) diff --git a/app/models/tr_ex_client_model.py b/app/models/tr_ex_client_model.py index 0674967..608e5dc 100644 --- a/app/models/tr_ex_client_model.py +++ b/app/models/tr_ex_client_model.py @@ -24,11 +24,11 @@ class TrenchExcavationClient(db.Model): Pipe_Dia_mm = db.Column(db.Float) # width - Width_0_to_1_5 = db.Column(db.Float) - Width_1_5_to_3_0 = db.Column(db.Float) - Width_3_0_to_4_5 = db.Column(db.Float) - Width_4_5_to_6_0 = db.Column(db.Float) - + Width_0_to_1_5_m = db.Column(db.Float) + Width_1_5_to_3_0_m = db.Column(db.Float) + Width_3_0_to_4_5_m = db.Column(db.Float) + Width_4_5_to_6_0_m = db.Column(db.Float) + Width_6_0_to_7_5_m = db.Column(db.Float) # Excavation categories Marshi_Muddy_Slushy_0_to_1_5 = db.Column(db.Float) diff --git a/app/services/file_service.py b/app/services/file_service.py index f538157..d3b3b0a 100644 --- a/app/services/file_service.py +++ b/app/services/file_service.py @@ -1,8 +1,574 @@ +# import os +# import pandas as pd +# from werkzeug.utils import secure_filename +# from app.config import Config +# from app import db +# from app.models.trench_excavation_model import TrenchExcavation +# from app.models.manhole_excavation_model import ManholeExcavation +# from app.models.manhole_domestic_chamber_model import ManholeDomesticChamber + +# from app.models.tr_ex_client_model import TrenchExcavationClient +# from app.models.mh_ex_client_model import ManholeExcavationClient +# from app.models.mh_dc_client_model import ManholeDomesticChamberClient + + +# from app.utils.file_utils import ensure_upload_folder + + +# class FileService: + +# def allowed_file(self, filename): +# return "." in filename and filename.rsplit(".", 1)[1].lower() in Config.ALLOWED_EXTENSIONS + +# # def handle_file_upload(self, file, subcontractor_id, file_type): +# def handle_file_upload(self, file, subcontractor_id, RA_Bill_No): + +# if not subcontractor_id: +# return False, "Please select subcontractor." +# if not RA_Bill_No: +# return False, "Please Enter RA Bill No)." +# if not file or file.filename == "": +# return False, "No file selected." +# if not self.allowed_file(file.filename): +# return False, "Invalid file type! Allowed: CSV, XLSX, XLS" + +# ensure_upload_folder() + +# folder = os.path.join(Config.UPLOAD_FOLDER, f"sub_{subcontractor_id}") +# os.makedirs(folder, exist_ok=True) + +# filename = secure_filename(file.filename) +# filepath = os.path.join(folder, filename) +# file.save(filepath) + +# try: + +# df_tr_ex = pd.read_excel(filepath, sheet_name ="Tr.Ex.", header=12) +# df_mh_ex = pd.read_excel(filepath, sheet_name="MH Ex.", header=12) +# df_mh_dc = pd.read_excel(filepath, sheet_name="MH & DC", header=11) + +# self.process_trench_excavation(df_tr_ex, subcontractor_id, RA_Bill_No) +# self.process_manhole_excavation(df_mh_ex, subcontractor_id, RA_Bill_No) +# self.process_manhole_domestic_chamber(df_mh_dc, subcontractor_id, RA_Bill_No) + +# return True, "File uploaded successfully." + +# except Exception as e: +# return False, f"Processing failed: {e}" + + +# # Trench Excavation save method (TrenchExcavation model) +# def process_trench_excavation(self, df, subcontractor_id,RA_Bill_No): + +# print("trench_excavation RA_Bill_No :",RA_Bill_No) + +# print("=== trench_excavation ===") +# print(df.columns.tolist()) +# print("===================") + +# # Clean column names +# df.columns = ( +# df.columns.astype(str) +# .str.strip() +# .str.replace(r"[^\w]", "_", regex=True) +# .str.replace("__+", "_", regex=True) +# .str.strip("_") +# ) + +# # Remove completely empty rows +# df = df.dropna(how="all") + +# # Forward fill merged Location +# if "Location" in df.columns: +# df["Location"] = df["Location"].ffill() + +# saved_count = 0 +# skipped_count = 0 + +# try: +# for index, row in df.iterrows(): +# record_data = {} +# location = row.get("Location") +# mh_no = row.get("MH_NO") + +# if (pd.isna(location) or str(location).strip() == "" or pd.isna(mh_no) or str(mh_no).strip() == ""): +# skipped_count += 1 +# continue + +# # Map only model columns +# for col in df.columns: +# if hasattr(TrenchExcavation, col): +# value = row[col] + +# # Normalize empty values +# if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan"]: +# value = None + +# record_data[col] = value + +# # If all mapped fields are None → skip +# if all(v is None for v in record_data.values()): +# skipped_count += 1 +# continue + +# record = TrenchExcavation( +# subcontractor_id=subcontractor_id, RA_Bill_No=RA_Bill_No, +# **record_data +# ) + +# print("trench_excavation Saving Row → Location:", record.Location, " MH_NO:", record.MH_NO) + +# db.session.add(record) +# saved_count += 1 + +# db.session.commit() + +# return True, ( +# f"Trench Excavation saved successfully. " +# f"Inserted: {saved_count}, Skipped: {skipped_count}" +# ) + +# except Exception as e: +# db.session.rollback() +# return False, f"Trench Excavation save failed: {e}" + + +# # Manhole Excavation save method (ManholeExcavation model) +# def process_manhole_excavation(self, df, subcontractor_id, RA_Bill_No): + +# print("manhole_excavation RA_Bill_No:",RA_Bill_No) + +# print("=== manhole_excavation ===") +# print(df.columns.tolist()) +# print("===================") + +# # Clean column names +# df.columns = ( +# df.columns.astype(str) +# .str.strip() +# .str.replace(r"[^\w]", "_", regex=True) +# .str.replace("__+", "_", regex=True) +# .str.strip("_") +# ) + +# # Remove completely empty rows +# df = df.dropna(how="all") + +# # Forward fill merged Location +# if "Location" in df.columns: +# df["Location"] = df["Location"].ffill() + +# saved_count = 0 +# skipped_count = 0 + +# try: +# for index, row in df.iterrows(): +# record_data = {} +# location = row.get("Location") +# mh_no = row.get("MH_NO") + +# if (pd.isna(location) or str(location).strip() == "" or pd.isna(mh_no) or str(mh_no).strip() == ""): +# skipped_count += 1 +# continue + +# # Map only model columns +# for col in df.columns: +# if hasattr(ManholeExcavation, col): +# value = row[col] + +# # Normalize empty values +# if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan"]: +# value = None + +# record_data[col] = value + +# # If all mapped fields are None → skip +# if all(v is None for v in record_data.values()): +# skipped_count += 1 +# continue + +# record = ManholeExcavation( +# subcontractor_id=subcontractor_id, RA_Bill_No=RA_Bill_No, +# **record_data +# ) + +# print("manhole_excavation Saving Row → Location:", record.Location, " MH_NO:", record.MH_NO) + +# db.session.add(record) +# saved_count += 1 + +# db.session.commit() + +# return True, ( +# f"Manhole Excavation saved successfully. " +# f"Inserted: {saved_count}, Skipped: {skipped_count}" +# ) + +# except Exception as e: +# db.session.rollback() +# return False, f"Manhole Excavation save failed: {e}" + +# # Manhole and Domestic Chamber Construction save method (ManholeDomesticChamber model) +# def process_manhole_domestic_chamber(self, df, subcontractor_id, RA_Bill_No): + +# print("manhole_domestic_chamber RA_Bill_No :",RA_Bill_No) + +# print("=== manhole_domestic_chamber ===") +# print(df.columns.tolist()) +# print("===================") + +# # Clean column names +# df.columns = ( +# df.columns.astype(str) +# .str.strip() +# .str.replace(r"[^\w]", "_", regex=True) +# .str.replace("__+", "_", regex=True) +# .str.strip("_") +# ) + +# # Remove completely empty rows +# df = df.dropna(how="all") + +# # Forward fill merged Location +# if "Location" in df.columns: +# df["Location"] = df["Location"].ffill() + +# saved_count = 0 +# skipped_count = 0 + +# try: +# for index, row in df.iterrows(): +# record_data = {} +# location = row.get("Location") +# Node_No = row.get("Node_No") + +# if (pd.isna(location) or str(location).strip() == "" or pd.isna(Node_No) or str(Node_No).strip() == ""): +# skipped_count += 1 +# continue + +# # Map only model columns +# for col in df.columns: +# if hasattr(ManholeDomesticChamber, col): +# value = row[col] + +# # Normalize empty values +# if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan"]: +# value = None + +# record_data[col] = value + +# # If all mapped fields are None → skip +# if all(v is None for v in record_data.values()): +# skipped_count += 1 +# continue + +# record = ManholeDomesticChamber( +# subcontractor_id=subcontractor_id, RA_Bill_No=RA_Bill_No, +# **record_data +# ) + +# print("manhole_domestic_chamber Saving Row → Location:", record.Location, " Node_No:", record.Node_No) + +# db.session.add(record) +# saved_count += 1 + +# db.session.commit() + +# return True, ( +# f"Manhole Domestic Chamber saved successfully. " +# f"Inserted: {saved_count}, Skipped: {skipped_count}" +# ) + +# except Exception as e: +# db.session.rollback() +# return False, f"Manhole Domestic Chamber save failed: {e}" + + + +# # ---------------------- client ---------------------------------- + +# # def handle_client_file_upload(self, file, RA_Bill_No): + +# # if not RA_Bill_No: +# # return False, "Please Enter RA Bill No ." + +# # if not file or file.filename == "": +# # return False, "No file selected." +# # if not self.allowed_file(file.filename): +# # return False, "Invalid file type! Allowed: CSV, XLSX, XLS" + +# # ensure_upload_folder() + +# # folder = os.path.join(Config.UPLOAD_FOLDER, f"Client_Bill_{RA_Bill_No}") +# # os.makedirs(folder, exist_ok=True) + +# # filename = secure_filename(file.filename) +# # filepath = os.path.join(folder, filename) +# # file.save(filepath) + +# # try: +# # df_tr_ex = pd.read_excel(filepath, sheet_name ="Tr.Ex.", header=12) +# # df_mh_ex = pd.read_excel(filepath, sheet_name="MH Ex.", header=12) +# # df_mh_dc = pd.read_excel(filepath, sheet_name="MH & DC", header=11) + +# # print("\n=== Uploaded File tr ex ===") +# # print(df_tr_ex.head()) +# # print("=============================\n") +# # print("\n=== Uploaded File mh ex ===") +# # print(df_mh_ex.head()) +# # print("========================================\n") +# # print("\n=== Uploaded File MH DC ===") +# # print(df_mh_dc.head()) +# # print("========================================\n") + + +# # self.client_trench_excavation(df_tr_ex, RA_Bill_No) +# # self.client_manhole_excavation(df_mh_ex, RA_Bill_No) +# # self.client_manhole_domestic_chamber(df_mh_dc, RA_Bill_No) + +# # return True, "File uploaded successfully." + +# # except Exception as e: +# # return False, f"Processing failed: {e}" + +# # # Tr Ex save method (TrenchExcavationClient model) +# # def client_trench_excavation(self, df, RA_Bill_No): +# # df.columns = [str(c).strip() for c in df.columns] +# # # If the sheet has merged cells -> forward fill Location +# # if "Location" in df.columns: +# # df["Location"] = df["Location"].ffill() + +# # df = df.dropna(how="all") # REMOVE empty rows +# # # Identify missing location rows before insert +# # missing_loc = df[df["Location"].isna() | (df["Location"].astype(str).str.strip() == "")] +# # if not missing_loc.empty: +# # return False, f"Error: Some rows have empty Location. Rows: {missing_loc.index.tolist()}" + +# # saved_count = 0 + +# # try: +# # for index, row in df.iterrows(): +# # record_data = {} +# # # Insert only fields that exist in model +# # for col in df.columns: +# # if hasattr(TrenchExcavationClient, col): +# # value = row[col] +# # # Normalize empty values +# # if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan", "NaN"]: +# # value = None + +# # record_data[col] = value + +# # record = TrenchExcavationClient( +# # RA_Bill_No=RA_Bill_No, +# # **record_data +# # ) + +# # db.session.add(record) +# # saved_count += 1 + +# # db.session.commit() +# # return True, f"Clinnt Tr Ex data saved successfully. Total rows: {saved_count}" + +# # except Exception as e: +# # db.session.rollback() +# # return False, f"Clinnt Tr Ex Save Failed: {e}" + + +# # # Mh Ex save method (ManholeExcavationClient model) +# # def client_manhole_excavation(self, df, RA_Bill_No): +# # # Clean column names (strip whitespace) +# # df.columns = [str(c).strip() for c in df.columns] +# # # If the sheet has merged cells -> forward fill Location +# # if "Location" in df.columns: +# # df["Location"] = df["Location"].ffill() + +# # # REMOVE empty rows +# # df = df.dropna(how="all") +# # # Identify missing location rows before insert +# # missing_loc = df[df["Location"].isna() | (df["Location"].astype(str).str.strip() == "")] +# # if not missing_loc.empty: +# # return False, f"Error: Some rows have empty Location. Rows: {missing_loc.index.tolist()}" + +# # saved_count = 0 + +# # try: +# # for index, row in df.iterrows(): +# # record_data = {} +# # # Insert only fields that exist in model +# # for col in df.columns: +# # if hasattr(ManholeExcavationClient, col): +# # value = row[col] + +# # # Normalize empty values +# # if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan", "NaN"]: +# # value = None + +# # record_data[col] = value + +# # record = ManholeExcavationClient( +# # RA_Bill_No=RA_Bill_No, +# # **record_data +# # ) + +# # db.session.add(record) +# # saved_count += 1 + +# # db.session.commit() +# # return True, f" Client Mh Ex. data saved successfully. Total rows: {saved_count}" + +# # except Exception as e: +# # db.session.rollback() +# # return False, f"Client Mh Ex. Save Failed: {e}" + + +# # # Mh and Dc save method (ManholeDomesticChamberClient model) +# # def client_manhole_domestic_chamber(self, df, RA_Bill_No): +# # Clean column names (strip whitespace) +# # df.columns = [str(c).strip() for c in df.columns] +# # # If the sheet has merged cells -> forward fill Location +# # if "Location" in df.columns: +# # df["Location"] = df["Location"].ffill() + +# # # REMOVE empty rows +# # df = df.dropna(how="all") +# # # Identify missing location rows before insert +# # missing_loc = df[df["Location"].isna() | (df["Location"].astype(str).str.strip() == "")] +# # if not missing_loc.empty: +# # return False, f"Error: Some rows have empty Location. Rows: {missing_loc.index.tolist()}" + +# # saved_count = 0 + +# # try: +# # for index, row in df.iterrows(): +# # record_data = {} + +# # # Insert only fields that exist in model +# # for col in df.columns: +# # if hasattr(ManholeDomesticChamberClient, col): +# # value = row[col] +# # # Normalize empty values +# # if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan", "NaN"]: +# # value = None + +# # record_data[col] = value + +# # record = ManholeDomesticChamberClient( +# # RA_Bill_No=RA_Bill_No, +# # **record_data +# # ) + +# # db.session.add(record) +# # saved_count += 1 + +# # db.session.commit() +# # return True, f"Mh and Dc data saved successfully. Total rows: {saved_count}" + +# # except Exception as e: +# # db.session.rollback() +# # return False, f"Mh and Dc data Save Failed: {e}" + +# def handle_client_file_upload(self, file, RA_Bill_No): + +# if not RA_Bill_No: +# return False, "Please Enter RA Bill No." + +# if not file or file.filename == "": +# return False, "No file selected." + +# if not self.allowed_file(file.filename): +# return False, "Invalid file type! Allowed: CSV, XLSX, XLS" + +# ensure_upload_folder() + +# folder = os.path.join(Config.UPLOAD_FOLDER, f"Client_Bill_{RA_Bill_No}") +# os.makedirs(folder, exist_ok=True) + +# filename = secure_filename(file.filename) +# filepath = os.path.join(folder, filename) +# file.save(filepath) + +# try: +# df_tr_ex = pd.read_excel(filepath, sheet_name="Tr.Ex.", header=4) +# df_mh_ex = pd.read_excel(filepath, sheet_name="MH Ex.", header=4) +# df_mh_dc = pd.read_excel(filepath, sheet_name="MH & DC", header=3) +# print("------------------") +# print("df_tr_ex :",df_tr_ex) +# print("------------------") +# print("df_mh_ex :",df_mh_ex) +# print("------------------") +# print("df_mh_dc :",df_mh_dc) +# print("------------------") + +# # Single transaction +# self._save_client_data(df_tr_ex, TrenchExcavationClient, "Trench Excavation", RA_Bill_No) +# self._save_client_data(df_mh_ex, ManholeExcavationClient, "Manhole Excavation", RA_Bill_No) +# self._save_client_data(df_mh_dc, ManholeDomesticChamberClient, "MH & DC", RA_Bill_No) + +# db.session.commit() +# return True, "Client file uploaded and all data saved successfully." + +# except ClientDataSaveError as e: +# db.session.rollback() +# return False, str(e) + +# except Exception as e: +# db.session.rollback() +# return False, f"Unexpected system error: {e}" + + +# def _save_client_data(self, df, model, RA_Bill_No): + +# # Clean column names +# df.columns = [str(c).strip() for c in df.columns] + +# # Forward fill Location (merged cells issue) +# if "Location" in df.columns: +# df["Location"] = df["Location"].ffill() + +# # Remove fully empty rows +# df = df.dropna(how="all") + +# # Validate Location +# if "Location" in df.columns: +# missing_loc = df[ +# df["Location"].isna() | +# (df["Location"].astype(str).str.strip() == "") +# ] +# if not missing_loc.empty: +# raise Exception( +# f"{model.__name__}: Empty Location at rows {missing_loc.index.tolist()}" +# ) + +# # Insert rows +# for _, row in df.iterrows(): +# record_data = {} + +# for col in df.columns: +# if hasattr(model, col): +# value = row[col] + +# if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan", "NaN"]: +# value = None + +# record_data[col] = value + +# record = model( +# RA_Bill_No=RA_Bill_No, +# **record_data +# ) + +# db.session.add(record) + + + import os import pandas as pd from werkzeug.utils import secure_filename + from app.config import Config from app import db + from app.models.trench_excavation_model import TrenchExcavation from app.models.manhole_excavation_model import ManholeExcavation from app.models.manhole_domestic_chamber_model import ManholeDomesticChamber @@ -11,24 +577,34 @@ from app.models.tr_ex_client_model import TrenchExcavationClient from app.models.mh_ex_client_model import ManholeExcavationClient from app.models.mh_dc_client_model import ManholeDomesticChamberClient - from app.utils.file_utils import ensure_upload_folder class FileService: + # --------------------------------------------------- + # Common helpers + # --------------------------------------------------- def allowed_file(self, filename): - return "." in filename and filename.rsplit(".", 1)[1].lower() in Config.ALLOWED_EXTENSIONS + return ( + "." in filename + and filename.rsplit(".", 1)[1].lower() in Config.ALLOWED_EXTENSIONS + ) - # def handle_file_upload(self, file, subcontractor_id, file_type): + # =================================================== + # SUBCONTRACTOR FILE UPLOAD + # =================================================== def handle_file_upload(self, file, subcontractor_id, RA_Bill_No): if not subcontractor_id: return False, "Please select subcontractor." + if not RA_Bill_No: - return False, "Please Enter RA Bill No)." + return False, "Please Enter RA Bill No." + if not file or file.filename == "": return False, "No file selected." + if not self.allowed_file(file.filename): return False, "Invalid file type! Allowed: CSV, XLSX, XLS" @@ -42,8 +618,7 @@ class FileService: file.save(filepath) try: - - df_tr_ex = pd.read_excel(filepath, sheet_name ="Tr.Ex.", header=12) + df_tr_ex = pd.read_excel(filepath, sheet_name="Tr.Ex.", header=12) df_mh_ex = pd.read_excel(filepath, sheet_name="MH Ex.", header=12) df_mh_dc = pd.read_excel(filepath, sheet_name="MH & DC", header=11) @@ -54,19 +629,14 @@ class FileService: return True, "File uploaded successfully." except Exception as e: + db.session.rollback() return False, f"Processing failed: {e}" - - # Trench Excavation save method (TrenchExcavation model) - def process_trench_excavation(self, df, subcontractor_id,RA_Bill_No): + # --------------------------------------------------- + # Trench Excavation (Subcontractor) + # --------------------------------------------------- + def process_trench_excavation(self, df, subcontractor_id, RA_Bill_No): - print("trench_excavation RA_Bill_No :",RA_Bill_No) - - print("=== trench_excavation ===") - print(df.columns.tolist()) - print("===================") - - # Clean column names df.columns = ( df.columns.astype(str) .str.strip() @@ -75,399 +645,155 @@ class FileService: .str.strip("_") ) - # Remove completely empty rows df = df.dropna(how="all") - # Forward fill merged Location if "Location" in df.columns: df["Location"] = df["Location"].ffill() - saved_count = 0 - skipped_count = 0 - try: - for index, row in df.iterrows(): - record_data = {} + for _, row in df.iterrows(): location = row.get("Location") mh_no = row.get("MH_NO") - if (pd.isna(location) or str(location).strip() == "" or pd.isna(mh_no) or str(mh_no).strip() == ""): - skipped_count += 1 + if ( + pd.isna(location) + or str(location).strip() == "" + or pd.isna(mh_no) + or str(mh_no).strip() == "" + ): continue - # Map only model columns + record_data = {} for col in df.columns: if hasattr(TrenchExcavation, col): - value = row[col] - - # Normalize empty values - if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan"]: - value = None - - record_data[col] = value - - # If all mapped fields are None → skip - if all(v is None for v in record_data.values()): - skipped_count += 1 - continue + val = row[col] + if pd.isna(val) or str(val).strip() in ["", "-", "—", "nan"]: + val = None + record_data[col] = val record = TrenchExcavation( - subcontractor_id=subcontractor_id, RA_Bill_No=RA_Bill_No, - **record_data + subcontractor_id=subcontractor_id, + RA_Bill_No=RA_Bill_No, + **record_data, ) - - print("trench_excavation Saving Row → Location:", record.Location, " MH_NO:", record.MH_NO) - db.session.add(record) - saved_count += 1 db.session.commit() - - return True, ( - f"Trench Excavation saved successfully. " - f"Inserted: {saved_count}, Skipped: {skipped_count}" - ) + return True, "Trench Excavation saved successfully." except Exception as e: db.session.rollback() - return False, f"Trench Excavation save failed: {e}" - + raise e - # Manhole Excavation save method (ManholeExcavation model) + # --------------------------------------------------- + # Manhole Excavation (Subcontractor) + # --------------------------------------------------- def process_manhole_excavation(self, df, subcontractor_id, RA_Bill_No): - print("manhole_excavation RA_Bill_No:",RA_Bill_No) + df.columns = ( + df.columns.astype(str) + .str.strip() + .str.replace(r"[^\w]", "_", regex=True) + .str.replace("__+", "_", regex=True) + .str.strip("_") + ) - print("=== manhole_excavation ===") - print(df.columns.tolist()) - print("===================") + df = df.dropna(how="all") - # Clean column names - df.columns = ( - df.columns.astype(str) - .str.strip() - .str.replace(r"[^\w]", "_", regex=True) - .str.replace("__+", "_", regex=True) - .str.strip("_") - ) + if "Location" in df.columns: + df["Location"] = df["Location"].ffill() - # Remove completely empty rows - df = df.dropna(how="all") + try: + for _, row in df.iterrows(): + location = row.get("Location") + mh_no = row.get("MH_NO") - # Forward fill merged Location - if "Location" in df.columns: - df["Location"] = df["Location"].ffill() + if ( + pd.isna(location) + or str(location).strip() == "" + or pd.isna(mh_no) + or str(mh_no).strip() == "" + ): + continue - saved_count = 0 - skipped_count = 0 + record_data = {} + for col in df.columns: + if hasattr(ManholeExcavation, col): + val = row[col] + if pd.isna(val) or str(val).strip() in ["", "-", "—", "nan"]: + val = None + record_data[col] = val - try: - for index, row in df.iterrows(): - record_data = {} - location = row.get("Location") - mh_no = row.get("MH_NO") - - if (pd.isna(location) or str(location).strip() == "" or pd.isna(mh_no) or str(mh_no).strip() == ""): - skipped_count += 1 - continue - - # Map only model columns - for col in df.columns: - if hasattr(ManholeExcavation, col): - value = row[col] - - # Normalize empty values - if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan"]: - value = None - - record_data[col] = value - - # If all mapped fields are None → skip - if all(v is None for v in record_data.values()): - skipped_count += 1 - continue - - record = ManholeExcavation( - subcontractor_id=subcontractor_id, RA_Bill_No=RA_Bill_No, - **record_data - ) - - print("manhole_excavation Saving Row → Location:", record.Location, " MH_NO:", record.MH_NO) - - db.session.add(record) - saved_count += 1 - - db.session.commit() - - return True, ( - f"Manhole Excavation saved successfully. " - f"Inserted: {saved_count}, Skipped: {skipped_count}" + record = ManholeExcavation( + subcontractor_id=subcontractor_id, + RA_Bill_No=RA_Bill_No, + **record_data, ) + db.session.add(record) - except Exception as e: - db.session.rollback() - return False, f"Manhole Excavation save failed: {e}" + db.session.commit() + return True, "Manhole Excavation saved successfully." - # Manhole and Domestic Chamber Construction save method (ManholeDomesticChamber model) + except Exception as e: + db.session.rollback() + raise e + + # --------------------------------------------------- + # Manhole & Domestic Chamber (Subcontractor) + # --------------------------------------------------- def process_manhole_domestic_chamber(self, df, subcontractor_id, RA_Bill_No): - print("manhole_domestic_chamber RA_Bill_No :",RA_Bill_No) + df.columns = ( + df.columns.astype(str) + .str.strip() + .str.replace(r"[^\w]", "_", regex=True) + .str.replace("__+", "_", regex=True) + .str.strip("_") + ) - print("=== manhole_domestic_chamber ===") - print(df.columns.tolist()) - print("===================") + df = df.dropna(how="all") - # Clean column names - df.columns = ( - df.columns.astype(str) - .str.strip() - .str.replace(r"[^\w]", "_", regex=True) - .str.replace("__+", "_", regex=True) - .str.strip("_") - ) + if "Location" in df.columns: + df["Location"] = df["Location"].ffill() - # Remove completely empty rows - df = df.dropna(how="all") + try: + for _, row in df.iterrows(): + location = row.get("Location") + node_no = row.get("Node_No") - # Forward fill merged Location - if "Location" in df.columns: - df["Location"] = df["Location"].ffill() + if ( + pd.isna(location) + or str(location).strip() == "" + or pd.isna(node_no) + or str(node_no).strip() == "" + ): + continue - saved_count = 0 - skipped_count = 0 + record_data = {} + for col in df.columns: + if hasattr(ManholeDomesticChamber, col): + val = row[col] + if pd.isna(val) or str(val).strip() in ["", "-", "—", "nan"]: + val = None + record_data[col] = val - try: - for index, row in df.iterrows(): - record_data = {} - location = row.get("Location") - Node_No = row.get("Node_No") - - if (pd.isna(location) or str(location).strip() == "" or pd.isna(Node_No) or str(Node_No).strip() == ""): - skipped_count += 1 - continue - - # Map only model columns - for col in df.columns: - if hasattr(ManholeDomesticChamber, col): - value = row[col] - - # Normalize empty values - if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan"]: - value = None - - record_data[col] = value - - # If all mapped fields are None → skip - if all(v is None for v in record_data.values()): - skipped_count += 1 - continue - - record = ManholeDomesticChamber( - subcontractor_id=subcontractor_id, RA_Bill_No=RA_Bill_No, - **record_data - ) - - print("manhole_domestic_chamber Saving Row → Location:", record.Location, " Node_No:", record.Node_No) - - db.session.add(record) - saved_count += 1 - - db.session.commit() - - return True, ( - f"Manhole Domestic Chamber saved successfully. " - f"Inserted: {saved_count}, Skipped: {skipped_count}" + record = ManholeDomesticChamber( + subcontractor_id=subcontractor_id, + RA_Bill_No=RA_Bill_No, + **record_data, ) + db.session.add(record) - except Exception as e: - db.session.rollback() - return False, f"Manhole Domestic Chamber save failed: {e}" - + db.session.commit() + return True, "Manhole Domestic Chamber saved successfully." + except Exception as e: + db.session.rollback() + raise e - # ---------------------- client ---------------------------------- - - # def handle_client_file_upload(self, file, RA_Bill_No): - - # if not RA_Bill_No: - # return False, "Please Enter RA Bill No ." - - # if not file or file.filename == "": - # return False, "No file selected." - # if not self.allowed_file(file.filename): - # return False, "Invalid file type! Allowed: CSV, XLSX, XLS" - - # ensure_upload_folder() - - # folder = os.path.join(Config.UPLOAD_FOLDER, f"Client_Bill_{RA_Bill_No}") - # os.makedirs(folder, exist_ok=True) - - # filename = secure_filename(file.filename) - # filepath = os.path.join(folder, filename) - # file.save(filepath) - - # try: - # df_tr_ex = pd.read_excel(filepath, sheet_name ="Tr.Ex.", header=12) - # df_mh_ex = pd.read_excel(filepath, sheet_name="MH Ex.", header=12) - # df_mh_dc = pd.read_excel(filepath, sheet_name="MH & DC", header=11) - - # print("\n=== Uploaded File tr ex ===") - # print(df_tr_ex.head()) - # print("=============================\n") - # print("\n=== Uploaded File mh ex ===") - # print(df_mh_ex.head()) - # print("========================================\n") - # print("\n=== Uploaded File MH DC ===") - # print(df_mh_dc.head()) - # print("========================================\n") - - - # self.client_trench_excavation(df_tr_ex, RA_Bill_No) - # self.client_manhole_excavation(df_mh_ex, RA_Bill_No) - # self.client_manhole_domestic_chamber(df_mh_dc, RA_Bill_No) - - # return True, "File uploaded successfully." - - # except Exception as e: - # return False, f"Processing failed: {e}" - - # # Tr Ex save method (TrenchExcavationClient model) - # def client_trench_excavation(self, df, RA_Bill_No): - # df.columns = [str(c).strip() for c in df.columns] - # # If the sheet has merged cells -> forward fill Location - # if "Location" in df.columns: - # df["Location"] = df["Location"].ffill() - - # df = df.dropna(how="all") # REMOVE empty rows - # # Identify missing location rows before insert - # missing_loc = df[df["Location"].isna() | (df["Location"].astype(str).str.strip() == "")] - # if not missing_loc.empty: - # return False, f"Error: Some rows have empty Location. Rows: {missing_loc.index.tolist()}" - - # saved_count = 0 - - # try: - # for index, row in df.iterrows(): - # record_data = {} - # # Insert only fields that exist in model - # for col in df.columns: - # if hasattr(TrenchExcavationClient, col): - # value = row[col] - # # Normalize empty values - # if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan", "NaN"]: - # value = None - - # record_data[col] = value - - # record = TrenchExcavationClient( - # RA_Bill_No=RA_Bill_No, - # **record_data - # ) - - # db.session.add(record) - # saved_count += 1 - - # db.session.commit() - # return True, f"Clinnt Tr Ex data saved successfully. Total rows: {saved_count}" - - # except Exception as e: - # db.session.rollback() - # return False, f"Clinnt Tr Ex Save Failed: {e}" - - - # # Mh Ex save method (ManholeExcavationClient model) - # def client_manhole_excavation(self, df, RA_Bill_No): - # # Clean column names (strip whitespace) - # df.columns = [str(c).strip() for c in df.columns] - # # If the sheet has merged cells -> forward fill Location - # if "Location" in df.columns: - # df["Location"] = df["Location"].ffill() - - # # REMOVE empty rows - # df = df.dropna(how="all") - # # Identify missing location rows before insert - # missing_loc = df[df["Location"].isna() | (df["Location"].astype(str).str.strip() == "")] - # if not missing_loc.empty: - # return False, f"Error: Some rows have empty Location. Rows: {missing_loc.index.tolist()}" - - # saved_count = 0 - - # try: - # for index, row in df.iterrows(): - # record_data = {} - # # Insert only fields that exist in model - # for col in df.columns: - # if hasattr(ManholeExcavationClient, col): - # value = row[col] - - # # Normalize empty values - # if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan", "NaN"]: - # value = None - - # record_data[col] = value - - # record = ManholeExcavationClient( - # RA_Bill_No=RA_Bill_No, - # **record_data - # ) - - # db.session.add(record) - # saved_count += 1 - - # db.session.commit() - # return True, f" Client Mh Ex. data saved successfully. Total rows: {saved_count}" - - # except Exception as e: - # db.session.rollback() - # return False, f"Client Mh Ex. Save Failed: {e}" - - - # # Mh and Dc save method (ManholeDomesticChamberClient model) - # def client_manhole_domestic_chamber(self, df, RA_Bill_No): - # Clean column names (strip whitespace) - # df.columns = [str(c).strip() for c in df.columns] - # # If the sheet has merged cells -> forward fill Location - # if "Location" in df.columns: - # df["Location"] = df["Location"].ffill() - - # # REMOVE empty rows - # df = df.dropna(how="all") - # # Identify missing location rows before insert - # missing_loc = df[df["Location"].isna() | (df["Location"].astype(str).str.strip() == "")] - # if not missing_loc.empty: - # return False, f"Error: Some rows have empty Location. Rows: {missing_loc.index.tolist()}" - - # saved_count = 0 - - # try: - # for index, row in df.iterrows(): - # record_data = {} - - # # Insert only fields that exist in model - # for col in df.columns: - # if hasattr(ManholeDomesticChamberClient, col): - # value = row[col] - # # Normalize empty values - # if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan", "NaN"]: - # value = None - - # record_data[col] = value - - # record = ManholeDomesticChamberClient( - # RA_Bill_No=RA_Bill_No, - # **record_data - # ) - - # db.session.add(record) - # saved_count += 1 - - # db.session.commit() - # return True, f"Mh and Dc data saved successfully. Total rows: {saved_count}" - - # except Exception as e: - # db.session.rollback() - # return False, f"Mh and Dc data Save Failed: {e}" - + # =================================================== + # CLIENT FILE UPLOAD (FINAL FIXED VERSION) + # =================================================== def handle_client_file_upload(self, file, RA_Bill_No): if not RA_Bill_No: @@ -489,66 +815,60 @@ class FileService: file.save(filepath) try: - df_tr_ex = pd.read_excel(filepath, sheet_name="Tr.Ex.", header=12) - df_mh_ex = pd.read_excel(filepath, sheet_name="MH Ex.", header=12) - df_mh_dc = pd.read_excel(filepath, sheet_name="MH & DC", header=11) + df_tr_ex = pd.read_excel(filepath, sheet_name="Tr.Ex.", header=4) + df_mh_ex = pd.read_excel(filepath, sheet_name="MH Ex.", header=4) + df_mh_dc = pd.read_excel(filepath, sheet_name="MH & DC", header=3) - # Single transaction - self._save_client_data(df_tr_ex, TrenchExcavationClient, "Trench Excavation", RA_Bill_No) - self._save_client_data(df_mh_ex, ManholeExcavationClient, "Manhole Excavation", RA_Bill_No) - self._save_client_data(df_mh_dc, ManholeDomesticChamberClient, "MH & DC", RA_Bill_No) + print("------------------") + print("df_tr_ex :",df_tr_ex) + print("------------------") + print("df_mh_ex :",df_mh_ex) + print("------------------") + print("df_mh_dc :",df_mh_dc) + print("------------------") + + self._save_client_data(df_tr_ex, TrenchExcavationClient, RA_Bill_No) + self._save_client_data(df_mh_ex, ManholeExcavationClient, RA_Bill_No) + self._save_client_data(df_mh_dc, ManholeDomesticChamberClient, RA_Bill_No) db.session.commit() return True, "Client file uploaded and all data saved successfully." - except ClientDataSaveError as e: - db.session.rollback() - return False, str(e) - except Exception as e: db.session.rollback() - return False, f"Unexpected system error: {e}" - + return False, f"Client data save failed: {e}" + # --------------------------------------------------- + # Common Client Save Method (FIXED) + # --------------------------------------------------- def _save_client_data(self, df, model, RA_Bill_No): - - # Clean column names - df.columns = [str(c).strip() for c in df.columns] - # Forward fill Location (merged cells issue) - if "Location" in df.columns: - df["Location"] = df["Location"].ffill() + df.columns = [str(c).strip() for c in df.columns] - # Remove fully empty rows - df = df.dropna(how="all") + if "Location" in df.columns: + df["Location"] = df["Location"].ffill() - # Validate Location - if "Location" in df.columns: - missing_loc = df[ - df["Location"].isna() | - (df["Location"].astype(str).str.strip() == "") - ] - if not missing_loc.empty: - raise Exception( - f"{model.__name__}: Empty Location at rows {missing_loc.index.tolist()}" - ) + df = df.dropna(how="all") - # Insert rows - for _, row in df.iterrows(): - record_data = {} - - for col in df.columns: - if hasattr(model, col): - value = row[col] - - if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan", "NaN"]: - value = None - - record_data[col] = value - - record = model( - RA_Bill_No=RA_Bill_No, - **record_data + if "Location" in df.columns: + missing_loc = df[ + df["Location"].isna() + | (df["Location"].astype(str).str.strip() == "") + ] + if not missing_loc.empty: + raise Exception( + f"{model.__name__}: Empty Location at rows {missing_loc.index.tolist()}" ) - db.session.add(record) + for _, row in df.iterrows(): + record_data = {} + + for col in df.columns: + if hasattr(model, col): + val = row[col] + if pd.isna(val) or str(val).strip() in ["", "-", "—", "nan", "NaN"]: + val = None + record_data[col] = val + + record = model(RA_Bill_No=RA_Bill_No, **record_data) + db.session.add(record)