# import os # import pandas as pd # from werkzeug.utils import secure_filename # from app.config import Config # from app import db # from app.models.trench_excavation_model import TrenchExcavation # from app.models.manhole_excavation_model import ManholeExcavation # from app.models.manhole_domestic_chamber_model import ManholeDomesticChamber # from app.models.tr_ex_client_model import TrenchExcavationClient # from app.models.mh_ex_client_model import ManholeExcavationClient # from app.models.mh_dc_client_model import ManholeDomesticChamberClient # from app.utils.file_utils import ensure_upload_folder # class FileService: # def allowed_file(self, filename): # return "." in filename and filename.rsplit(".", 1)[1].lower() in Config.ALLOWED_EXTENSIONS # # def handle_file_upload(self, file, subcontractor_id, file_type): # def handle_file_upload(self, file, subcontractor_id, RA_Bill_No): # if not subcontractor_id: # return False, "Please select subcontractor." # if not RA_Bill_No: # return False, "Please Enter RA Bill No)." # if not file or file.filename == "": # return False, "No file selected." # if not self.allowed_file(file.filename): # return False, "Invalid file type! Allowed: CSV, XLSX, XLS" # ensure_upload_folder() # folder = os.path.join(Config.UPLOAD_FOLDER, f"sub_{subcontractor_id}") # os.makedirs(folder, exist_ok=True) # filename = secure_filename(file.filename) # filepath = os.path.join(folder, filename) # file.save(filepath) # try: # df_tr_ex = pd.read_excel(filepath, sheet_name ="Tr.Ex.", header=12) # df_mh_ex = pd.read_excel(filepath, sheet_name="MH Ex.", header=12) # df_mh_dc = pd.read_excel(filepath, sheet_name="MH & DC", header=11) # self.process_trench_excavation(df_tr_ex, subcontractor_id, RA_Bill_No) # self.process_manhole_excavation(df_mh_ex, subcontractor_id, RA_Bill_No) # self.process_manhole_domestic_chamber(df_mh_dc, subcontractor_id, RA_Bill_No) # return True, "File uploaded successfully." # except Exception as e: # return False, f"Processing failed: {e}" # # Trench Excavation save method (TrenchExcavation model) # def process_trench_excavation(self, df, subcontractor_id,RA_Bill_No): # print("trench_excavation RA_Bill_No :",RA_Bill_No) # print("=== trench_excavation ===") # print(df.columns.tolist()) # print("===================") # # Clean column names # df.columns = ( # df.columns.astype(str) # .str.strip() # .str.replace(r"[^\w]", "_", regex=True) # .str.replace("__+", "_", regex=True) # .str.strip("_") # ) # # Remove completely empty rows # df = df.dropna(how="all") # # Forward fill merged Location # if "Location" in df.columns: # df["Location"] = df["Location"].ffill() # saved_count = 0 # skipped_count = 0 # try: # for index, row in df.iterrows(): # record_data = {} # location = row.get("Location") # mh_no = row.get("MH_NO") # if (pd.isna(location) or str(location).strip() == "" or pd.isna(mh_no) or str(mh_no).strip() == ""): # skipped_count += 1 # continue # # Map only model columns # for col in df.columns: # if hasattr(TrenchExcavation, col): # value = row[col] # # Normalize empty values # if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan"]: # value = None # record_data[col] = value # # If all mapped fields are None → skip # if all(v is None for v in record_data.values()): # skipped_count += 1 # continue # record = TrenchExcavation( # subcontractor_id=subcontractor_id, RA_Bill_No=RA_Bill_No, # **record_data # ) # print("trench_excavation Saving Row → Location:", record.Location, " MH_NO:", record.MH_NO) # db.session.add(record) # saved_count += 1 # db.session.commit() # return True, ( # f"Trench Excavation saved successfully. " # f"Inserted: {saved_count}, Skipped: {skipped_count}" # ) # except Exception as e: # db.session.rollback() # return False, f"Trench Excavation save failed: {e}" # # Manhole Excavation save method (ManholeExcavation model) # def process_manhole_excavation(self, df, subcontractor_id, RA_Bill_No): # print("manhole_excavation RA_Bill_No:",RA_Bill_No) # print("=== manhole_excavation ===") # print(df.columns.tolist()) # print("===================") # # Clean column names # df.columns = ( # df.columns.astype(str) # .str.strip() # .str.replace(r"[^\w]", "_", regex=True) # .str.replace("__+", "_", regex=True) # .str.strip("_") # ) # # Remove completely empty rows # df = df.dropna(how="all") # # Forward fill merged Location # if "Location" in df.columns: # df["Location"] = df["Location"].ffill() # saved_count = 0 # skipped_count = 0 # try: # for index, row in df.iterrows(): # record_data = {} # location = row.get("Location") # mh_no = row.get("MH_NO") # if (pd.isna(location) or str(location).strip() == "" or pd.isna(mh_no) or str(mh_no).strip() == ""): # skipped_count += 1 # continue # # Map only model columns # for col in df.columns: # if hasattr(ManholeExcavation, col): # value = row[col] # # Normalize empty values # if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan"]: # value = None # record_data[col] = value # # If all mapped fields are None → skip # if all(v is None for v in record_data.values()): # skipped_count += 1 # continue # record = ManholeExcavation( # subcontractor_id=subcontractor_id, RA_Bill_No=RA_Bill_No, # **record_data # ) # print("manhole_excavation Saving Row → Location:", record.Location, " MH_NO:", record.MH_NO) # db.session.add(record) # saved_count += 1 # db.session.commit() # return True, ( # f"Manhole Excavation saved successfully. " # f"Inserted: {saved_count}, Skipped: {skipped_count}" # ) # except Exception as e: # db.session.rollback() # return False, f"Manhole Excavation save failed: {e}" # # Manhole and Domestic Chamber Construction save method (ManholeDomesticChamber model) # def process_manhole_domestic_chamber(self, df, subcontractor_id, RA_Bill_No): # print("manhole_domestic_chamber RA_Bill_No :",RA_Bill_No) # print("=== manhole_domestic_chamber ===") # print(df.columns.tolist()) # print("===================") # # Clean column names # df.columns = ( # df.columns.astype(str) # .str.strip() # .str.replace(r"[^\w]", "_", regex=True) # .str.replace("__+", "_", regex=True) # .str.strip("_") # ) # # Remove completely empty rows # df = df.dropna(how="all") # # Forward fill merged Location # if "Location" in df.columns: # df["Location"] = df["Location"].ffill() # saved_count = 0 # skipped_count = 0 # try: # for index, row in df.iterrows(): # record_data = {} # location = row.get("Location") # Node_No = row.get("Node_No") # if (pd.isna(location) or str(location).strip() == "" or pd.isna(Node_No) or str(Node_No).strip() == ""): # skipped_count += 1 # continue # # Map only model columns # for col in df.columns: # if hasattr(ManholeDomesticChamber, col): # value = row[col] # # Normalize empty values # if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan"]: # value = None # record_data[col] = value # # If all mapped fields are None → skip # if all(v is None for v in record_data.values()): # skipped_count += 1 # continue # record = ManholeDomesticChamber( # subcontractor_id=subcontractor_id, RA_Bill_No=RA_Bill_No, # **record_data # ) # print("manhole_domestic_chamber Saving Row → Location:", record.Location, " Node_No:", record.Node_No) # db.session.add(record) # saved_count += 1 # db.session.commit() # return True, ( # f"Manhole Domestic Chamber saved successfully. " # f"Inserted: {saved_count}, Skipped: {skipped_count}" # ) # except Exception as e: # db.session.rollback() # return False, f"Manhole Domestic Chamber save failed: {e}" # # ---------------------- client ---------------------------------- # # def handle_client_file_upload(self, file, RA_Bill_No): # # if not RA_Bill_No: # # return False, "Please Enter RA Bill No ." # # if not file or file.filename == "": # # return False, "No file selected." # # if not self.allowed_file(file.filename): # # return False, "Invalid file type! Allowed: CSV, XLSX, XLS" # # ensure_upload_folder() # # folder = os.path.join(Config.UPLOAD_FOLDER, f"Client_Bill_{RA_Bill_No}") # # os.makedirs(folder, exist_ok=True) # # filename = secure_filename(file.filename) # # filepath = os.path.join(folder, filename) # # file.save(filepath) # # try: # # df_tr_ex = pd.read_excel(filepath, sheet_name ="Tr.Ex.", header=12) # # df_mh_ex = pd.read_excel(filepath, sheet_name="MH Ex.", header=12) # # df_mh_dc = pd.read_excel(filepath, sheet_name="MH & DC", header=11) # # print("\n=== Uploaded File tr ex ===") # # print(df_tr_ex.head()) # # print("=============================\n") # # print("\n=== Uploaded File mh ex ===") # # print(df_mh_ex.head()) # # print("========================================\n") # # print("\n=== Uploaded File MH DC ===") # # print(df_mh_dc.head()) # # print("========================================\n") # # self.client_trench_excavation(df_tr_ex, RA_Bill_No) # # self.client_manhole_excavation(df_mh_ex, RA_Bill_No) # # self.client_manhole_domestic_chamber(df_mh_dc, RA_Bill_No) # # return True, "File uploaded successfully." # # except Exception as e: # # return False, f"Processing failed: {e}" # # # Tr Ex save method (TrenchExcavationClient model) # # def client_trench_excavation(self, df, RA_Bill_No): # # df.columns = [str(c).strip() for c in df.columns] # # # If the sheet has merged cells -> forward fill Location # # if "Location" in df.columns: # # df["Location"] = df["Location"].ffill() # # df = df.dropna(how="all") # REMOVE empty rows # # # Identify missing location rows before insert # # missing_loc = df[df["Location"].isna() | (df["Location"].astype(str).str.strip() == "")] # # if not missing_loc.empty: # # return False, f"Error: Some rows have empty Location. Rows: {missing_loc.index.tolist()}" # # saved_count = 0 # # try: # # for index, row in df.iterrows(): # # record_data = {} # # # Insert only fields that exist in model # # for col in df.columns: # # if hasattr(TrenchExcavationClient, col): # # value = row[col] # # # Normalize empty values # # if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan", "NaN"]: # # value = None # # record_data[col] = value # # record = TrenchExcavationClient( # # RA_Bill_No=RA_Bill_No, # # **record_data # # ) # # db.session.add(record) # # saved_count += 1 # # db.session.commit() # # return True, f"Clinnt Tr Ex data saved successfully. Total rows: {saved_count}" # # except Exception as e: # # db.session.rollback() # # return False, f"Clinnt Tr Ex Save Failed: {e}" # # # Mh Ex save method (ManholeExcavationClient model) # # def client_manhole_excavation(self, df, RA_Bill_No): # # # Clean column names (strip whitespace) # # df.columns = [str(c).strip() for c in df.columns] # # # If the sheet has merged cells -> forward fill Location # # if "Location" in df.columns: # # df["Location"] = df["Location"].ffill() # # # REMOVE empty rows # # df = df.dropna(how="all") # # # Identify missing location rows before insert # # missing_loc = df[df["Location"].isna() | (df["Location"].astype(str).str.strip() == "")] # # if not missing_loc.empty: # # return False, f"Error: Some rows have empty Location. Rows: {missing_loc.index.tolist()}" # # saved_count = 0 # # try: # # for index, row in df.iterrows(): # # record_data = {} # # # Insert only fields that exist in model # # for col in df.columns: # # if hasattr(ManholeExcavationClient, col): # # value = row[col] # # # Normalize empty values # # if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan", "NaN"]: # # value = None # # record_data[col] = value # # record = ManholeExcavationClient( # # RA_Bill_No=RA_Bill_No, # # **record_data # # ) # # db.session.add(record) # # saved_count += 1 # # db.session.commit() # # return True, f" Client Mh Ex. data saved successfully. Total rows: {saved_count}" # # except Exception as e: # # db.session.rollback() # # return False, f"Client Mh Ex. Save Failed: {e}" # # # Mh and Dc save method (ManholeDomesticChamberClient model) # # def client_manhole_domestic_chamber(self, df, RA_Bill_No): # # Clean column names (strip whitespace) # # df.columns = [str(c).strip() for c in df.columns] # # # If the sheet has merged cells -> forward fill Location # # if "Location" in df.columns: # # df["Location"] = df["Location"].ffill() # # # REMOVE empty rows # # df = df.dropna(how="all") # # # Identify missing location rows before insert # # missing_loc = df[df["Location"].isna() | (df["Location"].astype(str).str.strip() == "")] # # if not missing_loc.empty: # # return False, f"Error: Some rows have empty Location. Rows: {missing_loc.index.tolist()}" # # saved_count = 0 # # try: # # for index, row in df.iterrows(): # # record_data = {} # # # Insert only fields that exist in model # # for col in df.columns: # # if hasattr(ManholeDomesticChamberClient, col): # # value = row[col] # # # Normalize empty values # # if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan", "NaN"]: # # value = None # # record_data[col] = value # # record = ManholeDomesticChamberClient( # # RA_Bill_No=RA_Bill_No, # # **record_data # # ) # # db.session.add(record) # # saved_count += 1 # # db.session.commit() # # return True, f"Mh and Dc data saved successfully. Total rows: {saved_count}" # # except Exception as e: # # db.session.rollback() # # return False, f"Mh and Dc data Save Failed: {e}" # def handle_client_file_upload(self, file, RA_Bill_No): # if not RA_Bill_No: # return False, "Please Enter RA Bill No." # if not file or file.filename == "": # return False, "No file selected." # if not self.allowed_file(file.filename): # return False, "Invalid file type! Allowed: CSV, XLSX, XLS" # ensure_upload_folder() # folder = os.path.join(Config.UPLOAD_FOLDER, f"Client_Bill_{RA_Bill_No}") # os.makedirs(folder, exist_ok=True) # filename = secure_filename(file.filename) # filepath = os.path.join(folder, filename) # file.save(filepath) # try: # df_tr_ex = pd.read_excel(filepath, sheet_name="Tr.Ex.", header=4) # df_mh_ex = pd.read_excel(filepath, sheet_name="MH Ex.", header=4) # df_mh_dc = pd.read_excel(filepath, sheet_name="MH & DC", header=3) # print("------------------") # print("df_tr_ex :",df_tr_ex) # print("------------------") # print("df_mh_ex :",df_mh_ex) # print("------------------") # print("df_mh_dc :",df_mh_dc) # print("------------------") # # Single transaction # self._save_client_data(df_tr_ex, TrenchExcavationClient, "Trench Excavation", RA_Bill_No) # self._save_client_data(df_mh_ex, ManholeExcavationClient, "Manhole Excavation", RA_Bill_No) # self._save_client_data(df_mh_dc, ManholeDomesticChamberClient, "MH & DC", RA_Bill_No) # db.session.commit() # return True, "Client file uploaded and all data saved successfully." # except ClientDataSaveError as e: # db.session.rollback() # return False, str(e) # except Exception as e: # db.session.rollback() # return False, f"Unexpected system error: {e}" # def _save_client_data(self, df, model, RA_Bill_No): # # Clean column names # df.columns = [str(c).strip() for c in df.columns] # # Forward fill Location (merged cells issue) # if "Location" in df.columns: # df["Location"] = df["Location"].ffill() # # Remove fully empty rows # df = df.dropna(how="all") # # Validate Location # if "Location" in df.columns: # missing_loc = df[ # df["Location"].isna() | # (df["Location"].astype(str).str.strip() == "") # ] # if not missing_loc.empty: # raise Exception( # f"{model.__name__}: Empty Location at rows {missing_loc.index.tolist()}" # ) # # Insert rows # for _, row in df.iterrows(): # record_data = {} # for col in df.columns: # if hasattr(model, col): # value = row[col] # if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan", "NaN"]: # value = None # record_data[col] = value # record = model( # RA_Bill_No=RA_Bill_No, # **record_data # ) # db.session.add(record) import os import pandas as pd from werkzeug.utils import secure_filename from app.config import Config from app import db from app.models.trench_excavation_model import TrenchExcavation from app.models.manhole_excavation_model import ManholeExcavation from app.models.manhole_domestic_chamber_model import ManholeDomesticChamber from app.models.tr_ex_client_model import TrenchExcavationClient from app.models.mh_ex_client_model import ManholeExcavationClient from app.models.mh_dc_client_model import ManholeDomesticChamberClient from app.utils.file_utils import ensure_upload_folder class FileService: # --------------------------------------------------- # Common helpers # --------------------------------------------------- def allowed_file(self, filename): return ( "." in filename and filename.rsplit(".", 1)[1].lower() in Config.ALLOWED_EXTENSIONS ) # =================================================== # SUBCONTRACTOR FILE UPLOAD # =================================================== def handle_file_upload(self, file, subcontractor_id, RA_Bill_No): if not subcontractor_id: return False, "Please select subcontractor." if not RA_Bill_No: return False, "Please Enter RA Bill No." if not file or file.filename == "": return False, "No file selected." if not self.allowed_file(file.filename): return False, "Invalid file type! Allowed: CSV, XLSX, XLS" ensure_upload_folder() folder = os.path.join(Config.UPLOAD_FOLDER, f"sub_{subcontractor_id}") os.makedirs(folder, exist_ok=True) filename = secure_filename(file.filename) filepath = os.path.join(folder, filename) file.save(filepath) try: df_tr_ex = pd.read_excel(filepath, sheet_name="Tr.Ex.", header=12) df_mh_ex = pd.read_excel(filepath, sheet_name="MH Ex.", header=12) df_mh_dc = pd.read_excel(filepath, sheet_name="MH & DC", header=11) self.process_trench_excavation(df_tr_ex, subcontractor_id, RA_Bill_No) self.process_manhole_excavation(df_mh_ex, subcontractor_id, RA_Bill_No) self.process_manhole_domestic_chamber(df_mh_dc, subcontractor_id, RA_Bill_No) return True, "File uploaded successfully." except Exception as e: db.session.rollback() return False, f"Processing failed: {e}" # --------------------------------------------------- # Trench Excavation (Subcontractor) # --------------------------------------------------- def process_trench_excavation(self, df, subcontractor_id, RA_Bill_No): df.columns = ( df.columns.astype(str) .str.strip() .str.replace(r"[^\w]", "_", regex=True) .str.replace("__+", "_", regex=True) .str.strip("_") ) df = df.dropna(how="all") if "Location" in df.columns: df["Location"] = df["Location"].ffill() try: for _, row in df.iterrows(): location = row.get("Location") mh_no = row.get("MH_NO") if ( pd.isna(location) or str(location).strip() == "" or pd.isna(mh_no) or str(mh_no).strip() == "" ): continue record_data = {} for col in df.columns: if hasattr(TrenchExcavation, col): val = row[col] if pd.isna(val) or str(val).strip() in ["", "-", "—", "nan"]: val = None record_data[col] = val record = TrenchExcavation( subcontractor_id=subcontractor_id, RA_Bill_No=RA_Bill_No, **record_data, ) db.session.add(record) db.session.commit() return True, "Trench Excavation saved successfully." except Exception as e: db.session.rollback() raise e # --------------------------------------------------- # Manhole Excavation (Subcontractor) # --------------------------------------------------- def process_manhole_excavation(self, df, subcontractor_id, RA_Bill_No): df.columns = ( df.columns.astype(str) .str.strip() .str.replace(r"[^\w]", "_", regex=True) .str.replace("__+", "_", regex=True) .str.strip("_") ) df = df.dropna(how="all") if "Location" in df.columns: df["Location"] = df["Location"].ffill() try: for _, row in df.iterrows(): location = row.get("Location") mh_no = row.get("MH_NO") if ( pd.isna(location) or str(location).strip() == "" or pd.isna(mh_no) or str(mh_no).strip() == "" ): continue record_data = {} for col in df.columns: if hasattr(ManholeExcavation, col): val = row[col] if pd.isna(val) or str(val).strip() in ["", "-", "—", "nan"]: val = None record_data[col] = val record = ManholeExcavation( subcontractor_id=subcontractor_id, RA_Bill_No=RA_Bill_No, **record_data, ) db.session.add(record) db.session.commit() return True, "Manhole Excavation saved successfully." except Exception as e: db.session.rollback() raise e # --------------------------------------------------- # Manhole & Domestic Chamber (Subcontractor) # --------------------------------------------------- def process_manhole_domestic_chamber(self, df, subcontractor_id, RA_Bill_No): df.columns = ( df.columns.astype(str) .str.strip() .str.replace(r"[^\w]", "_", regex=True) .str.replace("__+", "_", regex=True) .str.strip("_") ) df = df.dropna(how="all") if "Location" in df.columns: df["Location"] = df["Location"].ffill() try: for _, row in df.iterrows(): location = row.get("Location") node_no = row.get("Node_No") if ( pd.isna(location) or str(location).strip() == "" or pd.isna(node_no) or str(node_no).strip() == "" ): continue record_data = {} for col in df.columns: if hasattr(ManholeDomesticChamber, col): val = row[col] if pd.isna(val) or str(val).strip() in ["", "-", "—", "nan"]: val = None record_data[col] = val record = ManholeDomesticChamber( subcontractor_id=subcontractor_id, RA_Bill_No=RA_Bill_No, **record_data, ) db.session.add(record) db.session.commit() return True, "Manhole Domestic Chamber saved successfully." except Exception as e: db.session.rollback() raise e # =================================================== # CLIENT FILE UPLOAD (FINAL FIXED VERSION) # =================================================== def handle_client_file_upload(self, file, RA_Bill_No): if not RA_Bill_No: return False, "Please Enter RA Bill No." if not file or file.filename == "": return False, "No file selected." if not self.allowed_file(file.filename): return False, "Invalid file type! Allowed: CSV, XLSX, XLS" ensure_upload_folder() folder = os.path.join(Config.UPLOAD_FOLDER, f"Client_Bill_{RA_Bill_No}") os.makedirs(folder, exist_ok=True) filename = secure_filename(file.filename) filepath = os.path.join(folder, filename) file.save(filepath) try: df_tr_ex = pd.read_excel(filepath, sheet_name="Tr.Ex.", header=4) df_mh_ex = pd.read_excel(filepath, sheet_name="MH Ex.", header=4) df_mh_dc = pd.read_excel(filepath, sheet_name="MH & DC", header=3) print("------------------") print("df_tr_ex :",df_tr_ex) print("------------------") print("df_mh_ex :",df_mh_ex) print("------------------") print("df_mh_dc :",df_mh_dc) print("------------------") self._save_client_data(df_tr_ex, TrenchExcavationClient, RA_Bill_No) self._save_client_data(df_mh_ex, ManholeExcavationClient, RA_Bill_No) self._save_client_data(df_mh_dc, ManholeDomesticChamberClient, RA_Bill_No) db.session.commit() return True, "Client file uploaded and all data saved successfully." except Exception as e: db.session.rollback() return False, f"Client data save failed: {e}" # --------------------------------------------------- # Common Client Save Method (FIXED) # --------------------------------------------------- def _save_client_data(self, df, model, RA_Bill_No): df.columns = [str(c).strip() for c in df.columns] if "Location" in df.columns: df["Location"] = df["Location"].ffill() df = df.dropna(how="all") if "Location" in df.columns: missing_loc = df[ df["Location"].isna() | (df["Location"].astype(str).str.strip() == "") ] if not missing_loc.empty: raise Exception( f"{model.__name__}: Empty Location at rows {missing_loc.index.tolist()}" ) for _, row in df.iterrows(): record_data = {} for col in df.columns: if hasattr(model, col): val = row[col] if pd.isna(val) or str(val).strip() in ["", "-", "—", "nan", "NaN"]: val = None record_data[col] = val record = model(RA_Bill_No=RA_Bill_No, **record_data) db.session.add(record)