diff --git a/app/models/manhole_domestic_chamber_model.py b/app/models/manhole_domestic_chamber_model.py index caee3a2..a748744 100644 --- a/app/models/manhole_domestic_chamber_model.py +++ b/app/models/manhole_domestic_chamber_model.py @@ -11,7 +11,7 @@ class ManholeDomesticChamber(db.Model): subcontractor = db.relationship("Subcontractor", backref="manhole_domestic_chamber_records") # Basic Fields - Location = db.Column(db.String(255)) + Location = db.Column(db.String(500)) Node_No = db.Column(db.String(100)) Depth_of_MH = db.Column(db.Float) diff --git a/app/models/manhole_excavation_model.py b/app/models/manhole_excavation_model.py index 996fd10..56e3a81 100644 --- a/app/models/manhole_excavation_model.py +++ b/app/models/manhole_excavation_model.py @@ -11,7 +11,7 @@ class ManholeExcavation(db.Model): subcontractor = db.relationship("Subcontractor", backref="manhole_records") # Basic Fields - Location = db.Column(db.String(255)) + Location = db.Column(db.String(500)) MH_NO = db.Column(db.String(100)) Upto_IL_Depth = db.Column(db.Float) diff --git a/app/models/mh_dc_client_model.py b/app/models/mh_dc_client_model.py index 63fcc4d..b8b37a8 100644 --- a/app/models/mh_dc_client_model.py +++ b/app/models/mh_dc_client_model.py @@ -6,12 +6,13 @@ class ManholeDomesticChamberClient(db.Model): id = db.Column(db.Integer, primary_key=True) # Foreign Key to Subcontractor table - subcontractor_id = db.Column(db.Integer, db.ForeignKey("subcontractors.id"), nullable=False) + # subcontractor_id = db.Column(db.Integer, db.ForeignKey("subcontractors.id"), nullable=False) # Relationship for easy access (subcontractor.subcontractor_name) - subcontractor = db.relationship("Subcontractor", backref="mh_dc_records") + # subcontractor = db.relationship("Subcontractor", backref="mh_dc_records") # Basic Fields - Location = db.Column(db.String(255)) + RA_Bill_No=db.Column(db.String(500)) + Location = db.Column(db.String(500)) Node_No = db.Column(db.String(100)) MH_TOP_LEVEL = db.Column(db.Float) MH_IL_LEVEL = db.Column(db.Float) diff --git a/app/models/mh_ex_client_model.py b/app/models/mh_ex_client_model.py index a67f5c0..172b0ac 100644 --- a/app/models/mh_ex_client_model.py +++ b/app/models/mh_ex_client_model.py @@ -6,12 +6,13 @@ class ManholeExcavationClient(db.Model): id = db.Column(db.Integer, primary_key=True) # Foreign Key to Subcontractor table - subcontractor_id = db.Column(db.Integer, db.ForeignKey("subcontractors.id"), nullable=False) + # subcontractor_id = db.Column(db.Integer, db.ForeignKey("subcontractors.id"), nullable=False) # Relationship for easy access (subcontractor.subcontractor_name) - subcontractor = db.relationship("Subcontractor", backref="mh_ex_records") + # subcontractor = db.relationship("Subcontractor", backref="mh_ex_records") # Basic Fields - Location = db.Column(db.String(255)) + RA_Bill_No=db.Column(db.String(500)) + Location = db.Column(db.String(500)) MH_NO = db.Column(db.String(100)) GL_m = db.Column(db.Float) MH_Invert_Level = db.Column(db.Float) diff --git a/app/models/tr_ex_client_model.py b/app/models/tr_ex_client_model.py index bbddd0b..0674967 100644 --- a/app/models/tr_ex_client_model.py +++ b/app/models/tr_ex_client_model.py @@ -6,12 +6,13 @@ class TrenchExcavationClient(db.Model): id = db.Column(db.Integer, primary_key=True) # Foreign Key to Subcontractor table - subcontractor_id = db.Column(db.Integer, db.ForeignKey("subcontractors.id"), nullable=False) + # subcontractor_id = db.Column(db.Integer, db.ForeignKey("subcontractors.id"), nullable=False) # Relationship for easy access (subcontractor.subcontractor_name) - subcontractor = db.relationship("Subcontractor", backref="tr_ex_records") + # subcontractor = db.relationship("Subcontractor", backref="tr_ex_records") # Basic Fields - Location = db.Column(db.String(255)) + RA_Bill_No=db.Column(db.String(500)) + Location = db.Column(db.String(500)) MH_NO = db.Column(db.String(100)) CC_length = db.Column(db.Float) Actual_Trench_Length = db.Column(db.Float) diff --git a/app/models/trench_excavation_model.py b/app/models/trench_excavation_model.py index 6e1cae4..d31a679 100644 --- a/app/models/trench_excavation_model.py +++ b/app/models/trench_excavation_model.py @@ -11,7 +11,7 @@ class TrenchExcavation(db.Model): subcontractor = db.relationship("Subcontractor", backref="trench_records") # Basic Fields - Location = db.Column(db.String(255)) + Location = db.Column(db.String(500)) MH_NO = db.Column(db.String(100)) CC_length = db.Column(db.Float) Invert_Level = db.Column(db.Float) diff --git a/app/routes/file_import.py b/app/routes/file_import.py index d16071f..6011a4e 100644 --- a/app/routes/file_import.py +++ b/app/routes/file_import.py @@ -31,11 +31,12 @@ def client_import_file(): if request.method == "POST": file = request.files.get("file") - subcontractor_id = request.form.get("subcontractor_id") - file_type = request.form.get("file_type") + # subcontractor_id = request.form.get("subcontractor_id") + # file_type = request.form.get("file_type") + RA_Bill_No = request.form.get("RA_Bill_No") service = FileService() - success, msg = service.handle_file_upload(file, subcontractor_id, file_type) + success, msg = service.handle_client_file_upload(file, RA_Bill_No) flash(msg, "success" if success else "danger") diff --git a/app/services/file_service.py b/app/services/file_service.py index 53ca25b..f538157 100644 --- a/app/services/file_service.py +++ b/app/services/file_service.py @@ -287,139 +287,268 @@ class FileService: # ---------------------- client ---------------------------------- - def client_trench_excavation(self, df, subcontractor_id): - df.columns = [str(c).strip() for c in df.columns] - # If the sheet has merged cells -> forward fill Location - if "Location" in df.columns: - df["Location"] = df["Location"].ffill() + # def handle_client_file_upload(self, file, RA_Bill_No): - df = df.dropna(how="all") # REMOVE empty rows - # Identify missing location rows before insert - missing_loc = df[df["Location"].isna() | (df["Location"].astype(str).str.strip() == "")] - if not missing_loc.empty: - return False, f"Error: Some rows have empty Location. Rows: {missing_loc.index.tolist()}" + # if not RA_Bill_No: + # return False, "Please Enter RA Bill No ." - saved_count = 0 + # if not file or file.filename == "": + # return False, "No file selected." + # if not self.allowed_file(file.filename): + # return False, "Invalid file type! Allowed: CSV, XLSX, XLS" - try: - for index, row in df.iterrows(): - record_data = {} - # Insert only fields that exist in model - for col in df.columns: - if hasattr(TrenchExcavationClient, col): - value = row[col] - # Normalize empty values - if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan", "NaN"]: - value = None + # ensure_upload_folder() - record_data[col] = value + # folder = os.path.join(Config.UPLOAD_FOLDER, f"Client_Bill_{RA_Bill_No}") + # os.makedirs(folder, exist_ok=True) - record = TrenchExcavationClient( - subcontractor_id=subcontractor_id, - **record_data - ) + # filename = secure_filename(file.filename) + # filepath = os.path.join(folder, filename) + # file.save(filepath) - db.session.add(record) - saved_count += 1 + # try: + # df_tr_ex = pd.read_excel(filepath, sheet_name ="Tr.Ex.", header=12) + # df_mh_ex = pd.read_excel(filepath, sheet_name="MH Ex.", header=12) + # df_mh_dc = pd.read_excel(filepath, sheet_name="MH & DC", header=11) - db.session.commit() - return True, f"Clinnt Tr Ex data saved successfully. Total rows: {saved_count}" + # print("\n=== Uploaded File tr ex ===") + # print(df_tr_ex.head()) + # print("=============================\n") + # print("\n=== Uploaded File mh ex ===") + # print(df_mh_ex.head()) + # print("========================================\n") + # print("\n=== Uploaded File MH DC ===") + # print(df_mh_dc.head()) + # print("========================================\n") - except Exception as e: - db.session.rollback() - return False, f"Clinnt Tr Ex Save Failed: {e}" + + # self.client_trench_excavation(df_tr_ex, RA_Bill_No) + # self.client_manhole_excavation(df_mh_ex, RA_Bill_No) + # self.client_manhole_domestic_chamber(df_mh_dc, RA_Bill_No) + + # return True, "File uploaded successfully." + + # except Exception as e: + # return False, f"Processing failed: {e}" + + # # Tr Ex save method (TrenchExcavationClient model) + # def client_trench_excavation(self, df, RA_Bill_No): + # df.columns = [str(c).strip() for c in df.columns] + # # If the sheet has merged cells -> forward fill Location + # if "Location" in df.columns: + # df["Location"] = df["Location"].ffill() + + # df = df.dropna(how="all") # REMOVE empty rows + # # Identify missing location rows before insert + # missing_loc = df[df["Location"].isna() | (df["Location"].astype(str).str.strip() == "")] + # if not missing_loc.empty: + # return False, f"Error: Some rows have empty Location. Rows: {missing_loc.index.tolist()}" + + # saved_count = 0 + + # try: + # for index, row in df.iterrows(): + # record_data = {} + # # Insert only fields that exist in model + # for col in df.columns: + # if hasattr(TrenchExcavationClient, col): + # value = row[col] + # # Normalize empty values + # if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan", "NaN"]: + # value = None + + # record_data[col] = value + + # record = TrenchExcavationClient( + # RA_Bill_No=RA_Bill_No, + # **record_data + # ) + + # db.session.add(record) + # saved_count += 1 + + # db.session.commit() + # return True, f"Clinnt Tr Ex data saved successfully. Total rows: {saved_count}" + + # except Exception as e: + # db.session.rollback() + # return False, f"Clinnt Tr Ex Save Failed: {e}" - # Mh Ex save method (ManholeExcavationClient model) - def client_manhole_excavation(self, df, subcontractor_id): + # # Mh Ex save method (ManholeExcavationClient model) + # def client_manhole_excavation(self, df, RA_Bill_No): + # # Clean column names (strip whitespace) + # df.columns = [str(c).strip() for c in df.columns] + # # If the sheet has merged cells -> forward fill Location + # if "Location" in df.columns: + # df["Location"] = df["Location"].ffill() + + # # REMOVE empty rows + # df = df.dropna(how="all") + # # Identify missing location rows before insert + # missing_loc = df[df["Location"].isna() | (df["Location"].astype(str).str.strip() == "")] + # if not missing_loc.empty: + # return False, f"Error: Some rows have empty Location. Rows: {missing_loc.index.tolist()}" + + # saved_count = 0 + + # try: + # for index, row in df.iterrows(): + # record_data = {} + # # Insert only fields that exist in model + # for col in df.columns: + # if hasattr(ManholeExcavationClient, col): + # value = row[col] + + # # Normalize empty values + # if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan", "NaN"]: + # value = None + + # record_data[col] = value + + # record = ManholeExcavationClient( + # RA_Bill_No=RA_Bill_No, + # **record_data + # ) + + # db.session.add(record) + # saved_count += 1 + + # db.session.commit() + # return True, f" Client Mh Ex. data saved successfully. Total rows: {saved_count}" + + # except Exception as e: + # db.session.rollback() + # return False, f"Client Mh Ex. Save Failed: {e}" + + + # # Mh and Dc save method (ManholeDomesticChamberClient model) + # def client_manhole_domestic_chamber(self, df, RA_Bill_No): # Clean column names (strip whitespace) - df.columns = [str(c).strip() for c in df.columns] - # If the sheet has merged cells -> forward fill Location - if "Location" in df.columns: - df["Location"] = df["Location"].ffill() + # df.columns = [str(c).strip() for c in df.columns] + # # If the sheet has merged cells -> forward fill Location + # if "Location" in df.columns: + # df["Location"] = df["Location"].ffill() - # REMOVE empty rows - df = df.dropna(how="all") - # Identify missing location rows before insert - missing_loc = df[df["Location"].isna() | (df["Location"].astype(str).str.strip() == "")] - if not missing_loc.empty: - return False, f"Error: Some rows have empty Location. Rows: {missing_loc.index.tolist()}" + # # REMOVE empty rows + # df = df.dropna(how="all") + # # Identify missing location rows before insert + # missing_loc = df[df["Location"].isna() | (df["Location"].astype(str).str.strip() == "")] + # if not missing_loc.empty: + # return False, f"Error: Some rows have empty Location. Rows: {missing_loc.index.tolist()}" - saved_count = 0 + # saved_count = 0 + + # try: + # for index, row in df.iterrows(): + # record_data = {} + + # # Insert only fields that exist in model + # for col in df.columns: + # if hasattr(ManholeDomesticChamberClient, col): + # value = row[col] + # # Normalize empty values + # if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan", "NaN"]: + # value = None + + # record_data[col] = value + + # record = ManholeDomesticChamberClient( + # RA_Bill_No=RA_Bill_No, + # **record_data + # ) + + # db.session.add(record) + # saved_count += 1 + + # db.session.commit() + # return True, f"Mh and Dc data saved successfully. Total rows: {saved_count}" + + # except Exception as e: + # db.session.rollback() + # return False, f"Mh and Dc data Save Failed: {e}" + + def handle_client_file_upload(self, file, RA_Bill_No): + + if not RA_Bill_No: + return False, "Please Enter RA Bill No." + + if not file or file.filename == "": + return False, "No file selected." + + if not self.allowed_file(file.filename): + return False, "Invalid file type! Allowed: CSV, XLSX, XLS" + + ensure_upload_folder() + + folder = os.path.join(Config.UPLOAD_FOLDER, f"Client_Bill_{RA_Bill_No}") + os.makedirs(folder, exist_ok=True) + + filename = secure_filename(file.filename) + filepath = os.path.join(folder, filename) + file.save(filepath) try: - for index, row in df.iterrows(): + df_tr_ex = pd.read_excel(filepath, sheet_name="Tr.Ex.", header=12) + df_mh_ex = pd.read_excel(filepath, sheet_name="MH Ex.", header=12) + df_mh_dc = pd.read_excel(filepath, sheet_name="MH & DC", header=11) + + # Single transaction + self._save_client_data(df_tr_ex, TrenchExcavationClient, "Trench Excavation", RA_Bill_No) + self._save_client_data(df_mh_ex, ManholeExcavationClient, "Manhole Excavation", RA_Bill_No) + self._save_client_data(df_mh_dc, ManholeDomesticChamberClient, "MH & DC", RA_Bill_No) + + db.session.commit() + return True, "Client file uploaded and all data saved successfully." + + except ClientDataSaveError as e: + db.session.rollback() + return False, str(e) + + except Exception as e: + db.session.rollback() + return False, f"Unexpected system error: {e}" + + + def _save_client_data(self, df, model, RA_Bill_No): + + # Clean column names + df.columns = [str(c).strip() for c in df.columns] + + # Forward fill Location (merged cells issue) + if "Location" in df.columns: + df["Location"] = df["Location"].ffill() + + # Remove fully empty rows + df = df.dropna(how="all") + + # Validate Location + if "Location" in df.columns: + missing_loc = df[ + df["Location"].isna() | + (df["Location"].astype(str).str.strip() == "") + ] + if not missing_loc.empty: + raise Exception( + f"{model.__name__}: Empty Location at rows {missing_loc.index.tolist()}" + ) + + # Insert rows + for _, row in df.iterrows(): record_data = {} - # Insert only fields that exist in model + for col in df.columns: - if hasattr(ManholeExcavationClient, col): + if hasattr(model, col): value = row[col] - # Normalize empty values if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan", "NaN"]: value = None record_data[col] = value - record = ManholeExcavationClient( - subcontractor_id=subcontractor_id, + record = model( + RA_Bill_No=RA_Bill_No, **record_data ) db.session.add(record) - saved_count += 1 - - db.session.commit() - return True, f" Client Mh Ex. data saved successfully. Total rows: {saved_count}" - - except Exception as e: - db.session.rollback() - return False, f"Client Mh Ex. Save Failed: {e}" - - - # Mh and Dc save method (ManholeDomesticChamberClient model) - def client_manhole_domestic_chamber(self, df, subcontractor_id): - # Clean column names (strip whitespace) - df.columns = [str(c).strip() for c in df.columns] - # If the sheet has merged cells -> forward fill Location - if "Location" in df.columns: - df["Location"] = df["Location"].ffill() - - # REMOVE empty rows - df = df.dropna(how="all") - # Identify missing location rows before insert - missing_loc = df[df["Location"].isna() | (df["Location"].astype(str).str.strip() == "")] - if not missing_loc.empty: - return False, f"Error: Some rows have empty Location. Rows: {missing_loc.index.tolist()}" - - saved_count = 0 - - try: - for index, row in df.iterrows(): - record_data = {} - - # Insert only fields that exist in model - for col in df.columns: - if hasattr(ManholeDomesticChamberClient, col): - value = row[col] - # Normalize empty values - if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan", "NaN"]: - value = None - - record_data[col] = value - - record = ManholeDomesticChamberClient( - subcontractor_id=subcontractor_id, - **record_data - ) - - db.session.add(record) - saved_count += 1 - - db.session.commit() - return True, f"Mh and Dc data saved successfully. Total rows: {saved_count}" - - except Exception as e: - db.session.rollback() - return False, f"Mh and Dc data Save Failed: {e}" - \ No newline at end of file diff --git a/app/templates/file_import_client.html b/app/templates/file_import_client.html index 189e856..e44ffd9 100644 --- a/app/templates/file_import_client.html +++ b/app/templates/file_import_client.html @@ -8,17 +8,17 @@
- + - + + +