diff --git a/.gitignore b/.gitignore index 6500874..1b846b5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,11 @@ -# Ignore folders -instance/ +# Ingnor upload files app/static/uploads/ -# Ignore files -.env -# Ignore by type -*.log +# Ignore files +venv + +# Ignore Log files +logs + +# Ignore db folders +instance diff --git a/app/models/__pycache__/manhole_excavation_model.cpython-313.pyc b/app/models/__pycache__/manhole_excavation_model.cpython-313.pyc index 7166b84..626e5f4 100644 Binary files a/app/models/__pycache__/manhole_excavation_model.cpython-313.pyc and b/app/models/__pycache__/manhole_excavation_model.cpython-313.pyc differ diff --git a/app/models/__pycache__/mh_ex_client_model.cpython-313.pyc b/app/models/__pycache__/mh_ex_client_model.cpython-313.pyc index f3f6da3..c1dd544 100644 Binary files a/app/models/__pycache__/mh_ex_client_model.cpython-313.pyc and b/app/models/__pycache__/mh_ex_client_model.cpython-313.pyc differ diff --git a/app/models/__pycache__/tr_ex_client_model.cpython-313.pyc b/app/models/__pycache__/tr_ex_client_model.cpython-313.pyc index 4f26f93..713c2b8 100644 Binary files a/app/models/__pycache__/tr_ex_client_model.cpython-313.pyc and b/app/models/__pycache__/tr_ex_client_model.cpython-313.pyc differ diff --git a/app/models/__pycache__/trench_excavation_model.cpython-313.pyc b/app/models/__pycache__/trench_excavation_model.cpython-313.pyc index ffaab80..69eb016 100644 Binary files a/app/models/__pycache__/trench_excavation_model.cpython-313.pyc and b/app/models/__pycache__/trench_excavation_model.cpython-313.pyc differ diff --git a/app/models/manhole_excavation_model.py b/app/models/manhole_excavation_model.py index f096b15..996fd10 100644 --- a/app/models/manhole_excavation_model.py +++ b/app/models/manhole_excavation_model.py @@ -49,13 +49,14 @@ class ManholeExcavation(db.Model): Soft_Rock_1_5_and_above_total = db.Column(db.Float) Hard_Rock_0_to_1_5_total = db.Column(db.Float) - Hard_Rock_1_5_and_above_total = db.Column(db.Float) + Hard_Rock_1_5_to_3_0_total = db.Column(db.Float) Hard_Rock_3_0_to_4_5_total = db.Column(db.Float) Hard_Rock_4_5_to_6_0_total = db.Column(db.Float) Hard_Rock_6_0_to_7_5_total = db.Column(db.Float) - Remarks = db.Column(db.String(500)) Total = db.Column(db.Float) + Remarks = db.Column(db.String(500)) + RA_Bill_No=db.Column(db.String(500)) created_at = db.Column(db.DateTime, default=datetime.today) diff --git a/app/models/tr_ex_client_model.py b/app/models/tr_ex_client_model.py index ae79d7f..bbddd0b 100644 --- a/app/models/tr_ex_client_model.py +++ b/app/models/tr_ex_client_model.py @@ -75,8 +75,8 @@ class TrenchExcavationClient(db.Model): Hard_Rock_4_5_to_6_0_total = db.Column(db.Float) Hard_Rock_6_0_to_7_5_total = db.Column(db.Float) - Remarks = db.Column(db.String(500)) Total = db.Column(db.Float) + Remarks = db.Column(db.String(500)) created_at = db.Column(db.DateTime, default=datetime.today) diff --git a/app/models/trench_excavation_model.py b/app/models/trench_excavation_model.py index 1c98d57..6e1cae4 100644 --- a/app/models/trench_excavation_model.py +++ b/app/models/trench_excavation_model.py @@ -60,15 +60,17 @@ class TrenchExcavation(db.Model): Soft_Rock_1_5_and_above_total = db.Column(db.Float) Hard_Rock_0_to_1_5_total = db.Column(db.Float) - Hard_Rock_1_5_and_above_total = db.Column(db.Float) + Hard_Rock_1_5_to_3_0_total = db.Column(db.Float) Hard_Rock_3_0_to_4_5_total = db.Column(db.Float) Hard_Rock_4_5_to_6_0_total = db.Column(db.Float) Hard_Rock_6_0_to_7_5_total = db.Column(db.Float) - Remarks = db.Column(db.String(500)) Total = db.Column(db.Float) - + Remarks = db.Column(db.String(500)) + RA_Bill_No=db.Column(db.String(500)) + created_at = db.Column(db.DateTime, default=datetime.today) + def __repr__(self): return f"" diff --git a/app/services/__pycache__/file_service.cpython-313.pyc b/app/services/__pycache__/file_service.cpython-313.pyc index 24ad08c..676152a 100644 Binary files a/app/services/__pycache__/file_service.cpython-313.pyc and b/app/services/__pycache__/file_service.cpython-313.pyc differ diff --git a/app/services/file_service.py b/app/services/file_service.py index c7192fd..3750737 100644 --- a/app/services/file_service.py +++ b/app/services/file_service.py @@ -23,6 +23,8 @@ class FileService: # def handle_file_upload(self, file, subcontractor_id, file_type): def handle_file_upload(self, file, subcontractor_id): + RA_Bill_No = 1 # this RA bill define temp for upload (changes also) + if not subcontractor_id: return False, "Please select subcontractor." # if not file_type: @@ -41,26 +43,26 @@ class FileService: filepath = os.path.join(folder, filename) file.save(filepath) - print("name::::",filename) - try: # df = pd.read_csv(filepath) if filename.endswith(".csv") else pd.read_excel(filepath) - df = pd.read_excel(filepath, sheet_name ="Tr.Ex.", header=0) - df1 = pd.read_excel(filepath, sheet_name="MH Ex.", header=0) - df2 = pd.read_excel(filepath, sheet_name="MH & DC", header=0) - print("\n=== Uploaded File Preview ===") - print(" file name h:",df) + df_tr_ex = pd.read_excel(filepath, sheet_name ="Tr.Ex.", header=12) + df_mh_ex = pd.read_excel(filepath, sheet_name="MH Ex.", header=12) + # df2 = pd.read_excel(filepath, sheet_name="MH & DC", header=0) + + print("\n=== Uploaded File tr ex ===") + print(df_tr_ex.head()) print("=============================\n") - print(" file name h1:",df1) - # print(df.head()) - print("=============================\n") - print(" file name h1:",df2) - - self.process_trench_excavation(df,subcontractor_id) - self.process_manhole_excavation(df1,subcontractor_id) - self.process_manhole_domestic_chamber(df2, subcontractor_id) + + print("=== Uploaded File mh ex ===") + print(df_mh_ex.head()) + print("========================================") + + self.process_trench_excavation(df_tr_ex, subcontractor_id, RA_Bill_No) + self.process_manhole_excavation(df_mh_ex, subcontractor_id,RA_Bill_No) + + # df = pd.read_csv(filepath) if filename.endswith(".csv") else pd.read_excel(filepath) @@ -94,96 +96,316 @@ class FileService: return False, f"Processing failed: {e}" - # ---------------------- Sub contractor -------------------------- - # Trench Excavation save method (TrenchExcavation model) - def process_trench_excavation(self, df, subcontractor_id): - df.columns = [str(c).strip() for c in df.columns] - # If the sheet has merged cells -> forward fill Location + + + # new new + def process_trench_excavation(self, df, subcontractor_id,RA_Bill_No): + + print("RA_Bill_No of Tr Ex:",RA_Bill_No) + + print("=== RAW HEADERS ===") + print(df.columns.tolist()) + print("===================") + + # Clean column names + df.columns = ( + df.columns.astype(str) + .str.strip() + .str.replace(r"[^\w]", "_", regex=True) + .str.replace("__+", "_", regex=True) + .str.strip("_") + ) + + # Remove completely empty rows + df = df.dropna(how="all") + + # Forward fill merged Location if "Location" in df.columns: df["Location"] = df["Location"].ffill() - df = df.dropna(how="all") # REMOVE empty rows - # Identify missing location rows before insert - missing_loc = df[df["Location"].isna() | (df["Location"].astype(str).str.strip() == "")] - if not missing_loc.empty: - return False, f"Error: Some rows have empty Location. Rows: {missing_loc.index.tolist()}" - saved_count = 0 + skipped_count = 0 try: for index, row in df.iterrows(): record_data = {} - # Insert only fields that exist in model + location = row.get("Location") + mh_no = row.get("MH_NO") + + if (pd.isna(location) or str(location).strip() == "" or pd.isna(mh_no) or str(mh_no).strip() == ""): + skipped_count += 1 + continue + + # Map only model columns for col in df.columns: if hasattr(TrenchExcavation, col): value = row[col] + # Normalize empty values - if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan", "NaN"]: + if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan"]: value = None record_data[col] = value + + # If all mapped fields are None → skip + if all(v is None for v in record_data.values()): + skipped_count += 1 + continue record = TrenchExcavation( - subcontractor_id=subcontractor_id, + subcontractor_id=subcontractor_id, RA_Bill_No=RA_Bill_No, **record_data ) + print("Saving Row → Location:", record.Location, " MH_NO:", record.MH_NO) + db.session.add(record) saved_count += 1 db.session.commit() - return True, f"Trench Excavation data saved successfully. Total rows: {saved_count}" + + return True, ( + f"Trench Excavation saved successfully. " + f"Inserted: {saved_count}, Skipped: {skipped_count}" + ) except Exception as e: db.session.rollback() - return False, f"Trench Excavation Save Failed: {e}" - + return False, f"Trench Excavation save failed: {e}" + + # new new + def process_manhole_excavation(self, df, subcontractor_id, RA_Bill_No): + + print("RA_Bill_No of MH EX:",RA_Bill_No) + + print("=== RAW HEADERS ===") + print(df.columns.tolist()) + print("===================") + + # Clean column names + df.columns = ( + df.columns.astype(str) + .str.strip() + .str.replace(r"[^\w]", "_", regex=True) + .str.replace("__+", "_", regex=True) + .str.strip("_") + ) + + # Remove completely empty rows + df = df.dropna(how="all") + + # Forward fill merged Location + if "Location" in df.columns: + df["Location"] = df["Location"].ffill() + + saved_count = 0 + skipped_count = 0 + + try: + for index, row in df.iterrows(): + record_data = {} + location = row.get("Location") + mh_no = row.get("MH_NO") + + if (pd.isna(location) or str(location).strip() == "" or pd.isna(mh_no) or str(mh_no).strip() == ""): + skipped_count += 1 + continue + + # Map only model columns + for col in df.columns: + if hasattr(ManholeExcavation, col): + value = row[col] + + # Normalize empty values + if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan"]: + value = None + + record_data[col] = value + + # If all mapped fields are None → skip + if all(v is None for v in record_data.values()): + skipped_count += 1 + continue + + record = ManholeExcavation( + subcontractor_id=subcontractor_id, RA_Bill_No=RA_Bill_No, + **record_data + ) + + print("Saving Row → Location:", record.Location, " MH_NO:", record.MH_NO) + + db.session.add(record) + saved_count += 1 + + db.session.commit() + + return True, ( + f"Manhole Excavation saved successfully. " + f"Inserted: {saved_count}, Skipped: {skipped_count}" + ) + + except Exception as e: + db.session.rollback() + return False, f"Manhole Excavation save failed: {e}" + + + +# olds + # def handle_file_upload(self, file, subcontractor_id, file_type): + # # def handle_file_upload(self, file, subcontractor_id): + + # if not subcontractor_id: + # return False, "Please select subcontractor." + # # if not file_type: + # # return False, "Please select file type." + # if not file or file.filename == "": + # return False, "No file selected." + # if not self.allowed_file(file.filename): + # return False, "Invalid file type! Allowed: CSV, XLSX, XLS" + + # ensure_upload_folder() + + # folder = os.path.join(Config.UPLOAD_FOLDER, f"sub_{subcontractor_id}") + # os.makedirs(folder, exist_ok=True) + + # filename = secure_filename(file.filename) + # filepath = os.path.join(folder, filename) + # file.save(filepath) + + # try: + + # # df = pd.read_csv(filepath) if filename.endswith(".csv") else pd.read_excel(filepath) + # df = pd.read_excel(filepath, sheet_name ="Tr.Ex.", header=0) + # df1 = pd.read_excel(filepath, sheet_name="MH Ex.", header=0) + + # print("\n=== Uploaded File Preview ===") + # print(" file name h:",df) + # print("=============================\n") + # print(" file name h1:",df1) + # # print(df.head()) + # print("=============================\n") + + # # Trench Excavation save (subcontractor) + # if file_type == "trench_excavation": + # return self.process_trench_excavation(df, subcontractor_id) + + # # Manhole Excavation save (subcontractor) + # if file_type == "manhole_excavation": + # return self.process_manhole_excavation(df, subcontractor_id) + + # # Manhole and Domestic Chamber Construction save (subcontractor) + # if file_type == "manhole_domestic_chamber": + # return self.process_manhole_domestic_chamber(df, subcontractor_id) + + # # Tr Ex save (client) + # if file_type =="tr_ex_client": + # return self.client_trench_excavation(df, subcontractor_id) + + # # Mh Ex save (client) + # if file_type =="mh_ex_client": + # return self.client_manhole_excavation(df, subcontractor_id) + + # # Mh and Dc save (client) + # if file_type == "mh_dc_client": + # return self.client_manhole_domestic_chamber(df, subcontractor_id) + + # return True, "File uploaded successfully." + + # except Exception as e: + # return False, f"Processing failed: {e}" + + + # ---------------------- Sub contractor -------------------------- + # Trench Excavation save method (TrenchExcavation model) + # def process_trench_excavation(self, df, subcontractor_id): + # df.columns = [str(c).strip() for c in df.columns] + # # If the sheet has merged cells -> forward fill Location + # if "Location" in df.columns: + # df["Location"] = df["Location"].ffill() + + # df = df.dropna(how="all") # REMOVE empty rows + # # Identify missing location rows before insert + # missing_loc = df[df["Location"].isna() | (df["Location"].astype(str).str.strip() == "")] + # if not missing_loc.empty: + # return False, f"Error: Some rows have empty Location. Rows: {missing_loc.index.tolist()}" + + # saved_count = 0 + + # try: + # for index, row in df.iterrows(): + # record_data = {} + # # Insert only fields that exist in model + # for col in df.columns: + # if hasattr(TrenchExcavation, col): + # value = row[col] + # # Normalize empty values + # if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan", "NaN"]: + # value = None + + # record_data[col] = value + + # record = TrenchExcavation( + # subcontractor_id=subcontractor_id, + # **record_data + # ) + + # db.session.add(record) + # saved_count += 1 + + # db.session.commit() + # return True, f"Trench Excavation data saved successfully. Total rows: {saved_count}" + + # except Exception as e: + # db.session.rollback() + # return False, f"Trench Excavation Save Failed: {e}" + + # Manhole Excavation save method (ManholeExcavation model) - def process_manhole_excavation(self, df, subcontractor_id): - # Clean column names (strip whitespace) - df.columns = [str(c).strip() for c in df.columns] - # If the sheet has merged cells -> forward fill Location - if "Location" in df.columns: - df["Location"] = df["Location"].ffill() + # def process_manhole_excavation(self, df, subcontractor_id): + # # Clean column names (strip whitespace) + # df.columns = [str(c).strip() for c in df.columns] + # # If the sheet has merged cells -> forward fill Location + # if "Location" in df.columns: + # df["Location"] = df["Location"].ffill() - # REMOVE empty rows - df = df.dropna(how="all") - # Identify missing location rows before insert - missing_loc = df[df["Location"].isna() | (df["Location"].astype(str).str.strip() == "")] - if not missing_loc.empty: - return False, f"Error: Some rows have empty Location. Rows: {missing_loc.index.tolist()}" + # # REMOVE empty rows + # df = df.dropna(how="all") + # # Identify missing location rows before insert + # missing_loc = df[df["Location"].isna() | (df["Location"].astype(str).str.strip() == "")] + # if not missing_loc.empty: + # return False, f"Error: Some rows have empty Location. Rows: {missing_loc.index.tolist()}" - saved_count = 0 + # saved_count = 0 - try: - for index, row in df.iterrows(): - record_data = {} - # Insert only fields that exist in model - for col in df.columns: - if hasattr(ManholeExcavation, col): - value = row[col] + # try: + # for index, row in df.iterrows(): + # record_data = {} + # # Insert only fields that exist in model + # for col in df.columns: + # if hasattr(ManholeExcavation, col): + # value = row[col] - # Normalize empty values - if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan", "NaN"]: - value = None + # # Normalize empty values + # if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan", "NaN"]: + # value = None - record_data[col] = value + # record_data[col] = value - record = ManholeExcavation( - subcontractor_id=subcontractor_id, - **record_data - ) + # record = ManholeExcavation( + # subcontractor_id=subcontractor_id, + # **record_data + # ) - db.session.add(record) - saved_count += 1 + # db.session.add(record) + # saved_count += 1 - db.session.commit() - return True, f"Manhole Excavation data saved successfully. Total rows: {saved_count}" + # db.session.commit() + # return True, f"Manhole Excavation data saved successfully. Total rows: {saved_count}" - except Exception as e: - db.session.rollback() - return False, f"Manhole Excavation Save Failed: {e}" + # except Exception as e: + # db.session.rollback() + # return False, f"Manhole Excavation Save Failed: {e}" # Manhole and Domestic Chamber Construction save method (ManholeDomesticChamber model) diff --git a/app/templates/file_import.html b/app/templates/file_import.html index 71556d0..aaa977c 100644 --- a/app/templates/file_import.html +++ b/app/templates/file_import.html @@ -36,4 +36,4 @@ -{% endblock %} +{% endblock %} \ No newline at end of file