From f73f6524e809e27c3a768b3cee7c9014bb65d8e2 Mon Sep 17 00:00:00 2001 From: anishd100 Date: Fri, 16 Jan 2026 12:38:38 +0530 Subject: [PATCH] git issue solved --- .env | 2 +- README.md | 571 +----------------- app/models/laying_model.py | 47 ++ app/models/manhole_domestic_chamber_model.py | 1 + app/routes/dashboard.py | 59 +- app/routes/file_report.py | 105 ++-- app/services/file_service.py | 133 ++-- app/templates/base.html | 36 +- app/templates/dashboard.html | 85 ++- ...generate_comparison_client_vs_subcont.html | 20 +- app/templates/login.html | 2 +- app/templates/report.html | 73 ++- app/templates/subcontractor/list.html | 8 +- requirements.txt | 3 +- 14 files changed, 435 insertions(+), 710 deletions(-) create mode 100644 app/models/laying_model.py diff --git a/.env b/.env index f910974..abace4f 100644 --- a/.env +++ b/.env @@ -3,7 +3,7 @@ # ----------------------------- FLASK_ENV=development FLASK_DEBUG=True -FLASK_HOST=0.0.0.0 +FLASK_HOST='0.0.0.0' FLASK_PORT=5001 # ----------------------------- diff --git a/README.md b/README.md index af05bfa..47e1ae1 100644 --- a/README.md +++ b/README.md @@ -1,570 +1,3 @@ -# Comparison Project - Backend Documentation +# Comparison_Project -A Flask-based web application for comparing and managing construction project data (excavation and manhole work) between subcontractors and clients. The system imports data from Excel files, stores it in a database, and generates comparison reports. - -## Table of Contents - -- [Project Overview](#project-overview) -- [Tech Stack](#tech-stack) -- [Project Structure](#project-structure) -- [Database Schema](#database-schema) -- [Application Flow](#application-flow) -- [API Routes & Endpoints](#api-routes--endpoints) -- [Setup & Installation](#setup--installation) - ---- - -## Project Overview - -The Comparison Project is designed to: -- **Manage subcontractors** and their excavation data (Manhole Excavation, Trench Excavation, Domestic Chambers) -- **Import client-side project data** for comparison -- **Compare subcontractor work** against client specifications -- **Generate detailed reports** highlighting differences between client and subcontractor data -- **User authentication** with login/registration system - -### Key Features - -✅ User Authentication (Register/Login/Logout) -✅ Subcontractor Management (Add/Edit/List/Delete) -✅ File Import (Excel/CSV) for both Client and Subcontractor data -✅ Data validation and duplicate detection -✅ Comparison Reports with Excel export -✅ Dashboard with file management - ---- - -## Tech Stack - -**Backend Framework**: Flask -**Database**: SQL Database (MySQL/PostgreSQL/SQLite configured via environment variables) -**ORM**: SQLAlchemy -**File Processing**: Pandas, OpenPyXL, XlsxWriter -**Authentication**: Werkzeug (Password hashing) -**Configuration**: Python Dotenv - -### Dependencies - -``` -Flask -pandas -openpyxl -xlrd -Werkzeug -python-dotenv -cryptography -xlsxwriter -``` - ---- - -## Project Structure - -``` -Comparison_Project/ -├── run.py # Application entry point -├── requirements.txt # Python dependencies -├── .env # Environment variables (DB, secrets) -├── README.md # This file -│ -├── app/ -│ ├── __init__.py # Flask app initialization -│ ├── config.py # Configuration settings -│ │ -│ ├── models/ # Database models (SQLAlchemy ORM) -│ │ ├── user_model.py # User table schema -│ │ ├── subcontractor_model.py # Subcontractor table schema -│ │ ├── manhole_excavation_model.py # Subcontractor MH excavation -│ │ ├── trench_excavation_model.py # Subcontractor Trench excavation -│ │ ├── manhole_domestic_chamber_model.py # Subcontractor Domestic chamber -│ │ ├── mh_ex_client_model.py # Client MH excavation -│ │ ├── tr_ex_client_model.py # Client Trench excavation -│ │ └── mh_dc_client_model.py # Client Domestic chamber -│ │ -│ ├── routes/ # Flask Blueprints (API endpoints) -│ │ ├── auth.py # Login, Register, Logout -│ │ ├── dashboard.py # Dashboard page -│ │ ├── file_import.py # File upload endpoints -│ │ ├── file_report.py # Fetch and format data for reports -│ │ ├── generate_comparison_report.py # Comparison logic & Excel export -│ │ ├── subcontractor_routes.py # CRUD for subcontractors -│ │ ├── user.py # User management routes -│ │ └── file_format.py # File format validation -│ │ -│ ├── services/ # Business logic layer -│ │ ├── db_service.py # Database connection & SQLAlchemy init -│ │ ├── file_service.py # File parsing & data insertion -│ │ └── user_service.py # User authentication logic -│ │ -│ ├── utils/ # Helper functions -│ │ ├── file_utils.py # File path utilities -│ │ ├── helpers.py # Decorators, common helpers -│ │ └── __pycache__/ -│ │ -│ ├── static/ # Static assets -│ │ ├── css/ # Stylesheets -│ │ ├── images/ # Images -│ │ ├── uploads/ # Uploaded files directory -│ │ │ ├── Client_Bill_1/ -│ │ │ └── sub_1/ -│ │ └── downloads/ # Generated reports -│ │ -│ ├── templates/ # Jinja2 HTML templates -│ │ ├── base.html # Base template -│ │ ├── login.html -│ │ ├── register.html -│ │ ├── dashboard.html -│ │ ├── file_import.html -│ │ ├── file_import_client.html -│ │ ├── file_format.html -│ │ ├── file_report.html -│ │ ├── generate_comparison_report.html -│ │ ├── generate_comparison_client_vs_subcont.html -│ │ ├── list_user.html -│ │ ├── report.html -│ │ ├── users.htm -│ │ └── subcontractor/ -│ │ ├── add.html -│ │ ├── edit.html -│ │ └── list.html -│ │ -│ └── logs/ # Application logs -│ -├── instance/ # Instance folder (DB, temp files) -└── logs/ # Root level logs -``` - ---- - -## Database Schema - -### 1. **Users Table** (`users`) -Stores user authentication data. - -| Column | Type | Constraints | Purpose | -|--------|------|-------------|---------| -| `id` | Integer | Primary Key | User identifier | -| `name` | String(120) | NOT NULL | User's full name | -| `email` | String(120) | UNIQUE, NOT NULL | User's email | -| `password_hash` | String(255) | NOT NULL | Hashed password (Werkzeug) | - -**Key Methods:** -- `set_password(password)` - Hash and store password -- `check_password(password)` - Verify password - ---- - -### 2. **Subcontractors Table** (`subcontractors`) -Stores subcontractor company information. - -| Column | Type | Constraints | Purpose | -|--------|------|-------------|---------| -| `id` | Integer | Primary Key | Subcontractor ID | -| `subcontractor_name` | String(255) | NOT NULL | Company name | -| `address` | String(500) | - | Company address | -| `gst_no` | String(50) | - | GST number | -| `pan_no` | String(50) | - | PAN number | -| `mobile_no` | String(20) | - | Contact phone | -| `email_id` | String(150) | - | Contact email | -| `contact_person` | String(150) | - | Contact person name | -| `status` | String(20) | Default: "Active" | Active/Inactive status | -| `created_at` | DateTime | Default: today | Record creation date | - -**Relationships:** -- One-to-Many with `manhole_excavation` -- One-to-Many with `trench_excavation` -- One-to-Many with `manhole_domestic_chamber` - ---- - -### 3. **Manhole Excavation (Subcontractor)** (`manhole_excavation`) -Stores manhole excavation work data from subcontractors. - -| Column Type | Purpose | -|-----|---------| -| `id` | Primary Key | -| `subcontractor_id` | Foreign Key → Subcontractor | -| `Location` | Worksite location | -| `MH_NO` | Manhole number | -| `RA_Bill_No` | RA Bill reference (for grouping) | -| `Upto_IL_Depth`, `Cutting_Depth`, `ID_of_MH_m`, `Ex_Dia_of_Manhole`, `Area_of_Manhole` | Dimension fields | -| **Excavation Categories** (by depth ranges): | | -| Soft_Murum_0_to_1_5, Soft_Murum_1_5_to_3_0, etc. | Material type + depth range | -| Hard_Murum_0_to_1_5, Hard_Murum_1_5_to_3_0, etc. | Hard mudstone layers | -| Soft_Rock_0_to_1_5, Soft_Rock_1_5_to_3_0, etc. | Soft rock layers | -| Hard_Rock_0_to_1_5 through Hard_Rock_6_0_to_7_5 | Hard rock layers | -| **Totals** | Sum per category | -| `Total` | Grand total excavation | -| `Remarks`, `created_at` | Notes and timestamp | - ---- - -### 4. **Trench Excavation (Subcontractor)** (`trench_excavation`) -Stores trench/sewer line excavation work data from subcontractors. - -**Similar structure to Manhole Excavation with additional fields:** -- Width categories: `Width_0_to_2_5`, `Width_2_5_to_3_0`, etc. -- `Avg_Depth`, `Actual_Trench_Length`, `Pipe_Dia_mm` - ---- - -### 5. **Manhole Domestic Chamber (Subcontractor)** (`manhole_domestic_chamber`) -Stores domestic chamber construction data. - -| Key Fields | Purpose | -|-----------|---------| -| `id`, `subcontractor_id` | Primary & Foreign Key | -| `Location`, `MH_NO`, `RA_Bill_No` | Identification | -| `Depth_of_MH`, `MH_TOP_LEVEL`, `MH_IL_LEVEL` | Dimensions | -| `d_0_to_1_5` through `d_6_0_to_6_5` | Depth-based excavation categories | -| `Domestic_Chambers` | Count of chambers | -| `DWC_Pipe_Length`, `UPVC_Pipe_Length` | Pipe lengths | - ---- - -### 6-8. **Client Data Tables** (Similar structure to subcontractor models) - -- **`mh_ex_client`** - Manhole Excavation (Client specifications) -- **`tr_ex_client`** - Trench Excavation (Client specifications) -- **`mh_dc_client`** - Manhole Domestic Chamber (Client specifications) - -**Note:** Client tables do NOT have `subcontractor_id` as they represent client baseline data. - ---- - -## Application Flow - -### 1. **User Authentication Flow** -``` -User Access (/) - ↓ -Redirect to /login - ↓ -[Login Page] - ├─ POST /auth/login - │ ↓ - │ UserService.validate_login(email, password) - │ ↓ - │ Query User table, verify password - │ ↓ - │ Set session["user_id"], session["user_name"] - ↓ -Redirect to /dashboard -``` - -### 2. **Subcontractor File Import Flow** -``` -User → /file/import [GET] - ↓ -Display form with subcontractor dropdown - ↓ -User selects subcontractor & uploads Excel file - ↓ -POST /file/import - ↓ -FileService.handle_file_upload() - ├─ Validate file type (xlsx, xls, csv) - ├─ Create upload folder: static/uploads/sub_{id}/ - ├─ Save file - │ - ├─ Read Excel sheets: - │ ├─ "Tr.Ex." (Trench Excavation) - │ ├─ "MH Ex." (Manhole Excavation) - │ └─ "MH & DC" (Manhole & Domestic Chamber) - │ - ├─ Process each sheet: - │ ├─ Normalize column names - │ ├─ Clean data (handle NaN, empty values) - │ ├─ Check for duplicates by (Location, MH_NO, RA_Bill_No, subcontractor_id) - │ ├─ Insert records into respective tables - │ └─ Commit to database - │ - └─ Return success/error message -``` - -### 3. **Client File Import Flow** -``` -User → /file/import_client [GET] - ↓ -Display form for client file upload - ↓ -User uploads Excel file with client specifications - ↓ -POST /file/import_client - ↓ -FileService.handle_client_file_upload() - ├─ Validate & save file - ├─ Read sheets: "Tr.Ex.", "MH Ex.", "MH & DC" - ├─ Process and insert into: - │ ├─ mh_ex_client - │ ├─ tr_ex_client - │ └─ mh_dc_client - └─ Return status -``` - -### 4. **Comparison Report Generation Flow** -``` -User → /report/generate_comparison [GET] - ↓ -Display form to select RA Bill No. - ↓ -User selects bill number - ↓ -POST /report/generate_comparison - ↓ -generate_report_bp routes request - ├─ Fetch client data (mh_ex_client, tr_ex_client, mh_dc_client) - ├─ Fetch subcontractor data (by RA_Bill_No) - ├─ For each record type: - │ ├─ Create lookup table by (Location, MH_NO) - │ ├─ Match client records with subcontractor records - │ ├─ Calculate totals for each category - │ ├─ Compute differences/variances - │ └─ Format for Excel output - │ - ├─ Generate Excel file with comparison results: - │ ├─ Summary sheet - │ ├─ Detailed Trench Excavation comparison - │ ├─ Detailed Manhole Excavation comparison - │ └─ Detailed Domestic Chamber comparison - │ - └─ Send file to client as download -``` - -### 5. **Dashboard & Data Retrieval Flow** -``` -User → /dashboard [GET] - ↓ -Check session["user_id"] (authentication) - ↓ -Render dashboard.html - ├─ Display available reports - ├─ List recent RA Bills - ├─ Show subcontractor list - └─ Provide navigation to: - ├─ File import - ├─ Reports - ├─ Subcontractor management - └─ Logout -``` - ---- - -## API Routes & Endpoints - -### Authentication Routes (`/auth`) -| Method | Endpoint | Purpose | Auth | -|--------|----------|---------|------| -| GET/POST | `/login` | Login form & validation | ❌ | -| GET/POST | `/register` | User registration | ❌ | -| GET | `/logout` | Clear session & logout | ✅ | - -### Dashboard Route (`/dashboard`) -| Method | Endpoint | Purpose | Auth | -|--------|----------|---------|------| -| GET | `/dashboard/` | Main dashboard | ✅ | - -### Subcontractor Routes (`/subcontractor`) -| Method | Endpoint | Purpose | Auth | -|--------|----------|---------|------| -| GET | `/subcontractor/add` | Add form | ✅ | -| POST | `/subcontractor/save` | Save new subcontractor | ✅ | -| GET | `/subcontractor/list` | List all subcontractors | ✅ | -| GET | `/subcontractor/edit/` | Edit form | ✅ | -| POST | `/subcontractor/update/` | Update subcontractor | ✅ | -| GET | `/subcontractor/delete/` | Delete subcontractor | ✅ | - -### File Import Routes (`/file`) -| Method | Endpoint | Purpose | Auth | -|--------|----------|---------|------| -| GET/POST | `/file/import` | Subcontractor file upload | ✅ | -| GET/POST | `/file/import_client` | Client file upload | ✅ | -| GET/POST | `/file/report` | View imported data report | ✅ | - -### Report Routes (`/report`) -| Method | Endpoint | Purpose | Auth | -|--------|----------|---------|------| -| GET/POST | `/report/generate_comparison` | Generate comparison report | ✅ | -| GET/POST | `/report/generate_comparison_client_vs_subcont` | Alternative comparison | ✅ | - -### User Routes (`/user`) -| Method | Endpoint | Purpose | Auth | -|--------|----------|---------|------| -| GET | `/user/list` | List all users | ✅ | -| Other | (Check routes) | User management | ✅ | - -### File Format Route (`/format`) -| Method | Endpoint | Purpose | Auth | -|--------|----------|---------|------| -| GET/POST | `/format/...` | File format reference | ✅ | - ---- - -## Setup & Installation - -### Prerequisites -- Python 3.7+ -- MySQL/PostgreSQL (or SQLite for development) -- pip (Python package manager) - -### 1. Clone & Install Dependencies -```bash -cd d:\New folder\Comparison\Comparison_Project -pip install -r requirements.txt -``` - -### 2. Configure Environment Variables -Create a `.env` file in the project root: -```env -# Flask Configuration -FLASK_HOST=127.0.0.1 -FLASK_PORT=5000 -FLASK_DEBUG=True -SECRET_KEY=your-secret-key-here - -# Database Configuration -DB_DIALECT=mysql # or postgresql, sqlite -DB_DRIVER=pymysql # or psycopg2, etc. -DB_USER=root -DB_PASSWORD=your_password -DB_HOST=localhost -DB_PORT=3306 -DB_NAME=comparison_project -``` - -### 3. Initialize Database -```bash -python run.py -``` -This will: -- Load environment variables -- Create Flask app with configuration -- Initialize SQLAlchemy -- Create all tables (if not exist) -- Start Flask development server - -### 4. Access Application -Open browser: `http://127.0.0.1:5000/` -- First time: Redirect to `/login` -- Register a new account -- Login and start using the application - ---- - -## Key Services & Utilities - -### FileService (`app/services/file_service.py`) -**Purpose:** Handles file uploads, parsing, and data validation - -**Key Methods:** -- `handle_file_upload(file, subcontractor_id, RA_Bill_No)` - Upload & process subcontractor file -- `handle_client_file_upload(file, RA_Bill_No)` - Upload & process client file -- `process_trench_excavation(df, subcontractor_id, RA_Bill_No)` - Parse & insert trench data -- `process_manhole_excavation(df, subcontractor_id, RA_Bill_No)` - Parse & insert manhole data -- `process_manhole_domestic_chamber(df, subcontractor_id, RA_Bill_No)` - Parse & insert chamber data - -### UserService (`app/services/user_service.py`) -**Purpose:** User authentication & management - -**Key Methods:** -- `register_user(name, email, password)` - Register new user -- `validate_login(email, password)` - Authenticate user -- `get_all_users()` - Fetch all users - -### DBService (`app/services/db_service.py`) -**Purpose:** Database connection & SQLAlchemy initialization - -**Exports:** -- `db` - SQLAlchemy instance for ORM operations -- `migrate` - Flask-Migrate for schema migrations - ---- - -## Authentication & Security - -### Session Management -- Uses Flask `session` object -- Stores `user_id` and `user_name` after successful login -- `@login_required` decorator validates authenticated requests - -### Password Security -- Passwords hashed using Werkzeug's `generate_password_hash()` -- Verification via `check_password_hash()` -- No plaintext passwords stored in database - -### File Security -- File uploads sanitized with `secure_filename()` -- Only allowed extensions: `.xlsx`, `.xls`, `.csv` -- Files stored in user-specific subfolders - ---- - -## Error Handling & Validation - -### File Import Validation -1. **File Type Check** - Only CSV/XLS/XLSX allowed -2. **Sheet Validation** - Required sheets must exist -3. **Column Normalization** - Auto-fixes column name inconsistencies -4. **Data Type Conversion** - Converts to appropriate types -5. **Duplicate Detection** - Prevents duplicate records by (Location, MH_NO, RA_Bill_No) -6. **Null Handling** - Converts empty/NaN values to None - -### Database Constraints -- Foreign key relationships enforced -- Unique email constraint on users -- NOT NULL constraints on critical fields - ---- - -## Example: Typical User Workflow - -``` -1. User registers/logs in - → POST /auth/register (new user) or /auth/login - -2. User adds subcontractors - → GET /subcontractor/add - → POST /subcontractor/save - -3. User uploads subcontractor data (Excel file with 3 sheets) - → GET /file/import - → POST /file/import (file, subcontractor_id, RA_Bill_No) - → Database populated with excavation data - -4. User uploads client specifications (Excel file with same 3 sheets) - → GET /file/import_client - → POST /file/import_client (file, RA_Bill_No) - → Database populated with client data - -5. User generates comparison report - → GET /report/generate_comparison - → POST /report/generate_comparison (RA_Bill_No) - → System compares client vs subcontractor data - → Generates Excel file showing differences - → User downloads report - -6. User logs out - → GET /auth/logout - → Session cleared, redirected to login -``` - ---- - -## Future Enhancements - -- [ ] Email notifications for report generation -- [ ] Data visualization dashboards -- [ ] Role-based access control (Admin, User, Viewer) -- [ ] Bulk report generation -- [ ] Data export to PDF format -- [ ] Audit logs for data modifications -- [ ] API documentation (Swagger/OpenAPI) -- [ ] Unit & integration tests - ---- - -## Support & Contribution - -For issues, feature requests, or contributions, please contact the development team. - -**Last Updated:** January 2026 \ No newline at end of file +Comparison Project \ No newline at end of file diff --git a/app/models/laying_model.py b/app/models/laying_model.py new file mode 100644 index 0000000..e43a1f1 --- /dev/null +++ b/app/models/laying_model.py @@ -0,0 +1,47 @@ +from app import db +from datetime import datetime + +class Laying(db.Model): + __tablename__ = "laying" + + id = db.Column(db.Integer, primary_key=True) + # Foreign Key to Subcontractor table + subcontractor_id = db.Column(db.Integer, db.ForeignKey("subcontractors.id"), nullable=False) + # Relationship for easy access (subcontractor.subcontractor_name) + subcontractor = db.relationship("Subcontractor", backref="laying_records") + + # Basic Fields + Location = db.Column(db.String(500)) + MH_NO = db.Column(db.String(100)) + CC_length = db.Column(db.Float) + + Pipe_Dia_mm = db.Column(db.Float) + ID_of_MH_m = db.Column(db.Float) + Laying_Length = db.Column(db.Float) + + pipe_150_mm = db.Column(db.Float) + pipe_200_mm = db.Column(db.Float) + pipe_250_mm = db.Column(db.Float) + pipe_300_mm = db.Column(db.Float) + pipe_350_mm = db.Column(db.Float) + pipe_400_mm = db.Column(db.Float) + pipe_450_mm = db.Column(db.Float) + pipe_500_mm = db.Column(db.Float) + pipe_600_mm = db.Column(db.Float) + pipe_700_mm = db.Column(db.Float) + pipe_900_mm = db.Column(db.Float) + pipe_1200_mm = db.Column(db.Float) + + + Total = db.Column(db.Float) + Remarks = db.Column(db.String(500)) + RA_Bill_No=db.Column(db.String(500)) + + created_at = db.Column(db.DateTime, default=datetime.today) + + + def __repr__(self): + return f"" + + def serialize(self): + return {c.name: getattr(self, c.name) for c in self.__table__.columns} diff --git a/app/models/manhole_domestic_chamber_model.py b/app/models/manhole_domestic_chamber_model.py index 98d4881..0902f0d 100644 --- a/app/models/manhole_domestic_chamber_model.py +++ b/app/models/manhole_domestic_chamber_model.py @@ -49,3 +49,4 @@ class ManholeDomesticChamber(db.Model): def serialize(self): return {c.name: getattr(self, c.name) for c in self.__table__.columns} + diff --git a/app/routes/dashboard.py b/app/routes/dashboard.py index 04a4544..3aebc4c 100644 --- a/app/routes/dashboard.py +++ b/app/routes/dashboard.py @@ -1,11 +1,68 @@ +import matplotlib +matplotlib.use("Agg") from flask import Blueprint, render_template, session, redirect, url_for +import matplotlib.pyplot as plt +import io +import base64 dashboard_bp = Blueprint("dashboard", __name__, url_prefix="/dashboard") +# charts +def plot_to_base64(): + img = io.BytesIO() + plt.savefig(img, format="png", bbox_inches="tight") + plt.close() + img.seek(0) + return base64.b64encode(img.getvalue()).decode() + +# bar chart +def bar_chart(): + categories = ["Trench", "Manhole", "Pipe Laying", "Restoration"] + values = [120, 80, 150, 60] + + plt.figure() + plt.bar(categories, values) + plt.title("Work Category Report") + plt.xlabel("Category") + plt.ylabel("Count") + + + return plot_to_base64() + +# Pie chart +def pie_chart(): + labels = ["Completed", "In Progress", "Pending"] + sizes = [55, 20, 25] + + plt.figure() + plt.pie(sizes, labels=labels, autopct="%1.1f%%", startangle=140) + plt.title("Project Status") + + return plot_to_base64() + +# Histogram chart +def histogram_chart(): + daily_work = [5, 10, 15, 20, 20, 25, 30, 35, 40, 45, 50] + + plt.figure() + plt.hist(daily_work, bins=5) + plt.title("Daily Work Distribution") + plt.xlabel("Work Units") + plt.ylabel("Frequency") + + return plot_to_base64() + +# Dashboaed page @dashboard_bp.route("/") def dashboard(): if not session.get("user_id"): return redirect(url_for("auth.login")) - return render_template("dashboard.html", title="Dashboard") + return render_template( + "dashboard.html", + title="Dashboard", + bar_chart=bar_chart(), + pie_chart=pie_chart(), + histogram=histogram_chart() + ) diff --git a/app/routes/file_report.py b/app/routes/file_report.py index 681b626..40b7bd8 100644 --- a/app/routes/file_report.py +++ b/app/routes/file_report.py @@ -9,66 +9,72 @@ from app.models.mh_ex_client_model import ManholeExcavationClient from app.models.tr_ex_client_model import TrenchExcavationClient from app.models.mh_dc_client_model import ManholeDomesticChamberClient from app.utils.helpers import login_required - +from app.models.laying_model import Laying + # --- BLUEPRINT DEFINITION --- file_report_bp = Blueprint("file_report", __name__, url_prefix="/file") - + # --- DATA WRAPPERS --- - + class ClientBill: def __init__(self): self.df_tr = pd.DataFrame() self.df_mh = pd.DataFrame() self.df_dc = pd.DataFrame() - + def Fetch(self, RA_Bill_No): trench = TrenchExcavationClient.query.filter_by(RA_Bill_No=RA_Bill_No).all() mh = ManholeExcavationClient.query.filter_by(RA_Bill_No=RA_Bill_No).all() dc = ManholeDomesticChamberClient.query.filter_by(RA_Bill_No=RA_Bill_No).all() - + self.df_tr = pd.DataFrame([c.serialize() for c in trench]) self.df_mh = pd.DataFrame([c.serialize() for c in mh]) self.df_dc = pd.DataFrame([c.serialize() for c in dc]) - + if not self.df_dc.empty and "MH_NO" in self.df_dc.columns: self.df_dc.rename(columns={"MH_NO": "Node_No"}, inplace=True) - + drop_cols = ["id", "created_at", "_sa_instance_state"] for df in [self.df_tr, self.df_mh, self.df_dc]: if not df.empty: df.drop(columns=drop_cols, errors="ignore", inplace=True) - + class SubcontractorBill: def __init__(self): self.df_tr = pd.DataFrame() self.df_mh = pd.DataFrame() self.df_dc = pd.DataFrame() - + self.df_lay = pd.DataFrame() + def Fetch(self, RA_Bill_No=None, subcontractor_id=None): filters = {} if subcontractor_id: filters["subcontractor_id"] = subcontractor_id if RA_Bill_No: filters["RA_Bill_No"] = RA_Bill_No - + trench = TrenchExcavation.query.filter_by(**filters).all() mh = ManholeExcavation.query.filter_by(**filters).all() dc = ManholeDomesticChamber.query.filter_by(**filters).all() + lay= Laying.query.filter_by(**filters).all() + print("Laying datas :",lay) + self.df_tr = pd.DataFrame([c.serialize() for c in trench]) self.df_mh = pd.DataFrame([c.serialize() for c in mh]) self.df_dc = pd.DataFrame([c.serialize() for c in dc]) - + self.df_lay = pd.DataFrame([c.serialize() for c in lay]) + if not self.df_dc.empty and "MH_NO" in self.df_dc.columns: self.df_dc.rename(columns={"MH_NO": "Node_No"}, inplace=True) - + drop_cols = ["id", "created_at", "_sa_instance_state"] - for df in [self.df_tr, self.df_mh, self.df_dc]: + for df in [self.df_tr, self.df_mh, self.df_dc, self.df_lay]: if not df.empty: df.drop(columns=drop_cols, errors="ignore", inplace=True) - + # --- ROUTES --- - + # @file_report_bp.route("/report", methods=["GET", "POST"]) # @login_required # def report_file(): @@ -77,14 +83,14 @@ class SubcontractorBill: # subcontractor_id = request.form.get("subcontractor_id") # ra_bill_no = request.form.get("ra_bill_no") # download_all = request.form.get("download_all") == "true" - + # if not subcontractor_id: # flash("Please select a subcontractor.", "danger") # return render_template("report.html", subcontractors=subcontractors) - + # subcontractor = Subcontractor.query.get(subcontractor_id) # bill_gen = SubcontractorBill() - + # if download_all: # bill_gen.Fetch(subcontractor_id=subcontractor_id) # file_name = f"{subcontractor.subcontractor_name}_ALL_BILLS.xlsx" @@ -94,11 +100,11 @@ class SubcontractorBill: # return render_template("report.html", subcontractors=subcontractors) # bill_gen.Fetch(RA_Bill_No=ra_bill_no, subcontractor_id=subcontractor_id) # file_name = f"{subcontractor.subcontractor_name}_RA_{ra_bill_no}_Report.xlsx" - + # if bill_gen.df_tr.empty and bill_gen.df_mh.empty and bill_gen.df_dc.empty: # flash("No data found for this selection.", "warning") # return render_template("report.html", subcontractors=subcontractors) - + # output = io.BytesIO() # with pd.ExcelWriter(output, engine="xlsxwriter") as writer: # bill_gen.df_tr.to_excel(writer, index=False, sheet_name="Tr.Ex.") @@ -106,7 +112,7 @@ class SubcontractorBill: # bill_gen.df_dc.to_excel(writer, index=False, sheet_name="MH & DC") # output.seek(0) # return send_file(output, download_name=file_name, as_attachment=True, mimetype="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet") - + # return render_template("report.html", subcontractors=subcontractors) @file_report_bp.route("/report", methods=["GET", "POST"]) @login_required @@ -116,20 +122,20 @@ def report_file(): selected_sc_id = None ra_bill_no = None download_all = False - + if request.method == "POST": subcontractor_id = request.form.get("subcontractor_id") ra_bill_no = request.form.get("ra_bill_no") download_all = request.form.get("download_all") == "true" - action = request.form.get("action") - + action = request.form.get("action") + if not subcontractor_id: flash("Please select a subcontractor.", "danger") return render_template("report.html", subcontractors=subcontractors) - + subcontractor = Subcontractor.query.get(subcontractor_id) bill_gen = SubcontractorBill() - + if download_all: bill_gen.Fetch(subcontractor_id=subcontractor_id) file_name = f"{subcontractor.subcontractor_name}_ALL_BILLS.xlsx" @@ -139,11 +145,11 @@ def report_file(): return render_template("report.html", subcontractors=subcontractors) bill_gen.Fetch(RA_Bill_No=ra_bill_no, subcontractor_id=subcontractor_id) file_name = f"{subcontractor.subcontractor_name}_RA_{ra_bill_no}_Report.xlsx" - + if bill_gen.df_tr.empty and bill_gen.df_mh.empty and bill_gen.df_dc.empty: flash("No data found for this selection.", "warning") return render_template("report.html", subcontractors=subcontractors) - + # If download is clicked, return file immediately if action == "download": output = io.BytesIO() @@ -151,57 +157,59 @@ def report_file(): bill_gen.df_tr.to_excel(writer, index=False, sheet_name="Tr.Ex.") bill_gen.df_mh.to_excel(writer, index=False, sheet_name="MH.Ex.") bill_gen.df_dc.to_excel(writer, index=False, sheet_name="MH & DC") + bill_gen.df_lay.to_excel(writer, index=False, sheet_name="Laying") output.seek(0) return send_file(output, download_name=file_name, as_attachment=True) - + # PREVIEW LOGIC: Convert DataFrames to HTML for the 'tables' variable # We add bootstrap classes directly to the pandas output table_classes = "table table-hover table-bordered table-striped" tables = { "tr": bill_gen.df_tr.to_html(classes=table_classes, index=False), "mh": bill_gen.df_mh.to_html(classes=table_classes, index=False), - "dc": bill_gen.df_dc.to_html(classes=table_classes, index=False) + "dc": bill_gen.df_dc.to_html(classes=table_classes, index=False), + "lay": bill_gen.df_lay.to_html(classes=table_classes, index=False) } selected_sc_id = subcontractor_id - + return render_template( - "report.html", - subcontractors=subcontractors, + "report.html", + subcontractors=subcontractors, tables=tables, # This now matches your HTML selected_sc_id=selected_sc_id, ra_bill_no=ra_bill_no, download_all=download_all ) - + @file_report_bp.route("/client_vs_subcont", methods=["GET", "POST"]) @login_required def client_vs_all_subcontractor(): tables = {"tr": None, "mh": None, "dc": None} ra_val = "" - + if request.method == "POST": RA_Bill_No = request.form.get("RA_Bill_No") ra_val = RA_Bill_No - + if not RA_Bill_No: flash("Please enter RA Bill No.", "danger") return render_template("generate_comparison_client_vs_subcont.html", tables=tables, ra_val=ra_val) - + clientBill = ClientBill() clientBill.Fetch(RA_Bill_No=RA_Bill_No) contractorBill = SubcontractorBill() contractorBill.Fetch(RA_Bill_No=RA_Bill_No) - + # --- SAFETY CHECK: Verify data exists before merging --- if clientBill.df_tr.empty and clientBill.df_mh.empty: flash(f"No Client records found for RA Bill {RA_Bill_No}", "warning") return render_template("generate_comparison_client_vs_subcont.html", tables=tables, ra_val=ra_val) - + qty_cols = [...] # (Keep your existing list) mh_dc_qty_cols = [...] # (Keep your existing list) - + def aggregate_df(df, group_cols, sum_cols): - if df.empty: + if df.empty: # Create an empty DF with the correct columns to avoid Merge/Key Errors return pd.DataFrame(columns=group_cols + sum_cols) existing_cols = [c for c in sum_cols if c in df.columns] @@ -210,18 +218,18 @@ def client_vs_all_subcontractor(): if col not in df.columns: df[col] = "N/A" # Fill missing join keys return df.groupby(group_cols, as_index=False)[existing_cols].sum() - + # Aggregate data df_sub_tr_grp = aggregate_df(contractorBill.df_tr, ["Location", "MH_NO"], qty_cols) df_sub_mh_grp = aggregate_df(contractorBill.df_mh, ["Location", "MH_NO"], qty_cols) df_sub_dc_grp = aggregate_df(contractorBill.df_dc, ["Location", "Node_No"], mh_dc_qty_cols) - + # --- FINAL MERGE LOGIC --- # We check if "Location" exists in the client data. If not, we add it to prevent the KeyError. for df_client in [clientBill.df_tr, clientBill.df_mh, clientBill.df_dc]: if not df_client.empty and "Location" not in df_client.columns: df_client["Location"] = "Unknown" - + try: df_tr_cmp = clientBill.df_tr.merge(df_sub_tr_grp, on=["Location", "MH_NO"], how="left", suffixes=("_Client", "_Sub")) df_mh_cmp = clientBill.df_mh.merge(df_sub_mh_grp, on=["Location", "MH_NO"], how="left", suffixes=("_Client", "_Sub")) @@ -229,12 +237,13 @@ def client_vs_all_subcontractor(): except KeyError as e: flash(f"Merge Error: Missing column {str(e)}. Check if 'Location' is defined in your database models.", "danger") return render_template("generate_comparison_client_vs_subcont.html", tables=tables, ra_val=ra_val) - + # ... (Rest of your calculation and download logic) ... - + # Convert to HTML for preview tables["tr"] = df_tr_cmp.to_html(classes='table table-striped table-hover table-sm', index=False) tables["mh"] = df_mh_cmp.to_html(classes='table table-striped table-hover table-sm', index=False) tables["dc"] = df_dc_cmp.to_html(classes='table table-striped table-hover table-sm', index=False) - - return render_template("generate_comparison_client_vs_subcont.html", tables=tables, ra_val=ra_val) \ No newline at end of file + + return render_template("generate_comparison_client_vs_subcont.html", tables=tables, ra_val=ra_val) + \ No newline at end of file diff --git a/app/services/file_service.py b/app/services/file_service.py index e98e3f7..cbb07f3 100644 --- a/app/services/file_service.py +++ b/app/services/file_service.py @@ -1,19 +1,22 @@ import os import pandas as pd from werkzeug.utils import secure_filename +from app.utils.file_utils import ensure_upload_folder from app.config import Config from app import db +# Subcontractor models import from app.models.trench_excavation_model import TrenchExcavation from app.models.manhole_excavation_model import ManholeExcavation from app.models.manhole_domestic_chamber_model import ManholeDomesticChamber +from app.models.laying_model import Laying +# Client models import from app.models.tr_ex_client_model import TrenchExcavationClient from app.models.mh_ex_client_model import ManholeExcavationClient from app.models.mh_dc_client_model import ManholeDomesticChamberClient -from app.utils.file_utils import ensure_upload_folder class FileService: @@ -60,10 +63,12 @@ class FileService: df_tr_ex = pd.read_excel(filepath, sheet_name="Tr.Ex.", header=12, dtype={"MH No": str}) df_mh_ex = pd.read_excel(filepath, sheet_name="MH Ex.", header=12, dtype={"MH No": str}) df_mh_dc = pd.read_excel(filepath, sheet_name="MH & DC", header=11, dtype={"MH No": str}) + df_laying = pd.read_excel(filepath, sheet_name="Laying", header=11, dtype={"MH No": str}) self.process_trench_excavation(df_tr_ex, subcontractor_id, RA_Bill_No) self.process_manhole_excavation(df_mh_ex, subcontractor_id, RA_Bill_No) self.process_manhole_domestic_chamber(df_mh_dc, subcontractor_id, RA_Bill_No) + self.process_laying(df_laying, subcontractor_id, RA_Bill_No) return True, "SUBCONTRACTOR File uploaded successfully." @@ -96,18 +101,18 @@ class FileService: if not location or not mh_no: continue - exists = TrenchExcavation.query.filter_by( - subcontractor_id=subcontractor_id, - RA_Bill_No=RA_Bill_No, - Location=location, - MH_NO=mh_no, - ).first() + # exists = TrenchExcavation.query.filter_by( + # subcontractor_id=subcontractor_id, + # RA_Bill_No=RA_Bill_No, + # Location=location, + # MH_NO=mh_no, + # ).first() - if exists: - errors.append( - f"Model-Tr.Ex. (Row {idx+1}): Duplicate → Location={location}, MH_NO={mh_no}" - ) - continue + # if exists: + # errors.append( + # f"Model-Tr.Ex. (Row {idx+1}): Duplicate → Location={location}, MH_NO={mh_no}" + # ) + # continue record_data = {} for col in df.columns: @@ -154,18 +159,18 @@ class FileService: if not location or not mh_no: continue - exists = ManholeExcavation.query.filter_by( - subcontractor_id=subcontractor_id, - RA_Bill_No=RA_Bill_No, - Location=location, - MH_NO=mh_no, - ).first() + # exists = ManholeExcavation.query.filter_by( + # subcontractor_id=subcontractor_id, + # RA_Bill_No=RA_Bill_No, + # Location=location, + # MH_NO=mh_no, + # ).first() - if exists: - errors.append( - f"Model-MH Ex. (Row {idx+1}): Duplicate → Location={location}, MH_NO={mh_no}" - ) - continue + # if exists: + # errors.append( + # f"Model-MH Ex. (Row {idx+1}): Duplicate → Location={location}, MH_NO={mh_no}" + # ) + # continue record_data = {} for col in df.columns: @@ -212,18 +217,18 @@ class FileService: if not location or not mh_no: continue - exists = ManholeDomesticChamber.query.filter_by( - subcontractor_id=subcontractor_id, - RA_Bill_No=RA_Bill_No, - Location=location, - MH_NO=mh_no, - ).first() + # exists = ManholeDomesticChamber.query.filter_by( + # subcontractor_id=subcontractor_id, + # RA_Bill_No=RA_Bill_No, + # Location=location, + # MH_NO=mh_no, + # ).first() - if exists: - errors.append( - f"Model-MH & DC (Row {idx+1}): Duplicate → Location={location}, MH_NO={mh_no}" - ) - continue + # if exists: + # errors.append( + # f"Model-MH & DC (Row {idx+1}): Duplicate → Location={location}, MH_NO={mh_no}" + # ) + # continue record_data = {} for col in df.columns: @@ -245,6 +250,66 @@ class FileService: db.session.commit() + # ---------------- Laying (Subcontractor) ---------------- + def process_laying(self, df, subcontractor_id, RA_Bill_No): + + df.columns = ( + df.columns.astype(str) + .str.strip() + .str.replace(r"[^\w]", "_", regex=True) + .str.replace("__+", "_", regex=True) + .str.strip("_") + ) + + df = df.dropna(how="all") + + if "Location" in df.columns: + df["Location"] = df["Location"].ffill() + + errors = [] + + for idx, row in df.iterrows(): + location = self.normalize(row.get("Location")) + mh_no = self.normalize(row.get("MH_NO")) + + if not location or not mh_no: + continue + + # exists = ManholeDomesticChamber.query.filter_by( + # subcontractor_id=subcontractor_id, + # RA_Bill_No=RA_Bill_No, + # Location=location, + # MH_NO=mh_no, + # ).first() + + # if exists: + # errors.append( + # f"Model-MH & DC (Row {idx+1}): Duplicate → Location={location}, MH_NO={mh_no}" + # ) + # continue + + record_data = {} + for col in df.columns: + if hasattr(Laying, col): + val = row[col] + if pd.isna(val) or str(val).strip() in ["", "-", "—", "nan"]: + val = None + record_data[col] = val + + record = Laying( + subcontractor_id=subcontractor_id, + RA_Bill_No=RA_Bill_No, + **record_data, + ) + db.session.add(record) + + if errors: + raise Exception(" | ".join(errors)) + + db.session.commit() + + + # ---------------- CLIENT FILE UPLOAD ---------------- def handle_client_file_upload(self, file, RA_Bill_No): diff --git a/app/templates/base.html b/app/templates/base.html index 3ea90da..affe12d 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -40,10 +40,10 @@ - +
  • - Subcontractor + Subcontractor Model
    • @@ -59,10 +59,10 @@
    • - +
  • - File System + Subcontractor File System