Client-Billing-software/app/Controllers/upload_controller.py

import os
import pandas as pd
from flask import request, redirect, url_for, current_app
from flask_login import current_user
from app import db
from app.models import Task, WorkDetail
from app.service.logger import log_activity


def to_2_decimal(value):
    try:
        if value is None or value == "":
            return None
        return round(float(str(value).replace(",", "")), 2)
    except (TypeError, ValueError):
        return None


def upload_controller():
    if 'file' not in request.files:
        return "No file part"

    file = request.files['file']

    if file.filename == '':
        return "No selected file"

    filepath = os.path.join(current_app.config['UPLOAD_FOLDER'], file.filename)
    file.save(filepath)

    log_activity(current_user.username, "File Upload", f"Uploaded file: {file.filename}")

    # =========================
    # READ WORK DETAILS (TOP PART)
    # =========================
    work_details_data = pd.read_excel(filepath, nrows=11, header=None, dtype=str)

    work_details_dict = {
        "name_of_work": work_details_data.iloc[0, 1],
        "cover_agreement_no": work_details_data.iloc[1, 1],
        "name_of_contractor": work_details_data.iloc[2, 1],
        "name_of_tpi_agency": work_details_data.iloc[3, 1],
        "name_of_division": work_details_data.iloc[4, 1],
        "name_of_village": work_details_data.iloc[5, 1],
        "block": work_details_data.iloc[6, 1],
        "scheme_id": work_details_data.iloc[7, 1],
        "date_of_billing": work_details_data.iloc[8, 1],
        "measurement_book": work_details_data.iloc[9, 1],
        "district": work_details_data.iloc[10, 1]
    }

    work_details_dict = {k: (None if pd.isna(v) else v) for k, v in work_details_dict.items()}

    # =========================
    # CHECK EXISTING WORKDETAIL (FOR OVERWRITE)
    # =========================
    existing_work = WorkDetail.query.filter_by(
        scheme_id=work_details_dict["scheme_id"],
        date_of_billing=work_details_dict["date_of_billing"],
        name_of_village=work_details_dict["name_of_village"]
    ).first()

    if existing_work:
        # 🔥 DELETE OLD TASKS
        Task.query.filter_by(work_detail_id=existing_work.id).delete()

        # UPDATE WORK DETAIL
        for key, value in work_details_dict.items():
            setattr(existing_work, key, value)

        work_detail = existing_work

        log_activity(current_user.username, "Overwrite", "Old data deleted and replaced")

    else:
        # CREATE NEW WORK DETAIL
        work_detail = WorkDetail(**work_details_dict)
        db.session.add(work_detail)

    db.session.flush()  # 🔥 get work_detail.id

    # =========================
    # READ MAIN DATA
    # =========================
    data = pd.read_excel(filepath, skiprows=10)
    data = data.astype(object).where(pd.notna(data), None)

    expected_columns = [
        "serial_number", "task_name", "unit", "qty", "rate", "boq_amount",
        "previous_billed_qty", "previous_billing_amount",
        "in_this_ra_bill_qty", "in_this_ra_billing_amount",
        "cumulative_billed_qty", "cumulative_billed_amount",
        "variation_qty", "variation_amount", "remark"
    ]

    # Validate excel columns
    if data.shape[1] < len(expected_columns):

        missing_cols = expected_columns[data.shape[1]:]

        return (
            "Excel is missing required columns: "
            + ", ".join(missing_cols),
            400
        )

    elif data.shape[1] > len(expected_columns):

        return (
            "Invalid Excel format. Extra unexpected columns found.",
            400
        )

    else:
        data.columns = expected_columns
    # =========================
    # INSERT DATA (FRESH)
    # =========================
    tasks_to_add = []

    current_main_task_serial = None
    current_main_task_name = None

    for index, row in data.iterrows():
        task_name = str(row["task_name"]) if row["task_name"] else ""
        serial_number = str(row["serial_number"]) if row["serial_number"] else None

        if serial_number:
            current_main_task_serial = serial_number
            current_main_task_name = task_name
            parent_id = None    
        else:
            parent_id = current_main_task_serial

        task = Task(
            work_detail_id=work_detail.id,
            district=work_details_dict.get("district"),
            block_name=work_details_dict["block"],
            village_name=work_details_dict["name_of_village"],
            serial_number=serial_number,
            task_name=task_name,
            unit=row["unit"],
            qty=to_2_decimal(row["qty"]),
            rate=to_2_decimal(row["rate"]),
            boq_amount=to_2_decimal(row["boq_amount"]),
            previous_billed_qty=to_2_decimal(row["previous_billed_qty"]),
            previous_billing_amount=to_2_decimal(row["previous_billing_amount"]),
            in_this_ra_bill_qty=to_2_decimal(row["in_this_ra_bill_qty"]),
            in_this_ra_billing_amount=to_2_decimal(row["in_this_ra_billing_amount"]),
            cumulative_billed_qty=to_2_decimal(row["cumulative_billed_qty"]),
            cumulative_billed_amount=to_2_decimal(row["cumulative_billed_amount"]),
            variation_qty=to_2_decimal(row["variation_qty"]),
            variation_amount=to_2_decimal(row["variation_amount"]),
            parent_id=parent_id,
            parent_task_name=current_main_task_name if not serial_number else None,
            # remark=row["remark"],
            remark=None if pd.isna(row["remark"]) else str(row["remark"]).strip(),
            row_index=index   # 🔥 optional but useful
        )

        tasks_to_add.append(task)

    db.session.bulk_save_objects(tasks_to_add)  # 🔥 FAST INSERT
    db.session.commit()

    log_activity(
        current_user.username,
        "Database Insert",
        f"Inserted {len(tasks_to_add)} rows from {file.filename}"
    )

    return redirect(url_for('main.display_tasks'))