From 51088975d37d2363de3200688344c2e949f63e39 Mon Sep 17 00:00:00 2001 From: pjpatil12 Date: Thu, 18 Dec 2025 18:29:25 +0530 Subject: [PATCH] uploading file changes as per sheet also add field RA_bill in model tr_ex, mh_ex. --- .gitignore | 15 +- .../manhole_excavation_model.cpython-313.pyc | Bin 4591 -> 4690 bytes .../mh_ex_client_model.cpython-313.pyc | Bin 5955 -> 5955 bytes .../tr_ex_client_model.cpython-313.pyc | Bin 6268 -> 6268 bytes .../trench_excavation_model.cpython-313.pyc | Bin 5391 -> 5490 bytes app/models/manhole_excavation_model.py | 5 +- app/models/tr_ex_client_model.py | 2 +- app/models/trench_excavation_model.py | 8 +- .../__pycache__/file_service.cpython-313.pyc | Bin 15158 -> 17364 bytes app/services/file_service.py | 356 ++++++++++++++---- app/templates/file_import.html | 2 +- 11 files changed, 308 insertions(+), 80 deletions(-) diff --git a/.gitignore b/.gitignore index 6500874..1b846b5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,11 @@ -# Ignore folders -instance/ +# Ingnor upload files app/static/uploads/ -# Ignore files -.env -# Ignore by type -*.log +# Ignore files +venv + +# Ignore Log files +logs + +# Ignore db folders +instance diff --git a/app/models/__pycache__/manhole_excavation_model.cpython-313.pyc b/app/models/__pycache__/manhole_excavation_model.cpython-313.pyc index 7166b848a8ad04d43826d28534f498b5e803e7ae..626e5f4588385352a1e3859ba35e821b81768708 100644 GIT binary patch delta 303 zcmaE_d`X4(GcPX}0}v=bbI#P9$UBK~$HoN**e5$M$xQCxkeQsop)vUYgl06De2_z0 zJPD*46~q_?rA@pj%+F`ZRAds&FU3$~8Z1y`wpo<3kCD-S@)9mJM)S?5xn47gN_iv} zrNjs2CuheS#+$~MALU;bnZ zz8J=To2T+cGffr~RGq9LC=A3Rwmf|Oo?V_3lBQ=&%(%=W-{5+KPoO`uD{}_Ze9l>% z3zTQ`UFK77aKFJX*k9RIIYVf^e!*YyYbgAs^}rGdl;W=2NF dy9`2i8Dzh(*f5#_jp3Nd@s$C{ERq1q0|2xrSS$bl delta 248 zcmcbl@?M$uGcPX}0}wr9%!WZJ6E6z$^I0+#83*%AF%+2u3ly1dj^ylPWVD<7f=i9jY%@RiYbFU_MryiACAPw^&2+OA>P?xAMg>zS#VTFPdqx zxS-Hve?cpCK7szsuFM%s^Eqd6E>ND$cbQMU!TkmgU%zLU=Y*u`851)u^T;>2PCh0W tuj>c&0V5C>O9P1y%#4hTcNv84GRS^mv1T*_8o@D><0}J@StJ3J2LSi8N_+qS diff --git a/app/models/__pycache__/mh_ex_client_model.cpython-313.pyc b/app/models/__pycache__/mh_ex_client_model.cpython-313.pyc index f3f6da39a18e867bbc64c9f59a0f5baef8642803..c1dd54458906f55162269d55d94d3218f08aedd7 100644 GIT binary patch delta 20 acmX@CcUX`6GcPX}0}yOl=e&{ILL2}=?*+2} delta 20 acmX@CcUX`6GcPX}0}$jbciPBpAr1gPCO00000?z2O&3k?jDKmr_-Knxs{fB_bhE({X0cntgo4+T_j zbYX1T2U2BiVRCD8vx5^v1Sbp*k4cJ2kYbl;k!aHlAAm*J4iJxNifNbvp9z`?pd*_O M(+($qN0aUsjx)d{fdBvi delta 102 zcmexk@W+7rGcPX}0}xa$ciPC!%QHEEM`N-HkIZBTCYj9@Jb&2u*@IGZ6N|EoZ?T5t zmn7zFZWeZ7QRfrr&+N*a!8D(97Uu%x*?gDz)EnGy@bL9}c6m-nnw~K+<1&wYgX`pX G;(Y*#8Y2M! diff --git a/app/models/__pycache__/trench_excavation_model.cpython-313.pyc b/app/models/__pycache__/trench_excavation_model.cpython-313.pyc index ffaab80461ccaa8daa2ccfc0730d0c0c44d986fe..69eb016cd76b3707ebfd36db6e02a14f838a6858 100644 GIT binary patch delta 295 zcmeCz`lQACnU|M~0SFwQIcF}O$UBK~!^QT3`KUq0!8+ldAU0n8NDY@<56RD*u0mgfLT<^ zBe5tYJ}5sqJKiwfG`=K1-ZrtTUpYvj?T-CKhEE-(n5PFG$v43-%3zTQ`UFK77aKFLB*YDZoIU#9!#>9-vJn{{$4+Mp$OHPuUVKm=*ru7Dn?R*>g zE(+Q=cztD={7=MHKOE?2Mj$Sh1ri^a85tSxG6>yekp05q#>i+op=f%=#EP#BAa;=i GPz3<^l~6VS diff --git a/app/models/manhole_excavation_model.py b/app/models/manhole_excavation_model.py index f096b15..996fd10 100644 --- a/app/models/manhole_excavation_model.py +++ b/app/models/manhole_excavation_model.py @@ -49,13 +49,14 @@ class ManholeExcavation(db.Model): Soft_Rock_1_5_and_above_total = db.Column(db.Float) Hard_Rock_0_to_1_5_total = db.Column(db.Float) - Hard_Rock_1_5_and_above_total = db.Column(db.Float) + Hard_Rock_1_5_to_3_0_total = db.Column(db.Float) Hard_Rock_3_0_to_4_5_total = db.Column(db.Float) Hard_Rock_4_5_to_6_0_total = db.Column(db.Float) Hard_Rock_6_0_to_7_5_total = db.Column(db.Float) - Remarks = db.Column(db.String(500)) Total = db.Column(db.Float) + Remarks = db.Column(db.String(500)) + RA_Bill_No=db.Column(db.String(500)) created_at = db.Column(db.DateTime, default=datetime.today) diff --git a/app/models/tr_ex_client_model.py b/app/models/tr_ex_client_model.py index ae79d7f..bbddd0b 100644 --- a/app/models/tr_ex_client_model.py +++ b/app/models/tr_ex_client_model.py @@ -75,8 +75,8 @@ class TrenchExcavationClient(db.Model): Hard_Rock_4_5_to_6_0_total = db.Column(db.Float) Hard_Rock_6_0_to_7_5_total = db.Column(db.Float) - Remarks = db.Column(db.String(500)) Total = db.Column(db.Float) + Remarks = db.Column(db.String(500)) created_at = db.Column(db.DateTime, default=datetime.today) diff --git a/app/models/trench_excavation_model.py b/app/models/trench_excavation_model.py index 1c98d57..6e1cae4 100644 --- a/app/models/trench_excavation_model.py +++ b/app/models/trench_excavation_model.py @@ -60,15 +60,17 @@ class TrenchExcavation(db.Model): Soft_Rock_1_5_and_above_total = db.Column(db.Float) Hard_Rock_0_to_1_5_total = db.Column(db.Float) - Hard_Rock_1_5_and_above_total = db.Column(db.Float) + Hard_Rock_1_5_to_3_0_total = db.Column(db.Float) Hard_Rock_3_0_to_4_5_total = db.Column(db.Float) Hard_Rock_4_5_to_6_0_total = db.Column(db.Float) Hard_Rock_6_0_to_7_5_total = db.Column(db.Float) - Remarks = db.Column(db.String(500)) Total = db.Column(db.Float) - + Remarks = db.Column(db.String(500)) + RA_Bill_No=db.Column(db.String(500)) + created_at = db.Column(db.DateTime, default=datetime.today) + def __repr__(self): return f"" diff --git a/app/services/__pycache__/file_service.cpython-313.pyc b/app/services/__pycache__/file_service.cpython-313.pyc index 24ad08cf1d118de2a576ce9210eb0151d438c9c0..676152a93bb708b5b2080cb2a710ab13e9c5ecf4 100644 GIT binary patch literal 17364 zcmeHPYfM|&oj=zv{KjB|!MrYyWPk)G5Hd-4gaF|Y2qeSB69y7oV;gW{Yar`%w@fnA?WAA!f3AIP z<2VVsGuoB99O3^x_q^Z#-}#+$d92f^5t!OPc6wSy5&AdsA{trT@;C^Vw-JLFsvqIl zlfn{sm-I{fWLQQMH)+4TPk|MJUDlt~r^HIZF7H?Msj*tHEBZBkTC5f9S^c^`J=P0$ zWq)>`0UHFnsz0YM7v~Cgb-%ICgiV57)1TLukMn6%fGQB9wID`!SQ0BCHcwGC`cKHq zU=77*vu@`ccH1XCUboLN;|3>1r{6c}nF5={H38BvjNLxxbk|j<F-#N>iidHQmrsj2mC}INhJ*Lb}d)!wPwkVIYOy<8Ok1VL;iaX0WwV z)gXy+G$KSJ8#N@BE#@pphGHaQH!%`9xDAoC(#+xb`bpV*>tBq zRM{G->1$d8d=6=ncF{Jb z6>UR$(UyEN3Y$ca4kZScPnMDc)HB^wAe)TIVwA@uP!^-;`}|t@PH9M25P5IwW-H=QNMqpX8OrmWB+oXr%V;`0qG*mA7B%dSS zObs)YQ;xbf&P_P|z5sSO1Ac7fg&;YDewxhvco?_KN+Q;t@?Ca#J+7En z;M%OaoNkY;6isv|^Xh(jnCI?z#67z zr`_&=ohUV4G3|DMa+p^P^wJPyzF6Jb+Dh95#o>0*WC5ZBn08;K!F|3I#Iuchz63ne zBF{{d$g~i>`q^=}`_xE==Zl2AV=4?v!p=FJZkC;#^LnpYyXTFtoRW8sZ;GCDKq*~K zbWJWcLk)2u5u~oX#Lx1wSw~=+mtFLGe6fXHXFGfPO#4ZD_nFftyM}n>jN_8q<-sf_ zqXL@GaoNpFW?j4*L(|yZSDkJzFQ3I8Ux1g9*5V6iV|m*Hg8C;x;`*Prh+Z=>6p-qt z0!ZMV6<~rIlljWaNKf;IZR5c1ft-ih?HwMk*FNay6*28C)GVfVg!(95laM_m&lB6s zG_g@`mcTG*?6|>XE#Pyf9X=PR8IoBX67Z#@apSt$`m{fZ$DW1x{s!92%U@9cM|N4% zc=V47WH2oTmS10beN7TJ*G7$X(QIQRn+|2uLDPFBw@M<}2SV8gqQ>qmW$}ixbSuZW zY+f=i7cUjRI~gf$2$eRhySV0luIV(_I1tVmTu?kvs^8RnTeH|59R5gIyJf5hHon(- zt2Jz_Ur;_U<}DvtIWfSP1SYPV)cfy=%K7Z)%ckb6jeadkrD;;6m?b3Z%>Ba_cv=4G*+Ai`C0@OLaF6Y-lUsjmtf>p>5nE zmS_04Y-n3IwT6hcHl(dxJ9>A1Lpu~D@B2d9eQUkK`@|UxN_=P*PDxz>1u7mn2ufq$}uY-Vtst7XM{*-exLe<*g++xmcVj0gvL~2ZnlJfCaflw zozW8<={gEAUxaJaPvr-ABqhIu|BLf;z}U6Pnd~|kSw8u|Edk|bm^DV!dmch)k|=1b zI8i^1I42YJhZF`Z2#x#|>0zJtg*DQ?phgM*=hny|j4j9;aEW&0o=q4#1LFv35Udsn zT*Ty95TLAbvHZ!W*cZm9Rs^wT4w%=;_UfSqnxN$qXpku!sbq4k5}WcQ^)lttCD$Ur zO}e3qWKN=uF@bxP_?CPgj<1JM?!BVxkXQ)O0iHwqQeNU#TVm|e1i8ohRt9BVjJL$| zQ>=SBMX_CI1S?IRO{PaozD*7|P}V4^LVy&lVjMt<2cXCGFDP}r7(V$-$-~SP*!0^_ zroLS-piFqCpiIl)yeWwqX#XTVi~-O!JuwnUHq?HO?(J#^xWf=2COB?KT;M1l8$W;L z0#Dh8Z&SP+yQkb&pGfWYeY_O5p5P>%_Fv(Zr~OWW>cC`X@bZCP``{T&dR;)dw`GEz zr|Lc@TI&yQe2({wmXJzR4r3(pZn@8KF%Lag*74+dkUplTf9&SFMHgY zh`9_!Vacxg6Z8h+^4mVDJsx~z<>cGf!1BdoG6276$fa&eFzGGa0OE6kkr>Bi0zilS zSLi={>n&Pro2DmOw9qh3^9N)64s?Zx5orQ2)9@^P$_FGsfPmO1n*B6$$um1kUb^SY zGRIF~UOWKBs42u`?DJVpzjtoN$MTAR-wW8ES2$R*S>UB?0P}JXo>@K%14H3(0`zxb z|E$l!%O?RDdU>TM0Mv*73d>8U+yPue5|nvZ2#_`HU>$(~#$@x2JBWt@wj*N5H`=Nk0-m&C3A+`~18V8i1DzWdlw}2#JqC z?krKxA}US=$C%7;e$S4E%_K=^8HW8EXiKT%^i{#p4dp?>TDhUDdZ5%TOuTvV`o)O8 zETk_B>&rKk72wF}iSI)t(q+HdcfBv7EeUB$-gSnx6~EURt~1MdOL;fUF)Lu@m4e`* zHP@n<%Q=*E%L_`^8j0J%M3hDLhUK9KsWjhFKg>mX!<(;Ne{JRP!fTwad_nS1fwcK6 zxvQ3&mawKWT2dBN-yDyW)Q3vy!zG6nIu_=y4{qk07c|j=!Uf$!S)J+x6)mM#FWtPf zI06%nD|O3#u38Tc-Twp0|2ddLyPiA=928 z8`nmDR1&Fa3DvZOYmVQo2-lqaK=NVMud}$bwh#5c(r=7j;KrTdQCDR2VrcYYc+|_y z;_xVrn#zN(giL!E2YzcRhGG|&t~R~fwAlH;T)I(qVBHuo9}b%jN9oF2$`v`lcOI!D8GqtUABm9EWw2ZNnZ&QEht@!rq!k;xp& zZ(G|Jsci|>wrtcM59hZJK^AJfbxzRm`Kb69BGr#;(IXo8?pQ%w! zK9_%Reecf>{N%vhWA`RM^#01bVHn#olfC~z}6tAi+U)Pn)FZro&HadpP}$El0dgS z(CTBOrYxi_+t8K^TZ`s9?RO4wmNw4Vc3<1}=ZA6<^2uYO%YKp7Y3$EI@0WHqz~eVL z>eFiJZ;Gl;%cZ|*qrm=wTn=(~ivdj~7*W;kc*C%}#Yh0c-7N+gLbly4#_kqFSWEuj zZ7~REJko^8Rui`n(MrT|5W~2QSlfvpfDv~RA)rOFZ-_z0#I6DNfZLbfBjD2{X#$Ax zAc)jW0)7P?_+y!H-qTENhW)`7NiVTKuylT8?O4kR%D#ULEMH)TWI0TPjfhu4%-6@y zXm^hM@i~asJx@O)Vn)c1fhR%uH6q~L9pNz|fSo}20uiv^N7zop*NAWsL4ZA;Ai_xm zA?#vZ(j%$*@~XEJ-ffZi&x0;(;|kJcOmSY4XGY1sp? zx-@^W8(?mpg|jrSPyWpR6aU?7+|Wp5XgoADzG1SH4MyKm-}2znVDRM)Q*G2#7OW4M zs&4cXWL6g;llKiL39^QePw-_JqnQuPbAcNG{vzXX zxDmZ7WhDn-Ap&*-C_bXzz4E>O-gu2l&sU%wsegkZW0S zdOa_eVyYC&1U>qLZRC2!-iTAhsxfjdDVQ-r8wm!e@OV2UN)GP=ReVjTr+N%%x|$^E zJp^Mk-DIMaZHc1>KuU5(9VgoVDNe;ENc4>9Ri@{VJgNdJR>Noo5*7~Qt6;v-@bgJ0 z2_0J{#*5Rj+830YR6fm!eGd7^8J$hLjgHl}YXv$Mo+)%}_Fzntc0f`w>_fm#_mlly zK(XN_nt_X8G$GsQXHFh-w| z%LT*p{UT}*ub9#CJ7zT8Sp(v4&PjMOn$Tt$3CLl8z~L22)I`r8+F4@2e!A)I_~Cnu zh&06jNfL$#;6s2R6Qt|}r0{Z~x_E{-2+-wg#3rB-C9w$zg-n5G!jIcJfkVU>NgzVW z@|2U;%y@+R0Cr#n@g?FT7sGkSet2;$=X=hT@*9)O-X-rp_-h$$J5L2P!>{^4rWWSn0c28Y$c#D%>9~w60gIkN(6OY3KgR33hkl%uK^F3ZGmHa_!H=*-g`xnQ=_3%qmfgN&?!gwl#}aq zaTYgcbaPs_KpZ}l6{&_r&hQ_ToFNo!8-;gxosF2A!{+8Vg|}5y@*D!s5w3G=C-AZf zftQOc2S1H7bD+!9nK=RfJw?I2rTO$F!obz06L3+ZnJb#)rZ6|ja)klT7~r&lFCO(% zD8K1XpAAxR>TNhP^>&6k`^r5y)@L{)b6?8{+}kj4Z?!Ov-mL6V%6`7@SdR|orF$2+!ZFS`#%agCc+?ZqzI~$hZO(Y^_44}H*PWcDi8D6c*ER{dxA_VF zI~X#%3Kbe5&Fm`F#Lc(e&H1*mvAa3%g7!!`m=HGS&rzt2G-yZoEQ=nPCb#5ws8Wlz zNBYWjDj|iwtWGVAYfEvTn{tREg4nokvr5nY-^i2WAlA&v!GgAc2%pe z58dUxVVx1d#4hjs0{*yso2C_BcK52!QLP6+rNZ|~^hxIqwQ6x%gs)t!lKK8+)oNLq zzDy^C`F$C7tfCH14_pK1LhMI>VJf!uX%(AJ9yi}%BIfq6x&2F$$1i`yDz@)l^SwH* zaD+3CaN3clHEe4}4SV!X&K)UdY2l15_q8p8hHZ^$SmEeoutvu#?ZQWyb~~@O+h_c) zIrw1D&TH-Vug^KWXEXSk~cm& zFM-Di>d`29c2SRpv-Zh{sYer3wY>S!2(?e%^7%N()>xV(xtan@4)1uyEX;EUD@-k3 z58v5ens?7lS?2;CZ~WU!HGCUyg->GPe=Nj4)s%um>}Gid@q#9Cds*vK7y3zCxxK>m zbiOhrpg5tDpf(;7j*<~ylObgK)Qg8F$5MPrN_gSZOS?G9h1?0lwJnVBQRenA1hJ=w zaXGh#ACP}J{`>UYTa sF{=F-m3)kHKSpgIqxO%{>3>Hj1WWhl3Pip9pNb9@)%pk#DRkX`0Sli49RL6T delta 2559 zcmb`JOH3O_7{~YFU9Vwz_l|xdcmz-MFG_+KsNKLCqy|u{6A!*dAGY|p|ks2!I@bk>~ zee>-%|8K|I#rNqir!pUAWTa7eeDPaX@B`=d%$uw=&QjDTl%Lr}?06_NMUBz$0b8EM zI0q?8CTrGD^|Mf}HAB0un%2SNI@QSenM|sYX=YaUg<1HGZl{s;@jiV!BT*;Kt9H1| z=0TTEV8S{magFIs&o^*zXN|SN&o$#Gkb(E)uFfdRC-`-1ff7|roy=SF^cT!F=j3ue zgD=gO-p+>k%`WkoOaFrFJm<@J<`OnMD)XfUWcm5^0yacvzckwD*Z+64W95 zoF4$S`=@IG1LyU&c70E`R3Yv2#Ea89IyxkOB&5lTsz}~oNR|44Iu%sUNZ1q4kVc7# zq>;v*@nTpn6dBXvnH$aZPIyds*UCjeFdRcnL_ujlm6c(2I-rK&??M~=ewPha7^=(o zQ7xpX0Fdq?fn<)zv2nyGqli^T>j@g*MMEEd8ovn_DEN!v^7#is#?0v}d5eN=S?4e` zJ>n^o<S!`==Df~?!|u>ok#Ix{$g!Xn zMywnP&C`e-*UrGwN>?MIYbNnsq<+0{8L?p*2lUFlfV?xngfS3r9RO-LFfQ$%4#-n0 zIpA_j<@u6LX2~Z{sghU5LMu*bo~p7SeLxFMPK2X~jm7}tFoF@pjpAN}5U&6&5|)uJ z7{%n}XzXl6MVvYjiJe7UFs!K4h>vNZU^Iq=U<@}>I}=6vaXBi-Vn7dgQexGtr6gF3 zXGj2AD0E5=ygi=*ti;twK+b{a07{<-Mx(*-*l=Yl-2q8(IJ}3&VNWT2D~-fodYAexe3-j zr0a5$qF*D}LqIO57wjcd@_z?92{!bmtw#=zYxv5%&>U&A{omaI9-OJl0``%F`w6@R z2MD?e4iX$9I7~oP0zCx11V;(<1XTnaK_9^}g8nQ*(iPw%7$Cao!EQW974U0|1>Pz) z@gL!3bO}~geaE<01T;9Vl`iGB+|6x)Bi2gTU2QTLsT~!MEL4FVOM0}-mmC;Q`bLuZ za?&g(MfpkU@T#1>YjWa5SkAmXH;&!#BnwmqI3~u^QFa)o&`H82t0$uql z=?u?SUJ7l4Py!L*r9u(MCY@@CPP?D!B$yO!LXF0Pf#JZoJaI||Ur~wDs-J0Q>&LlR z_r`V{$FPOt6v8n_C-VoDxa# forward fill Location + + + # new new + def process_trench_excavation(self, df, subcontractor_id,RA_Bill_No): + + print("RA_Bill_No of Tr Ex:",RA_Bill_No) + + print("=== RAW HEADERS ===") + print(df.columns.tolist()) + print("===================") + + # Clean column names + df.columns = ( + df.columns.astype(str) + .str.strip() + .str.replace(r"[^\w]", "_", regex=True) + .str.replace("__+", "_", regex=True) + .str.strip("_") + ) + + # Remove completely empty rows + df = df.dropna(how="all") + + # Forward fill merged Location if "Location" in df.columns: df["Location"] = df["Location"].ffill() - df = df.dropna(how="all") # REMOVE empty rows - # Identify missing location rows before insert - missing_loc = df[df["Location"].isna() | (df["Location"].astype(str).str.strip() == "")] - if not missing_loc.empty: - return False, f"Error: Some rows have empty Location. Rows: {missing_loc.index.tolist()}" - saved_count = 0 + skipped_count = 0 try: for index, row in df.iterrows(): record_data = {} - # Insert only fields that exist in model + location = row.get("Location") + mh_no = row.get("MH_NO") + + if (pd.isna(location) or str(location).strip() == "" or pd.isna(mh_no) or str(mh_no).strip() == ""): + skipped_count += 1 + continue + + # Map only model columns for col in df.columns: if hasattr(TrenchExcavation, col): value = row[col] + # Normalize empty values - if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan", "NaN"]: + if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan"]: value = None record_data[col] = value + + # If all mapped fields are None → skip + if all(v is None for v in record_data.values()): + skipped_count += 1 + continue record = TrenchExcavation( - subcontractor_id=subcontractor_id, + subcontractor_id=subcontractor_id, RA_Bill_No=RA_Bill_No, **record_data ) + print("Saving Row → Location:", record.Location, " MH_NO:", record.MH_NO) + db.session.add(record) saved_count += 1 db.session.commit() - return True, f"Trench Excavation data saved successfully. Total rows: {saved_count}" + + return True, ( + f"Trench Excavation saved successfully. " + f"Inserted: {saved_count}, Skipped: {skipped_count}" + ) except Exception as e: db.session.rollback() - return False, f"Trench Excavation Save Failed: {e}" - + return False, f"Trench Excavation save failed: {e}" + + # new new + def process_manhole_excavation(self, df, subcontractor_id, RA_Bill_No): + + print("RA_Bill_No of MH EX:",RA_Bill_No) + + print("=== RAW HEADERS ===") + print(df.columns.tolist()) + print("===================") + + # Clean column names + df.columns = ( + df.columns.astype(str) + .str.strip() + .str.replace(r"[^\w]", "_", regex=True) + .str.replace("__+", "_", regex=True) + .str.strip("_") + ) + + # Remove completely empty rows + df = df.dropna(how="all") + + # Forward fill merged Location + if "Location" in df.columns: + df["Location"] = df["Location"].ffill() + + saved_count = 0 + skipped_count = 0 + + try: + for index, row in df.iterrows(): + record_data = {} + location = row.get("Location") + mh_no = row.get("MH_NO") + + if (pd.isna(location) or str(location).strip() == "" or pd.isna(mh_no) or str(mh_no).strip() == ""): + skipped_count += 1 + continue + + # Map only model columns + for col in df.columns: + if hasattr(ManholeExcavation, col): + value = row[col] + + # Normalize empty values + if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan"]: + value = None + + record_data[col] = value + + # If all mapped fields are None → skip + if all(v is None for v in record_data.values()): + skipped_count += 1 + continue + + record = ManholeExcavation( + subcontractor_id=subcontractor_id, RA_Bill_No=RA_Bill_No, + **record_data + ) + + print("Saving Row → Location:", record.Location, " MH_NO:", record.MH_NO) + + db.session.add(record) + saved_count += 1 + + db.session.commit() + + return True, ( + f"Manhole Excavation saved successfully. " + f"Inserted: {saved_count}, Skipped: {skipped_count}" + ) + + except Exception as e: + db.session.rollback() + return False, f"Manhole Excavation save failed: {e}" + + + +# olds + # def handle_file_upload(self, file, subcontractor_id, file_type): + # # def handle_file_upload(self, file, subcontractor_id): + + # if not subcontractor_id: + # return False, "Please select subcontractor." + # # if not file_type: + # # return False, "Please select file type." + # if not file or file.filename == "": + # return False, "No file selected." + # if not self.allowed_file(file.filename): + # return False, "Invalid file type! Allowed: CSV, XLSX, XLS" + + # ensure_upload_folder() + + # folder = os.path.join(Config.UPLOAD_FOLDER, f"sub_{subcontractor_id}") + # os.makedirs(folder, exist_ok=True) + + # filename = secure_filename(file.filename) + # filepath = os.path.join(folder, filename) + # file.save(filepath) + + # try: + + # # df = pd.read_csv(filepath) if filename.endswith(".csv") else pd.read_excel(filepath) + # df = pd.read_excel(filepath, sheet_name ="Tr.Ex.", header=0) + # df1 = pd.read_excel(filepath, sheet_name="MH Ex.", header=0) + + # print("\n=== Uploaded File Preview ===") + # print(" file name h:",df) + # print("=============================\n") + # print(" file name h1:",df1) + # # print(df.head()) + # print("=============================\n") + + # # Trench Excavation save (subcontractor) + # if file_type == "trench_excavation": + # return self.process_trench_excavation(df, subcontractor_id) + + # # Manhole Excavation save (subcontractor) + # if file_type == "manhole_excavation": + # return self.process_manhole_excavation(df, subcontractor_id) + + # # Manhole and Domestic Chamber Construction save (subcontractor) + # if file_type == "manhole_domestic_chamber": + # return self.process_manhole_domestic_chamber(df, subcontractor_id) + + # # Tr Ex save (client) + # if file_type =="tr_ex_client": + # return self.client_trench_excavation(df, subcontractor_id) + + # # Mh Ex save (client) + # if file_type =="mh_ex_client": + # return self.client_manhole_excavation(df, subcontractor_id) + + # # Mh and Dc save (client) + # if file_type == "mh_dc_client": + # return self.client_manhole_domestic_chamber(df, subcontractor_id) + + # return True, "File uploaded successfully." + + # except Exception as e: + # return False, f"Processing failed: {e}" + + + # ---------------------- Sub contractor -------------------------- + # Trench Excavation save method (TrenchExcavation model) + # def process_trench_excavation(self, df, subcontractor_id): + # df.columns = [str(c).strip() for c in df.columns] + # # If the sheet has merged cells -> forward fill Location + # if "Location" in df.columns: + # df["Location"] = df["Location"].ffill() + + # df = df.dropna(how="all") # REMOVE empty rows + # # Identify missing location rows before insert + # missing_loc = df[df["Location"].isna() | (df["Location"].astype(str).str.strip() == "")] + # if not missing_loc.empty: + # return False, f"Error: Some rows have empty Location. Rows: {missing_loc.index.tolist()}" + + # saved_count = 0 + + # try: + # for index, row in df.iterrows(): + # record_data = {} + # # Insert only fields that exist in model + # for col in df.columns: + # if hasattr(TrenchExcavation, col): + # value = row[col] + # # Normalize empty values + # if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan", "NaN"]: + # value = None + + # record_data[col] = value + + # record = TrenchExcavation( + # subcontractor_id=subcontractor_id, + # **record_data + # ) + + # db.session.add(record) + # saved_count += 1 + + # db.session.commit() + # return True, f"Trench Excavation data saved successfully. Total rows: {saved_count}" + + # except Exception as e: + # db.session.rollback() + # return False, f"Trench Excavation Save Failed: {e}" + + # Manhole Excavation save method (ManholeExcavation model) - def process_manhole_excavation(self, df, subcontractor_id): - # Clean column names (strip whitespace) - df.columns = [str(c).strip() for c in df.columns] - # If the sheet has merged cells -> forward fill Location - if "Location" in df.columns: - df["Location"] = df["Location"].ffill() + # def process_manhole_excavation(self, df, subcontractor_id): + # # Clean column names (strip whitespace) + # df.columns = [str(c).strip() for c in df.columns] + # # If the sheet has merged cells -> forward fill Location + # if "Location" in df.columns: + # df["Location"] = df["Location"].ffill() - # REMOVE empty rows - df = df.dropna(how="all") - # Identify missing location rows before insert - missing_loc = df[df["Location"].isna() | (df["Location"].astype(str).str.strip() == "")] - if not missing_loc.empty: - return False, f"Error: Some rows have empty Location. Rows: {missing_loc.index.tolist()}" + # # REMOVE empty rows + # df = df.dropna(how="all") + # # Identify missing location rows before insert + # missing_loc = df[df["Location"].isna() | (df["Location"].astype(str).str.strip() == "")] + # if not missing_loc.empty: + # return False, f"Error: Some rows have empty Location. Rows: {missing_loc.index.tolist()}" - saved_count = 0 + # saved_count = 0 - try: - for index, row in df.iterrows(): - record_data = {} - # Insert only fields that exist in model - for col in df.columns: - if hasattr(ManholeExcavation, col): - value = row[col] + # try: + # for index, row in df.iterrows(): + # record_data = {} + # # Insert only fields that exist in model + # for col in df.columns: + # if hasattr(ManholeExcavation, col): + # value = row[col] - # Normalize empty values - if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan", "NaN"]: - value = None + # # Normalize empty values + # if pd.isna(value) or str(value).strip() in ["", "-", "—", "nan", "NaN"]: + # value = None - record_data[col] = value + # record_data[col] = value - record = ManholeExcavation( - subcontractor_id=subcontractor_id, - **record_data - ) + # record = ManholeExcavation( + # subcontractor_id=subcontractor_id, + # **record_data + # ) - db.session.add(record) - saved_count += 1 + # db.session.add(record) + # saved_count += 1 - db.session.commit() - return True, f"Manhole Excavation data saved successfully. Total rows: {saved_count}" + # db.session.commit() + # return True, f"Manhole Excavation data saved successfully. Total rows: {saved_count}" - except Exception as e: - db.session.rollback() - return False, f"Manhole Excavation Save Failed: {e}" + # except Exception as e: + # db.session.rollback() + # return False, f"Manhole Excavation Save Failed: {e}" # Manhole and Domestic Chamber Construction save method (ManholeDomesticChamber model) diff --git a/app/templates/file_import.html b/app/templates/file_import.html index 71556d0..aaa977c 100644 --- a/app/templates/file_import.html +++ b/app/templates/file_import.html @@ -36,4 +36,4 @@ -{% endblock %} +{% endblock %} \ No newline at end of file