Files
intelligence_system/utils/__pycache__/mysql_agent.cpython-312.pyc
T

181 lines
33 KiB
Plaintext
Raw Normal View History

Ë
3òh)tãó~ddlZddlZddlZddlZddlZddlZddlZddlm Z ddl
m Z ddl m
Z
mZmZmZmZmZmZddlZddlmZddlmZddlmZGdd «Zd
Zed k(r[ee««Zej=«r5ed «ejAd
«Z!ede!djDd«yed«yy)éN)Úcursors)Ú
MySQLError)ÚUnionÚListÚDictÚAnyÚOptionalÚTupleÚLiteral)Údatetime)ÚPath)ÚlogcóÒeZdZdZdZej «ZˆfdZde fdZ
de jjfdZd"dededzfd „Z d#d
ed eee dfd eeeefdej0fd
Z d$dedej0dedededef dZdedeefdZdedeeeeefffdZdej0dedeeeeeffdej0fdZdedej0deeeefdefdZ dej0deeeffdZ! d%dedej0deeeedfdefdZ" d&d
ed eee dfdedeeeeeefffdZ#dedefdZ$dedefd „Z%defd!„Z&ˆxZ'S)'Ú
MySQLAgentu
全平å°å…¼å®¹çš„MySQLæ•°æ®åº“æ“作类
支æŒWindows/macOS/Linux系统
é…ç½®å‚æ•°ä»Žå¤–部传入,ä¸ä½¿ç”¨è¿žæŽ¥æ± å’Œäº‹åŠ¡ç®¡ç
Ncóä|jsA|j5|jst|
|«|_ddd«|jS|jS#1swY|jSxYw©N)Ú _instanceÚ_lockÚsuperÚ__new__)ÚclsÚargsÚkwargsÚ __class__s €ú8d:\Idea Project\intelligence_system\utils\mysql_agent.pyrzMySQLAgent.__new__sTø€Ø}Š}ØØ—}’}Ü$)¡G¡O°CÓ$8C”M÷ð}‰}Јs}‰}Ð÷ð}‰}Ðús š!AÁA/Úconfigc
óüt|d«r
|jrygd¢}tˆfd|D««s&tjd«t d|«ddd d
d j
d d
«dj
dd«j
dd«j
dd«j
d«dœ |_tj«}tjd|d¬«|_y)u;åˆå§‹åŒ–MySQLæ•°æ®åº“连接(原有逻辑完全ä¿ç•™ï¼‰rÚhostÚportÚuserÚpasswordÚdatabasec3ó&K|]}|vŒ
y­wr©)Ú.0Úkeyrs €rÚ <genexpr>z&MySQLAgent.__init__.<locals>.<genexpr>+søèø€Ð:©M S3˜&”=©MùsƒuCæ•°æ®åº“é…置缺少必è¦å‚æ•°,当剿•°æ®åº“链接信æ¯ä¸ºï¼šu,æ•°æ®åº“é…置缺少必è¦å‚数,需è¦: rr r!r"r#ÚcharsetÚutf8mb4TÚconnect_timeouté
Ú read_timeoutéÚ
write_timeoutÚssl) rr r!r"r#r)Ú
autocommitr+r-r/r0z MySQLAgent(Ú))Úmodule)
ÚhasattrrÚallrÚwarningÚ
ValueErrorÚgetÚplatformÚsystemÚbind)ÚselfrÚ
required_keysÚcurrent_platforms ` rÚ__init__zMySQLAgent.__init__$ø€ä 4˜Ô " t§{¢{Ø òIˆ
ÜÓ:©MÓ K‰KÐ]Ð^dÐ]eÐ ÐKÈMÈ?Ð ˜6‘Nؘ6‘Nؘ6‘NؘzјzÑ—z‘z )¨YÓØ%Ÿz™zÐ*;¸"ŸJ™J ~°rÓ#ŸZ™Z¨¸Ó—::˜ 
ˆŒ ô$Ÿ?™?ÓÜ—88 [Ð1AÐ0BÀ!Ð#DÔEˆóÚreturnc ó tjdij¤Ž}t|d«s ˆfd}||_t j «dk(r-‰jjd«r|jd¬«jjd«|S#t$rÜ}t|«}t j «dk(r4d |vr0‰jjd
«j«cYd }~Sjjd |t!|«j"jjd
«jjd«jjd«d¬«d }~wwxYw)u3èŽ·å–æ•°æ®åº“连接(原有逻辑完全ä¿ç•™ï¼‰Úcharacter_set_namecó<jjdd«S)Nr)r*)rr8)r<s€rÚ_character_set_namez6MySQLAgent.get_connection.<locals>._character_set_nameHsø€ØŸ;™;Ÿ?™?¨9°iÓ@r@ÚDarwinr0T)Ú reconnectuèŽ·å–æ•°æ®åº“连接æˆåŠŸÚWindowsz timed outu%Windows连接超时,正在é‡è¯•...Nu 连接失败rr r#)ÚerrorÚ
error_typerr r#Úexc_infor%)ÚpymysqlÚconnectrr4rCr9r:r8ÚpingrÚtraceÚ ExceptionÚstrr6Ú_retry_connectionrIÚtypeÚ__name__)r<ÚconnrEÚeÚ error_msgs` rÚget_connectionzMySQLAgent.get_connectionAs.ø€ð Ü—?‘?Ñ2 d§k¡kÑ2ˆDô˜4Ð!5ÔAð+>ÔÓ  HÒ·±·±ÀÔ1GØ— ‘  D Ô H‰HN‰NÐ ˆKøäò
ܘA›ˆIÜÓ  IÒ-°+ÀÑ2JØ× Ñ Ð!HÔ× H‰HN‰N˜(Ü$(¨£G×$4Ñ$4Ø"Ÿk™kŸo™o¨fÓ"Ÿk™kŸo™o¨fÓ5Ø"&§+¡+§/¡/°*Ó"=Ø"&ð
ô

ûð
ús&ƒBB FÂ$AE;Ã4FÃ:BE;Å;FÚ max_retriescót|«D]G} tjdi|j¤Ž}|jj d|dzd«|cSy#t $r!||dz
k(rddl}|jd«YŒtwxYw)u=Windowså¹³å°è¿žæŽ¥é‡è¯•机制(原有逻辑完全ä¿ç•™ï¼‰uç»è¿‡ éu 次å°è¯•åŽæˆåŠŸå»ºç«‹è¿žæŽ¥rNr%) ÚrangerLrMrrÚinforPÚtimeÚsleep)r<rYÚattemptrUr^s rrRzMySQLAgent._retry_connectioncs|ä˜)ˆ
Üѯ©Ñ5Ø
 ¨°!©  }Ð4SÐ ñ *øô
ò
ؘk¨A™oÒÛØ
˜1
ð 
úsAAÁ'BÂBÚsqlÚparamsÚ parse_datesc
óp
|jjd|¬«|j«Š
ddlm}ddlm}|dˆ
fd|d|jjdd «i¬
«}tj||||¬ «}|r&|jjd t|«¬
«|dt«vr|j«SS#t$rC} |jj!d||t#| «t%| «j&d¬«d} ~ wwxYw#dt«vrj«wwxYw)u?执行SQL查询并返回DataFrame(原有逻辑完全ä¿ç•™ï¼‰u执行SQL查询©rar)Ú
create_engine)Ú
StaticPoolzmysql+pymysql://cóSrr%)rUs€rÚ<lambda>z(MySQLAgent.query_to_df.<locals>.<lambda>~sø€¡r@r)r*)ÚcreatorÚ poolclassÚ connect_args)rbrcu查询执行æˆåŠŸ©ô行数ÚengineuSQL查询失败T©rarbrIrJrKN)rÚdebugrXÚ
sqlalchemyrfÚsqlalchemy.poolrgrr8ÚpdÚread_sqlr]ÚlenÚlocalsÚdisposerPrIrQrSrT) r<rarbrcÚis_printrfrgroÚdfrVrUs @rÚ query_to_dfzMySQLAgent.query_to_dfpsø€ð! !Ø H‰HN‰NÐ,°#ˆ ×(ˆ
¯©¯©¸ÀIÓ)NЈFô˜S &°À[ÔQˆBÙØ
Ð2¼3¸r»7
Ôðœ6›8ÑÕ ð$øôò Ø H‰HN‰NÐ Ø &Ü" 1vÜ$(¨£G×$4Ñ$4Ø"&ð
ô
ûð ûðœ6›8ÑÕ ð$ús$ƒB&Cà DÃ>DÄDÄDÄD5Ú
table_namerzÚ
chunk_sizeÚreplaceÚignore_duplicatesc
ón
‡#—|| }|jr|jjd|¬«yd}d}d}d} d}
g} |j«}|j «}|jj d|«|j
d|d«|j«} | D
cgc]}
|
dŒ }}
|jj d|d |«|jdtjtjd
d d d
gd«j«}|jj«}|D
cgc] }
|
|vsŒ|
Œ }}
|D
cgc] }
|
|vsŒ|
Œ }}
|r+|jjd|d|t!|«¬«|sF|jjd|d« |r|j#«|r|j#«yy||j«}t!|«}|jj d|d|d«|jD]p}
||
j%d«j'«}|sŒ)|jj d|d|
d«||
j%d«|j(dd|
f<Œrdj+|jD
cgc]}
d|
dŒ
c}
«}dj+dgt!|j«z«}d|d|d|d}|jj-d|d|«|j/d«}|j0j«}t3t5||««D]q\}\Š#} t7ˆ#fd „|jD««}|j
||«|d!z
}|d!zd"zdk(r(|jj-d#|d$|d!zd%|d&«Œs|jK«|jjMd|d0||| |
t!| «¬1«| rA|jjId|d2| D"cgc]}"|"d3|"d4|"d5|"d6d7œŒc}"| ¬8«||r|j#«|r|j#«SScc}
wcc}
wcc}
wcc}
w#t8$rS}|j:dd'k(r¿| d!z
} ‰#j=«D cic]4\}} |t?| t@tBtDf«rtA| «dd"d(zn| “Œ6ncc} }w}!}} |jjd|d)||j:d!|!¬*«| jG|d+|j:d|j:d!‰#d,œ«|sx|
d!z
}
| jG|d-|j:d|j:d!‰#d,œ«|jjId|d.||j:d|j:d!‰#¬/«|sYd}~Œˆd}~wwxYwcc}"w#tN$r}|r|jQ«|jjId|d9tA|«tS|«jT||js t!|«ndd:¬;«| r+|jjId|d<| t!| «¬=«d}~wwxYw#|r|j#«|r|j#«wwxYw)>uÛ
兼容旧接å£çš„通用æ’入方法:ä¿ç•™replace傿•°ï¼ŒåŒæ—¶æ”¯æŒæ–°çš„ignore_duplicates
自动处ç†é‡å¤æ•°æ®ï¼Œå¯¹æ‰€æœ‰æ•°æ®æºé€šç”¨ï¼Œæ’入失败的数æ®ä¼šé€šè¿‡æ—¥å¿—记录
Nuå°è¯•æ’入空的DataFrame)Útableru(已建立连接,准备æ’入数æ®åˆ° zSHOW COLUMNS FROM `Ú`u表 u 包å«ä»¥ä¸‹åˆ—:ÚnanÚNaNÚNANÚu+ 中存在ä¸åŒ¹é…的列,已自动丢弃)Úunmatched_columnsÚcountu( 没有匹é…的列,终止æ’å…¥æ“作u 的过滤åŽDataFrame:共 u
行待æ’å…¥có6|t|ttf«SdS©NF)Ú
isinstanceÚdictÚlist©Úxs rriz+MySQLAgent.insert_from_df.<locals>.<lambda>×s¸Q¸]œj¨¬T´4¨LÓUÐPUÐUr@u 中的 u, 列包å«å¤æ‚类型,正在转æ¢ä¸ºJSONcó8|tj|d¬«S|S)NF)Ú ensure_ascii)ÚjsonÚdumpsrŽs rriz+MySQLAgent.insert_from_df.<locals>.<lambda>ÝsÀqÀ}¤$§*¡*¨Q¸UÔ"CÐ"[ÐZ[Ð"[r@ú, z%sz
INSERT INTO `z` (z
) VALUES (r2u为表 u 生æˆçš„æ’å…¥SQL:Úrecordsc3ó(K|] }|Œ y­wrr%)r&ÚcolÚrecords €rr(z,MySQLAgent.insert_from_df.<locals>.<genexpr>ìsøèø€Ð LÑ8K° ¨¥Ñ8Kùsƒr[édu
å·²å‘表 u æ’å…¥ Ú/u
行数æ®i&z...u 中跳过é‡å¤è®°å½•)ÚindexÚ
error_messager˜Ú duplicate)rrSÚ
error_coderœr˜rIu æ’入记录失败)r›u æ’入结果汇总)Útotal_to_insertÚtotal_insertedÚtotal_duplicatesÚ total_failedÚfailed_records_countu æ’入失败记录详情r›rS)rrS)Úfailed_records_summaryÚdetailed_failed_recordsu æ‰¹é‡æ’入失败T)rIrJr|Ú
total_recordsrKu" 事务回滚,已失败的记录)Úfailed_recordsÚ failed_count)+Úemptyrr6rXÚcursorrqÚexecuteÚfetchallr~ÚnprƒrtÚNAÚcopyÚcolumnsÚtolistrvÚcloseÚapplyÚanyÚlocÚjoinrOÚto_dictrÚ enumerateÚzipÚtuplerrÚitemsrrQrÚappendrIÚcommitr]rPÚrollbackrSrT)$r<r|rzr}r~rrUr Ú columns_infor—Ú
db_columnsÚ
cleaned_dfÚ
df_columnsÚmatched_columnsr‡Ú filtered_dfrŸÚhas_complex_typeÚ columns_strÚ placeholdersÚ
insert_sqlr•ÚindicesÚidxÚdatarVÚ short_recordÚrr˜s$ @rÚinsert_from_dfzMySQLAgent.insert_from_dfsø€ð Ð $Ø$+  Ð à
8Š8Ø H‰H× Ñ Ð:À*Ð Ô àˆØˆØˆØÐ؈ ؈ðj à×(ˆ—[[“]ˆ H‰HN‰NÐEÀjÀ\Ð 
N‰Nа ¸ !Ÿ?™?Ó,ˆLÙ,8Ó9©L S˜#˜a&¨Lˆ H‰HN‰N˜T * Ð-@ÀÀ Ð ŸØ”r—v‘vœrŸu™u e¨U°E¸ó÷‰d
ð $×+×4ˆJÙ.8ÓN©j s¸CÀ:Ò<Mšs¨jˆNÙ0:Ó T±
¨¸Ò>S¢°
Ð Ð Tá Ø× Ñ Ø˜:˜,Ð&QÐRØ&7ÜÐñ × Ñ  4¨
 |Ð3[Ð!\ÔñNØ ÙØ
ððO% 5×<ˆ! .ˆ H‰HN‰NØzlÐ"?ÀÐ?PÐP]Ð
ð
#×*Ø#.¨sÑ#3×#9Ñ#9Ù$ç‘#“%ð—H‘H—NN T¨*¨°X¸c¸UÐBnÐ#oÔpØ.9¸#Ñ.>×.DÑ.DÙ/K—OO¢A s Ÿ)™)¸;×;NÒ;NÓ$OÑ;N°C ¨¨Q¢ZÐ;NÑ$OÓPˆŸ9™9 d V¬c°+×2EÑ2EÓ.FÑ%FÓGˆ¨ °C¸ °}ÀJÈ|ÈnÐ\]Ð^ˆ H‰HN‰N˜W Z LÐ0FÀzÀlÐ "×)¨)Ó4ˆ!×'×0ˆGä$-¬c°'¸7Ó.CÖ$DÑ =F˜5 Ó L¸ ×8KÒ8KÓ LÓL—NN :¨tÔ" '˜A™ ‘}¨ÒŸŸØ¨ °H¸QÀ¹U¸GÀ1À_ÐDUÐU_Ðøð%Eðr
K‰KŒMð
H‰HM‰MØzlÐ"5Ð6Ø /Ø-Ø!1Ø)Ü%(¨Ó%8ð
ô
ñØØ˜:˜,Ð&?Ð$2ó
,ñ $2˜&' w¡ZØ$% f¡IØ*+¨L©/Ø-.¨Ñ-?ó ð
$2ñ
,ð-;ðô ð"ñ(Ø ÙØ
ðùòA:ùòOùÚ Tùò@%Pøô(+—vv˜ay (¨AÑ)/¯ © ¬ô(á(6¡  1ð¼ÀAÌÌTÔSWÐGXÔ9Y¤ A£   ¨uÒ 4Ð_`Ñ`Ù(6ùó(˜ ñ(ðŸ×" : ,Ð.DÐEØ"%Ø*+¯&©&°©)Ø#/ð ð'×-Ø%(Ø$/Ø*+¯&©&°©)Ø-.¯V©V°A©YØ&,ñ /ôñÑ)˜ à&×-Ø%(Ø$+Ø*+¯&©&°©)Ø-.¯V©V°A©YØ&,ñ /ôðŸŸØ" : ,Ð.AÐBØ"%Ø'(§v¡v¨a¡yØ*+¯&©&°©)Ø#)ð ñ!ÿùðW+"üò|,øôò ÙØ
Ø H‰HN‰N˜T * Ð-@Ð" 1vÜ$(¨£G×$4Ñ$4Ø$.Ø68·h²h¤s¨2¤wÀAØ"&ð
ô
ØØ˜:˜,Ð&HÐIØ#1Ü!$ ^Ó!4ðôð
ûð! ûñ$Ø ÙØ
ðús½¾A'W$Â% Q+Â1BW$Ä5 Q0Ä?Q0ÅW$Å Q5ÅQ5ÅAW$ÇA4W$ÉA#W$Ê&
Q:Ê3B%W$ÍA(Q?ÏA W$Ð!W
Ð;
W$Ñ+W$Ñ?
WÒ *WÒ39S-
Ó,C%W×W$×W×W$×$ Z
×-BZÚZ
Ú
Z
Ú
'Z4cóø |jd|jd|f«|j«}|r|dSdS#t$r4}|jj d|dt
|«¬«Yd}~yd}~wwxYw)uLã€æ–°å¢žè¾…助方法】获å–表的主键(用于replace逻辑的去é‡ï¼‰a/
SELECT COLUMN_NAME
FROM INFORMATION_SCHEMA.KEY_COLUMN_USAGE
WHERE TABLE_SCHEMA = %s
AND TABLE_NAME = %s
AND CONSTRAINT_NAME = 'PRIMARY'
r#rNu
获å–表 u 的主键失败)rI)rÚfetchonerPrr6rQ)r<r|ÚresultrVs rÚ_get_primary_keyzMySQLAgent._get_primary_keyXs„ð Ø N‰Nðð "&§¡¨ZÑ!8¸*Ð Eô

Gð—_&ˆFÙ &6˜! 0¨DÐ 0øÜò Ø H‰H× Ñ ˜z¨*¨Ð5EÐFÌcÐRSËfÐ Ô ûð ús7<º<¼ A9Á*A4Á4A9có\d}|jd|f} |j«} |j«}|j||«|j «}t |«}|s?|j jd|¬«i|j«|j«Si}|D]N} t| d«j«}
t| d«j«} | dr| dnd} | | d œ||
<ŒP|j jd
|t |j««¬ «||j«|j«S#j«|j«wxYw#t$r-}