
    ~Zj                     z    d dl Z d dlZd dlZd dlZd dlmZ d dlmZmZ d Z	d	dZ
d Zedk    r e             dS dS )
    N)Path)Counterdefaultdictc                     |                                  } t          j        dd|           } t          j        dd|           } t          j        dd|                                           S )Nz\b\d+(?:\.\d+)*\bz<n>u   [^\wÀ-ỹĐđ<> ]+ z\s+)lowerresubstrip)ss    j/data/video-pipeline/GacMaiAudio/project/gacmai_20260718_075153/artifacts/scripts/outline_semantic_lint.pynormr      s[    7799rv*533Qbf=STWXY6Z6ZAbdbhiopstubvbvb|b|b~b~[~       c           	          t          |                                           fdt          t          dt	                    z
  dz                       D             S )Nc                 P    g | ]"}d                      ||z                      #S )r   )join).0inws     r   
<listcomp>zngrams.<locals>.<listcomp>	   s1    PPP!388Aa!eH--PPPr   r      )r   splitrangemaxlen)r   r   r   s    `@r   ngramsr      sT    77==??PPPPPuS3q66!8A:=N=N7O7OPPPPr   c                  D   t          t          j        d                   } t          j        |                                           }d |d         D             }g }i }dD ]fd|D             }t          d |D                       }t          d |D                       }d |                                D             }d	 |                                D             }	t          |d
           d d         t          |	d           d d         d|<   |r|	                     d           |	r|	                     d           g }
|D ]}|
                    di           }t          j        |d                                          }t          |
                    dd                                                    d|v sd|v rCt          fddD                       r(|
	                    |
                    d                     |
r|	                    d           |
|d<   d}d                    d  |d!d          D                                                       fd"|D             }|r|	                    d#           |sd$nd%t          |           t!          j        |                                                                           t)          |          |||d&}t+          t          j        |dd'(                     t-          |sd)nd          )*Nr   c                 (    g | ]}|d          D ]}|S )beats )r   xbs      r   r   zmain.<locals>.<listcomp>   s-    7b7b7baWXY`Wa7b7bRS7b7b7b7br   parts)scene_actioncausal_resultc                 V    g | ]%}t          |                    d                     &S ) strget)r   r$   fields     r   r   zmain.<locals>.<listcomp>   s-    ,,,AEE%OO,,,r   c              3      K   | ]@}d                      t          |                                          dd                   V  AdS )r   N   )r   r   r   )r   vs     r   	<genexpr>zmain.<locals>.<genexpr>   sI      ;h;h^_CHHT!WW]]__UWVWUWEX<Y<Y;h;h;h;h;h;hr   c              3   X   K   | ]%}t          t          |                    D ]}|V  &d S N)setr   )r   r0   gs      r   r1   zmain.<locals>.<genexpr>   s        w`  w`}~  QT  U[  \]  U^  U^  Q_  Q_  w`  w`  LMwx  w`  w`  w`  w`  w`  w`  w`r   c                 ,    g | ]\  }}||d k    ||fS )   r"   r   kr0   s      r   r   zmain.<locals>.<listcomp>   s*    AAASQqaAAqDDAa5DDDr   c                 (    g | ]\  }}|d k    ||fS )   r"   r8   s      r   r   zmain.<locals>.<listcomp>   s2    QyQyQy\_\]^_tuwxtxtxSTUVRWtxtxtxr   c                     | d          S Nr   r"   r#   s    r   <lambda>zmain.<locals>.<lambda>   s    QRSTQUPU r   )key   c                     | d          S r=   r"   r>   s    r   r?   zmain.<locals>.<lambda>   s       XY  Z[  X\  W\ r      )repeated_six_word_startsrepeated_seven_gramsz:repeated_sentence_framez:repeated_phrase_frameconsent_privacyF)ensure_asciir&   r)   not_applicableu&   không thu, dùng lại, chuyển giaoc              3       K   | ]}|v V  	d S r3   r"   )r   r#   acts     r   r1   zmain.<locals>.<genexpr>   sK        VL  VLcdVW[^V^  VL  VL  VL  VL  VL  VLr   )u   ảnhu   biên bảnu   đồng thuậnu	   làm mờu   tọa độu   mãu   dữ liệuu   hồ sơbeat_idconsent_action_contradictionconsent_action_contradictions)u-   chứng minh quyết định đã bảo toànu%   chứng minh tuyến đã bảo toànu5   đã bảo toàn lối tiếp cận trong thực tếr   c              3      K   | ]N}t          |                    d d                    dz   t          |                    dd                    z   V  OdS )r&   r)   r   r'   Nr*   )r   r$   s     r   r1   zmain.<locals>.<genexpr>   sY      ffVWQUU>"--..s23quu_R7P7P3Q3QQffffffr   ic                     g | ]}|v |	S r"   r"   )r   r#   endings     r   r   zmain.<locals>.<listcomp>   sO      uW  uW  uW{|  KL  PV  KV  KVuv  KV  KV  KVr   preconstruction_overclaimpassedfailed)statuspathsha256r!   reportending_overclaim_hitserrors   )rG   indentr   )r   sysargvjsonloads	read_textr   itemssortedappendr,   dumpsr   r+   anyr   hashlibrV   
read_bytes	hexdigestr   print
SystemExit)pdr!   errrW   valsstartsgramsrepeated_startsrepeated_gramsconsentr$   cr   bannedhitsoutrJ   rP   r-   s                    @@@r   mainrx   
   s   tz!++--00Q7b7b1W:7b7b7bgicfqsjp. A AU,,,,e,,,$G;h;hcg;h;h;h4h4hVov  w`  w`  CG  w`  w`  w`  p`  p`inAAFLLNNAAA/QyQychcncncpcpQyQyQy.,2?~~,V,V,VWZXZWZ,[sy  {I  N\  N\  t]  t]  t]  ^a  _a  ^a  tb  c  c&-CSZZ5 B B BCCC@CJJ%???@@@	
 m mQEE
B!$*QE"B"B"B"H"H"J"JsSTSXSXYghjSkSkOlOlOrOrOtOt3!G1LLRU  VL  VL  VL  VL  iK  VL  VL  VL  SL  SLL  MT  M[  M[  \]  \a  \a  bk  \l  \l  Mm  Mm  Mm6CJJ5666)0'( 	Zff[`acadad[efffffllnn  uW  uW  uW  uW  AG  uW  uW  uWos0/	0	0	0"0xxAWXWcWcWeWeHfHfHpHpHrHr{~  @E  |F  |F  PV  os  }@  A  A  BG  HL  HR  SV  di  qr  Hs  Hs  Hs  Bt  Bt  Bt  {E  OR  FY  FG  FG  XY  {Z  {Z  uZr   __main__)r   )r^   r	   r\   rf   pathlibr   collectionsr   r   r   r   rx   __name__r"   r   r   <module>r}      s                          + + + + + + + +  Q Q Q QZ Z Z$ Zr   