
    BZj                        d dl Z d dlZd dlZd dlZd dlZd dlmZ  eej        d                   Z e j	        e
                                          Zd Zd Zg Ze                    dg           D ]bZ ee                    dg           d          D ]?\  ZZe                    de                    d	           d
e  ee          d           @cg ZeD ]eZ ej        ded                   D ]KZ ee          Z ee                                          dk    re                    ed         ef           Lfd Z ee          Z ed eD                       Zi i dZeD ]Z eed                                                   ZeD ]zZ e  ee          ez
  dz             D ]]Zee         !                    d"                    eeeez                       e#                      $                    ed                    ^{d e%                                D             Z&g Z'eD ]tZ eed                                                   Ze'                    ed          e#d  e  e(d  ee          dz
                      D                       f           ug Z) ee'          D ]f\  Z\  Z*Z+e'edz   d         D ]P\  Z,Z-e+re-s
 ee+e-z             ee+e-z            z  Z.e.dk    r#e)                    e*e, e/e.d          d           Qg ee           ee          z    e0d e&1                                D                       z    ee)          z   Z2e2d k    rdnd ej3        e4                                          5                                 ee           ee          eee&e)e2d	Z6 e7 e j8        e6d                      e9e2rdnd           )    N)Path   c                     t          j        d|                                           } t          j        dd| t          j                  } d                    |                                           S )NNFCz[^\w\s] )flags)unicodedata	normalizelowerresubUNICODEjoinsplit)ss    f/data/video-pipeline/GacMaiAudio/project/gacmai_20260717_184527/artifacts/scripts/audit_duplication.pynormr      sZ    q!!''))BF:c!"*,U,U,U!]`]e]efgfmfmfofo]p]pVp    c                     t          | t                    r| n+|                     d          p|                     d          pdS )Ntext	narration )
isinstancestrget)xs    r   txtr      s;    z!C((W!!aeeFmm.Wquu[?Q?Q.WUWWr   parts
paragraphsPpart.locr   u   (?<=[.!?…])\s+r      r$   c                     i }| D ].\  }}|                     |g                               |           /d |                                D             S )Nc                 D    g | ]\  }}t          |          d k    ||dS )r   )r   	locationslen).0nlocss      r   
<listcomp>zgroups.<locals>.<listcomp>   s/    MMM4T1t	$	$r   )
setdefaultappenditems)r1   byr$   r,   s       r   groupsr3      sV    33USBMM!B''..s3333MM288::MMMMr   c                 H    g | ]}|d          t          |d                   f S r#   )r   )r+   qs     r   r.   r.      s+    +V+V+V1QuXd1V9oo,F+V+V+Vr   )      r   c                 l    i | ]1\  }}t          |          d  |                                D             2S )c                 ^    g | ]*\  }}t          |          d k    |t          |          d+S )r   )ngramr(   )r*   sorted)r+   gvs      r   r.   z<dictcomp>.<listcomp>   s6    RRR31QQPQaF1II..r   )r   r1   )r+   r,   tabs      r   
<dictcomp>r?      s>    oooW\WXY\SVVRR#))++RRRooor   c              #   T   K   | ]#}t          t          ||d z                      V  $dS )   N)tuplew)r+   is     r   	<genexpr>rE      s4      4e4eU1QqsU8__4e4e4e4e4e4er      g?)abjaccardc              #   4   K   | ]}t          |          V  d S )Nr)   )r+   r   s     r   rE   rE   %   s(      (F(FAQ(F(F(F(F(F(Fr   passedfailed)	statusstory_sha256r   	sentencesexact_sentence_duplicatesexact_paragraph_duplicatesrepeated_ngramsnear_duplicatesviolation_countF)ensure_ascii):jsonr   syshashlibr	   pathlibr   argvploads	read_textdr   r   parasr   r!   	enumeraterD   r   r0   sentsr5   r   r   r,   r*   r3   
exact_sent
exact_parangramsrC   ranger/   r   setaddr1   repsetsmaxnearlarG   lbrH   jroundsumvaluesviolsha256
read_bytes	hexdigestoutprintdumps
SystemExit r   r   <module>r{      s   & & & & & & & & & & & & & & & & & & & &      $sx{jdj//Aq q q W W WEE'" r rDIdhh|B//22qqSQq5<<Ga488TZK[K[GaGa^_GaGaililmnioio@p@p3q3q3q3qq	 2 2A
"(&qy
1
1 2 2QDGG!S^^Qu||QuXaL1112N N N 6%==
FF+V+VPU+V+V+V$W$W}}	 Z ZA4&	??
 Z ZQ5Q!YYaVAY11#((1QqsU82D2DSSUUKKOOPQRWPXYYYYYZoo`f`l`l`n`nooo	 h hA4&	??4;;%4e4eeeTWTWXYZ]Z]^_Z`Z`abZbTcTcNdNd4e4e4e1e1e'fgggg	$ ? ?HAfr!!A#$$Z ? ?TR	
!HC!HHSS1XX!WWT[[bR%%!**==>>>? 	S__SS__$SS(F(F(F(F(F%F%FFss4yyP''hhx~w~allnn?]?]?g?g?i?iwzwz  |A  xB  xB  OR  OR  SX  OY  OY  v@  ^h  {~  QU  hl  m  m  ns  ns  t~  tx  t~  B  PU  tV  tV  tV  nW  nW  nW  ^h  ^h  nr  iy  ij  ij  xy  ^z  ^z  Xzr   