
    Zj@                     x    d dl Z d dlZd dlZd dlZd dlmZ d dlmZmZ d Z	d Z
d Zedk    r e             dS dS )    N)Path)Counterdefaultdictc                     |                                  } t          j        dd|           } t          j        dd|                                           S )Nu   [^\wÀ-ỹĐđ ]+ z\s+)lowerresubstrip)ss    l/data/video-pipeline/GacMaiAudio/project/gacmai_20260718_075153/artifacts/scripts/story_duplication_check.pynormr      sA    7799rv*3q11Qs19M9M9S9S9U9U2U    c           	          t          |                                           fdt          t          dt	                    z
  dz                       D             S )Nc                 P    h | ]"}d                      ||z                      #S )r   )join).0inws     r   	<setcomp>zgrams.<locals>.<setcomp>	   s1    PPP!388Aa!eH--PPPr   r      )r   splitrangemaxlen)r   r   r   s    `@r   gramsr      sT    77==??PPPPPuS3q66!8A:=N=N7O7OPPPPr   c            
         t          t          j        d                   } t          j        |                                           }g }|                    dg           D ]}|                    d|                    d                    }t          |                    dg           d          D ]h\  }}|                    t          |          dd|dt          |t                    r|                    dd	          nt          |          f           ig }t          t                    }|D ]3\  }	}
t          |
d
          D ]}||                             |	           4|                                D ]M\  }}t#          t%          |                    }t'          |          dk    r|                    d||d           Nd |D             }t          |          D ]t\  }\  }}||dz   d          D ]^\  }}|r|s
t'          ||z            t'          ||z            z  }|dk    r)|                    d||gt)          |d          d           _u|sdndt          |           t+          j        |                                                                           t'          |          t'          |          |d d         d}t3          t          j        |dd                     t7          |sdnd          )Nr   partspart_numberpart
paragraphs02d.text       shared_12gram)typer%   r"   c                 8    g | ]\  }}|t          |d           fS )   )r   )r   pidr%   s      r   
<listcomp>zmain.<locals>.<listcomp>   s)    000xs4E$qMM000r   g
ףp=
?paragraph_similarity   )r*   r"   scorepassedfailedi  )status
story_pathstory_sha256r"   issue_countissuesF)ensure_asciiindentr   )r   sysargvjsonloads	read_textget	enumerateappendint
isinstancedictstrr   listr   itemssortedsetr   roundhashlibsha256
read_bytes	hexdigestprintdumps
SystemExit)pdrowsr!   pnr   qr8   idxr-   r%   gpidsusetsaxbyr1   outs                        r   mainrb   
   s   tz!++--00QbUU72 O OT	XXmDHHV,,--"txxR0033  O  OcaDKKCGGAYAYAYRSAYAYAYnxyz{  oA  oA  [MZ[Z_Z_`fgiZjZjZj  GJ  KL  GM  GM  AN  5O  5O  5O  5O  O
 - -XSb>>,,a#a&--,,,,,yy{{ O OVQt
3t99!VVQYYv}}_A1MMNNN004000$ k kWQu!!A#$$Z k kca
1X	QqS#ac((	5Tkk&--(>QqEZ_`efgZhZh i ijjjk  &3xx8QW^WefgfrfrftftWuWuWW  XB  XB  PS  TX  PY  PY  hk  lr  hs  hs  }C  DH  EH  DH  }I  J  J  KP  QU  Q[  \_  mr  z{  Q|  Q|  Q|  K}  K}  K}  DN  X^  Oe  OP  OP  de  Df  Df  ~fr   __main__)r=   r	   r;   rL   pathlibr   collectionsr   r   r   r   rb   __name__ r   r   <module>rh      s                          + + + + + + + +V V VQ Q Qf f f* Zr   