
    `\jh                     b   d dl Z d dlZd dlZd dlZd dlmZ d dlmZmZ d dlm	Z	  e	e
                                          j        d         Zedz  Z e	d           e	d          gZed	z  Z ej        d
ej                  Zd Zd Zd Zd Zedk    r ej         e                       dS dS )    N)defaultdict)datetimetimezone)Path   zstory/spoken-narration.txtz/data/video-pipeline/Chanel-HuyenAnAudio/project/002-Co-Ke-Toan-Bi-Ep-Nhan-Toi-Bien-Thu-Quy-Cuu-Tro-Den-Khi-Muoi-Hai-Chu-Ky-Cung-Phan-Chu/story/v3/spoken-narration.txtz/data/video-pipeline/Chanel-HuyenAnAudio/project/003-Co-Chu-Tro-Bi-Ep-Ban-Nha-Bay-Chia-Khoa-Bong-Tro-Ve/story/spoken-narration.txtzstory/originality-report.jsonu   [\wÀ-ỹĐđ]+c                 J    d t                               |           D             S )Nc                 6    g | ]}|                                 S  )casefold.0xs     script/validate-originality.py
<listcomp>ztokens.<locals>.<listcomp>   s     @@@qajjll@@@    )TOKEN_REfindallts    r   tokensr      s$    @@H,<,<Q,?,?@@@@r   c                 r    t          j        |                                                                           S )N)hashlibsha256encode	hexdigestr   s    r   digestr      s%    W^AHHJJ//99;;;r   c                    t          t                    }t          |          D ] \  }}||                             |           !d}d }i }t          |           D ]X\  }}i }	|                    |g           D ]8}|                    |dz
  d          dz   }
|
|	|<   |
|k    r|
}||
z
  dz   |dz   f}9|	}Y||r)d                    | |d         |d                            ndfS )Nr   r     )r   list	enumerateappendgetjoin)abposjr   bestsampleprevicurns              r   longestr/   	   s    aLL))SQqQq))))tDaLL  SQq#771R== ' 'a	XXac!__Q1as1vff!TAaCE!A#;F
$$?SXXaq	&)+,---R??r   c                     g } t                                           rt                                           nd}|s|                     d           t	          |          }g }t
          D ]}|                                s&|                     dt          |          z              =|                                }t	          |          dfdt          t          dt                    z
  dz                       D             }g }t          t          dt          |          z
  dz                       D ]`}t          |||z                      }	|	|v r+|                    |d                    |	          d	           t          |          d
k    r na|rrt          |          nd\  }
}|                    t          |          t          |          ||
|d d         d           |r7|                     d|j        j        j         dt          |                      |
dk    r*|                     d|j        j        j         d|
            d t!          j        d|          D             }i }g }t%          |          D ]e\  }}t'          j        |                                                                          }||v r|                    ||         |d           `|||<   f|r%|                     dt          |                      d|  | sdnd|rt          |          nd |d|| t/          j        t2          j                                                  d	}t8                              t=          j        |dd          dz              tA          t=          j        |d         t          |          | dd                     |d         rdndS )Nr   candidate_missingzreference_missing:   c                 D    h | ]}t          ||z                      S r
   )tuple)r   r,   r&   r.   s     r   	<setcomp>zmain.<locals>.<setcomp>   s+    4g4g4gU1QqsU8__4g4g4gr   r   r   r   )indextext   )r   r   i  )pathr   shingle_matcheslongest_exact_token_runr*   zreference_shingle_matches::zlong_exact_token_run:c                     g | ]\}t          t          |                    d k    "t          j        dd|                                                                          ]S )#   z\s+r   )lenr   resubstripr   r   s     r   r   zmain.<locals>.<listcomp>"   sX    sss`cdjkldmdm`n`npr`r`rRVF3qwwyy113344`r`r`rr   z\n\s*\n)first	duplicatezinternal_duplicate_paragraphs:passedfailed)	versionverifiedstatussource_canon_sha256
referencesshingle_sizeinternal_duplicate_paragraphsissues
checked_atF   )ensure_asciiindent
rH   )rH   rK   rN   )rQ   )!	CANDIDATEexists	read_textr"   r   
REFERENCESstrrangemaxr?   r4   r$   r/   r   parentnamer@   splitr!   r   r   r   r   r   nowr   utc	isoformatREPORT
write_textjsondumpsprint)rN   	candidater%   rowsrefr7   shinglesmatchesr,   shrunr*   
paragraphsseendupsr   hreportr&   r.   s                     @@r   mainrr      s,   
i.>.>.@.@HY((***b94&-- 3444	)"T 	S 	SS	Kfmm$8S$ABBB8
}}$qa4g4g4g4g4geTWXYZ]^_Z`Z`abZbcdZdTeTeNfNf4g4g4g8prhoQs1vvaxz""##  aAa!eHoo2HnnW^^Qchhrll$C$CDDD	'llBuu010WQq\\\&*#fSXXW]^bWcWcv}  Y\  fl  mq  nq  mq  fr  >s  >s  2t  2t  2t`V]]_
8I8N__QTU\Q]Q]__```"WWV]]Q3:3D3IQQCQQRRRssbhzR[>\>\sssy{tx  BD  }Aj!!  SQqN188::((**!$YYt{{DG::;;;;tAwwDCD		CC	D	D	DF
F4XHHPX  EN  pXouv  pA  pA  pA  TX  fj  z|  ]a  kq  G  K  LT  LX  Y  Y  c  c  e  e  	f  	f  gm  gx  gx  y}  yC  DJ  X]  ef  yg  yg  yg  hl  yl  gm  gm  gm  ns  tx  t~  KQ  R\  K]  kn  os  kt  kt  ~D  E  E  SX  tY  tY  tY  nZ  nZ  nZ  gm  nx  gy  b@  bc  bc  @  [@r   __main__)r   rc   r@   syscollectionsr   r   r   pathlibr   __file__resolveparentsPROJECTrT   rW   ra   compileUNICODEr   r   r   r/   rr   __name__exitr
   r   r   <module>r      s                   # # # # # # & & & & & & & &      X  (+g>Z6ZIgkgk  mV  hW  hW  X\  X\  ]a  Xb  Xb  gc[e  kr  sR  kR  dj  \f  \^  \f  gy  z|  zD  \E  \E  S[ @ @ @ ; ; ;
@ 
@ 
@@ @ @* Z(((((r   