
    *^jT              	          d dl Z d dlZd dlZd dlZd dlmZ d dlmZmZ d dlm	Z	  e	e
                                          j        d         Zedz  Zedz  Zej        Zedz  ed	z  ed
z  edz  edz  edz  edz  edz  gZ ej        dej                  Zd Zd Zd Zedk    r ej         e                       dS dS )    N)defaultdict)datetimetimezone)Path   zstory/spoken-narration.txtzstory/originality-report.jsonzs002-Co-Ke-Toan-Bi-Ep-Nhan-Toi-Bien-Thu-Quy-Cuu-Tro-Den-Khi-Muoi-Hai-Chu-Ky-Cung-Phan-Chu/story/spoken-narration.txtzQ003-Co-Chu-Tro-Bi-Ep-Ban-Nha-Bay-Chia-Khoa-Bong-Tro-Ve/story/spoken-narration.txtzS004-Co-Ky-Su-Bi-Vu-Toi-Sap-Cau-Muoi-Ba-Vet-Nut-Doi-Huong/story/spoken-narration.txtzT005-Co-Dau-Bep-Bi-Vu-Bo-Doc-Chin-Chiec-Dia-Cung-Len-Tieng/story/spoken-narration.txtz\006-Co-Nhac-Truong-Bi-Vu-Pha-Buoi-Dien-Bay-Nhip-Den-Dong-Loat-Tat/story/spoken-narration.txtzY007-Co-Duoc-Si-Bi-Vu-Doi-Thuoc-Tam-Lo-Thuy-Tinh-Cung-Len-Tieng/story/spoken-narration.txtz\008-Co-Giao-Bi-Vu-Lo-De-Thi-Chin-To-Giay-Than-Dong-Loat-Len-Tieng/story/spoken-narration.txtzV009-Ngay-Toi-Huy-Hon-Anh-Moi-Nhan-Ra-Chiec-Ve-Sau-Nam-Truoc/story/spoken-narration.txtu   [\wÀ-ỹĐđ]+c                 J    d t                               |           D             S )Nc                 6    g | ]}|                                 S  )lower).0tokens     script/validate-originality.py
<listcomp>ztokens.<locals>.<listcomp>   s     >>>eEKKMM>>>    )TOKEN_REfindall)texts    r   tokensr      s$    >>x'7'7'='=>>>>r   c           	            fdt          t          dt                     z
  dz                       D             S )Nc                 D    h | ]}t          ||z                      S r
   )tuple)r   iitemssizes     r   	<setcomp>zngrams.<locals>.<setcomp>    s-    MMMqE%!D&/""MMMr   r   r   )rangemaxlen)r   r   s   ``r   ngramsr      sB    MMMMME#aUD9J2K2K,L,LMMMMr   c            	      $   t                                           s't          t          j        ddgd                     dS t                                           } t          |           }g g }}d |                     d          D             }d t          d          	                    |          
                                D             }|r%|                    d	t          |                      t          |d
          }t          D ]}|                                s|                    d|            0t          |                                          }|t          |d
          z  }	t          |	          t          dt          |                    z  }
|                    t!          |          t          |	          |
d           t          |	          dk    s|
dk    r7|                    d|j        j        j         dt          |	                      d| t'          j        |                                                                           |t          |          |t/          j        t2          j                                                  d}t8                              t          j        |dd          dz              t          t          j        |d         t          |          |dd                     |d         rdndS )NFzmissing current narration)verifiedissuesr   c                     g | ]P}|                                 t          j        d d|                                                                           QS )z\s+ )stripresubr   )r   ps     r   r   zmain.<locals>.<listcomp>*   sJ    bbbQXYX_X_XaXab"&aggiioo&7&788bbbr   z

c                 J    g | ] \  }}|d k    t          |          dk    |!S )r      )r   )r   r(   counts      r   r   zmain.<locals>.<listcomp>+   s^      C  C  C(!Uglopgpgpuxyzu{u{  B  vB  vBA  vB  vB  vBr   collectionszduplicate long paragraphs:    zmissing originality reference: )pathshared_12gramsratio   gMb`?zexcessive 12-gram overlap with z: )versionr!   source_sha256comparisonsduplicate_long_paragraphsr"   
checked_at   )ensure_asciiindent
r!   )r!   
referencesr"   )r8   r   )CURRENTexistsprintjsondumps	read_textr   split
__import__Counterr   appendr   r   
REFERENCESr   strparentnamehashlibsha256encode	hexdigestr   nowr   utc	isoformatREPORT
write_text)r   currentr"   r4   
paragraphsduplicate_paragraphs
current_12r.   refoverlapr0   reports               r   mainrZ   #   s#   >> dje8S7TUUVVWWWqDTllGbKFbb$**VBTBTbbbJ C  Cj.G.G.O.OPZ.[.[.a.a.c.c  C  C  C QOC8L4M4MOOPPP$$J 	g 	g{{}} 	MMBDBBCCCT^^%%&&vc2.Gs1c*oo666CIIWX]^^___w<<"MMeDK<N<SeeWZ[bWcWceefffF
W^TXT_T_TaTaEbEbElElEnEn  @K  jm  nB  jC  jC  OU  em  eq  rz  r~  e  e  eI  eI  eK  eK  L  LF
djeAFFFMNNN	$*&"4CDTDT`fggv{
|
|
|}}}z")11)r   __main__)rJ   r?   r&   sysr,   r   r   r   pathlibr   __file__resolveparentsPROJECTr<   rQ   rH   BASErF   compileUNICODEr   r   r   rZ   __name__exitr
   r   r   <module>rg      ss     				 



 # # # # # # ' ' ' ' ' ' ' '      
$x..
 
 
"
"
*1
-
0
0	2	2~  A  A^^``aaiiffiicc	
 2:("*55? ? ?N N N* * *6 zCHTTVV r   