
    D\j                     v   d dl Z d dlZd dlZd dlZd dlmZ d dlmZmZ d dlm	Z	  e	e
                                          j        d         Zedz  Z e	d           e	d           e	d	          gZed
z  Z ej        dej                  Zd Zd Zd Zd Zedk    r ej         e                       dS dS )    N)defaultdict)datetimetimezone)Path   zstory/spoken-narration.txtz/data/video-pipeline/Chanel-HuyenAnAudio/project/002-Co-Ke-Toan-Bi-Ep-Nhan-Toi-Bien-Thu-Quy-Cuu-Tro-Den-Khi-Muoi-Hai-Chu-Ky-Cung-Phan-Chu/story/v3/spoken-narration.txtz/data/video-pipeline/Chanel-HuyenAnAudio/project/003-Co-Chu-Tro-Bi-Ep-Ban-Nha-Bay-Chia-Khoa-Bong-Tro-Ve/story/spoken-narration.txtz/data/video-pipeline/Chanel-HuyenAnAudio/project/004-Co-Ky-Su-Bi-Vu-Toi-Sap-Cau-Muoi-Ba-Vet-Nut-Doi-Huong/story/spoken-narration.txtzstory/originality-report.jsonu   [\wÀ-ỹĐđ]+c                 J    d t                               |           D             S )Nc                 6    g | ]}|                                 S  )casefold).0tokens     script/validate-originality.py
<listcomp>ztokens.<locals>.<listcomp>   s"    AAAENNAAA    )TOKEN_REfindalltexts    r   tokensr      s$    AA(*:*:4*@*@AAAAr   c                 r    t          j        |                                                                           S )N)hashlibsha256encode	hexdigestr   s    r   digestr      s&    >$++--((22444r   c                    t          t                    }t          |          D ] \  }}||                             |           !d}d }i }t          |           D ]X\  }}i }|                    |g           D ]8}	|                    |	dz
  d          dz   }
|
||	<   |
|k    r|
}||
z
  dz   |dz   f}9|}Y||r)d                    | |d         |d                            ndfS )Nr   r     )r   list	enumerateappendgetjoin)firstsecond	positionsindexr   bestsamplepreviouscurrentother_indexlengths              r   longestr.      s   D!!I!&)) ' 'u%&&&&DFH!%((  u$==33 	9 	9K\\+/1559F#)GK }}&.1,eai8G%q	&) 34555RGGr   c                     g } t                                           rt                                           nd}|s|                     d           t	          |          }g }t
          D ]}|                                s&|                     dt          |          z              =|                                }t	          |          dfdt          t          dt                    z
  dz                       D             }g }t          t          dt          |          z
  dz                       D ]`}t          |||z                      }	|	|v r+|                    |d                    |	          d	           t          |          d
k    r na|rrt          |          nd\  }
}|                    t          |          t          |          ||
|d d         d           |r7|                     d|j        j        j         dt          |                      |
dk    r*|                     d|j        j        j         d|
            d t!          j        d|          D             }i }g }t%          |          D ]e\  }}t'          j        |                                                                          }||v r|                    ||         |d           `|||<   f|r%|                     dt          |                      d|  | sdnd|rt          |          nd |d|| t/          j        t2          j                                                  d	}t8                              t=          j        |dd          dz              tA          t=          j        |d         t          |          | dd                     |d         rdndS )Nr   candidate_missingzreference_missing:   c                 D    h | ]}t          ||z                      S r
   )tuple)r   r'   reference_tokenssizes     r   	<setcomp>zmain.<locals>.<setcomp>>   s0    }}}EE*5+=>??}}}r   r   r   r   )r'   r      )r   r   i  )pathr   shingle_matcheslongest_exact_token_runr)   zreference_shingle_matches::zlong_exact_token_run:c                     g | ]\}t          t          |                    d k    "t          j        dd|                                                                          ]S )#   z\s+r   )lenr   resubstripr   )r   items     r   r   zmain.<locals>.<listcomp>L   sy      G  G  G4nqrxy}r~r~nn  DF  oF  oF"&djjll&;&;&=&=>>  oF  oF  oFr   z\n\s*\n)r$   	duplicatezinternal_duplicate_paragraphs:passedfailed)	versionverifiedstatussource_canon_sha256
referencesshingle_sizeinternal_duplicate_paragraphsissues
checked_atF   )ensure_asciiindent
rG   )rG   rJ   rM   )rP   )!	CANDIDATEexists	read_textr!   r   
REFERENCESstrrangemaxr>   r3   r#   r.   r   parentnamer?   splitr    r   r   r   r   r   nowr   utc	isoformatREPORT
write_textjsondumpsprint)rM   	candidatecandidate_tokensrows	referencer   shinglesmatchesr'   shinglerunr)   
paragraphsseen
duplicates	paragraph	item_hashreportr4   r5   s                     @@r   mainrs   0   sW   F)2)9)9););C	##%%%I +)***i((D X X	!! 	MM.Y?@@@""$$!$<<}}}}}USVWXZ]^nZoZorvZvyzZzS{S{M|M|}}}3q#&6"7"7$">"BCCDD 	 	E,U54<-?@AAG(""8I8IJJKKK7||r!! "EUwZjwg.0@AAApwVS^^vd||X_|  LR  SW  TW  SW  LX  Y  Y  	Z  	Z  	Z 	fMMdy7G7N7SddVYZaVbVbddeee"99MMV)2B2I2NVVQTVVWWW G  G28T^`iKjKj  G  G  GJDJ%j11 $ $yN9#3#3#5#566@@BB	YeLLMMMM#DOO JHs:HHIIIJ"(6((h4=Gvi0004)3l8<00::<<
 
F djeAFFFMNNN	$*&"4CIIY_``ot
u
u
uvvvz")11)r   __main__)r   rb   r?   syscollectionsr   r   r   pathlibr   __file__resolveparentsPROJECTrS   rV   r`   compileUNICODEr   r   r   r.   rs   __name__exitr
   r   r   <module>r      sv     				 



 # # # # # # ' ' ' ' ' ' ' '      
$x..
 
 
"
"
*1
-22	D  
s  t  tD  
N  O  OD  
P  Q  Q

 
2	22:("*55B B B5 5 5H H H&4* 4* 4*n zCHTTVV r   