
    Y[jd                     X   d dl Z d dlZd dlZd dlZd dlmZ d dlmZmZ d dlm	Z	  e	e
                                          j        d         Zedz  dz  Z e	d          Zedz  d	z  Z ej        d
ej                  Zd Zd Zd Zd Zedk    r ej         e                       dS dS )    N)defaultdict)datetimetimezone)Path   storyzspoken-narration.txtz/data/video-pipeline/Chanel-HuyenAnAudio/project/002-Co-Ke-Toan-Bi-Ep-Nhan-Toi-Bien-Thu-Quy-Cuu-Tro-Den-Khi-Muoi-Hai-Chu-Ky-Cung-Phan-Chu/story/v3/spoken-narration.txtzoriginality-report.jsonu   [\wÀ-ỹĐđ]+c                 J    d t                               |           D             S )Nc                 6    g | ]}|                                 S  )casefold).0xs     script/validate-originality.py
<listcomp>ztokens.<locals>.<listcomp>   s     GGG1!**,,GGG    )TOKEN_REfindalltexts    r   tokensr      s$    GG0@0@0F0FGGGGr   c                 r    t          j        |                                                                           S )N)hashlibsha256encode	hexdigestr   s    r   digestr      s%    66@@BBBr   c                    t          t                    }t          |          D ] \  }}||                             |           !d}d }i }t          |           D ]X\  }}i }	|                    |g           D ]8}|                    |dz
  d          dz   }
|
|	|<   |
|k    r|
}||
z
  dz   |dz   f}9|	}Y||r)d                    | |d         |d                            ndfS )Nr   r     )r   list	enumerateappendgetjoin)ab	positionsjtbestsampleprevicurns              r   longest_runr0      s   $I||33!Yq\003333	
D4F	D||  !q$$ 	1 	1Ahhqs1ooaAqAvvAtac!eAaC[ffD#((1VAYvay01222"EEr   c                     g } t                                           s|                     d           t                                          s|                     d           t                                           rt                               d          nd}t                                          rt                              d          nd}t          |          t          |          c}dfdt          t          dt                    z
  d	z                       D             }g }t          t          dt          |          z
  d	z                       D ]`}t          |||z                      }||v r@|                    |d

                    |          d           t          |          dk    r na|r%|                     dt          |                      |rrt          |          nd\  }}	|dk    r|                     d|            d t          j        d|          D             }
i }g }t          |
          D ]e\  }}t          j        |                                                                          }||v r|                    ||         |d           `|||<   f|r%|                     dt          |                      d	|  | sdnd|rt'          |          nd t)          t                    |rt'          |          nd |||	d d         || t+          j        t.          j                                                  d}t4                              t9          j        |dd          dz              t=          t9          j        |d         |t          |          t          |          | dd                     |d         rdnd	S )Ncandidate_missingreference_missingzutf-8)encodingr      c                 D    h | ]}t          ||z                      S r   )tuple)r   r-   r&   r/   s     r   	<setcomp>zmain.<locals>.<setcomp>'   s+    JJJAa!A#hJJJr   r   r   r   )indexr      zreference_shingle_matches:)r   r   zlong_exact_token_run:c                     g | ]\}t          t          |                    d k    "t          j        dd|                                                                          ]S )#   z\s+r   )lenr   resubstripr   )r   ps     r   r   zmain.<locals>.<listcomp>0   sX    vvvAcfgmnogpgpcqcqsucucuvc!''))"4"4"6"677cucucur   z\n\s*\n)first	duplicatezinternal_duplicate_paragraphs:passedfailedi  )versionverifiedstatussource_canon_sha256reference_pathreference_sha256shingle_sizeshingle_matcheslongest_exact_token_runlongest_run_sampleinternal_duplicate_paragraphsissues
checked_atF   )ensure_asciiindent
rG   )rG   r0   rM   duplicate_paragraphsrQ   )rT   )	CANDIDATEexistsr"   	REFERENCE	read_textr   rangemaxr=   r7   r$   r0   r>   splitr!   r   r   r   r   r   strr   nowr   utc	isoformatREPORT
write_textjsondumpsprint)rQ   	candidate	referencer%   ref_shinglesmatchesr-   shrunr+   
paragraphsseen
duplicatesrA   hreportr&   r/   s                   @@r   mainrs       s   FAv}}-@AAAAv}}-@AAA7@7G7G7I7IQi!!7!333rI7@7G7G7I7IQi!!7!333rIy&++CAaAJJJJJs1SVVAXaZ7H7H1I1IJJJ,TVG3qQ!$$%% ' '1QqS5??NNASXXb\\::;;;7||RJI3w<<IIJJJ#$66{1QJC
Bww;c;;<<<vv*U^A_A_vvvJ	DZ$$  !
.
$
$
.
.
0
099j''aQ(G(GHHHHd1ggT6==!S#j//!S!STTTv:v7[xxS[  HQ  s[rx  zC  sD  sD  sD  W[  mp  qz  m{  m{  dm  Ow  OU  V_  O`  O`  O`  sw  GH  [b  }@  V\  ]a  ^a  ]a  Vb  CM  W]  ks  kw  x@  xD  kE  kE  kO  kO  kQ  kQ  R  RF
djU!DDDTIJJJ	$*
!3#X[\cXdXd|  AK  }L  }L  V\  ]  ]  kp  q  q  q  r  r  rz")11)r   __main__)r   re   r>   syscollectionsr   r   r   pathlibr   __file__resolveparentsPROJECTrX   rZ   rc   compileUNICODEr   r   r   r0   rs   __name__exitr   r   r   <module>r      s8     				 



 # # # # # # ' ' ' ' ' ' ' '      
$x..
 
 
"
"
*1
-g 66	D  {  |  |		7	6	62:("*55 G G G B B BF F F* * *8 Z(((((r   