
    P\jJ                    .   d dl mZ d dlZd dlZd dlZd dlZd dlZd dlmZ  ee	          
                                j        d         Zedz  Zedz  Zedz  ZdZd	Zd!dZd"dZd#d$dZd%dZd&dZd'dZed k    r ej         e                       dS dS )(    )annotationsN)Path   z&work/story/run-001/story-candidate.txtzlog/candidate-validation.jsonzlog/originality-corpus.jsoni%  i0  databytesreturnstrc                N    t          j        |                                           S N)hashlibsha256	hexdigest)r   s    script/validate-candidate.pysha256_bytesr      s    >$))+++    textc                    t          j        d|                                           } d                    |                                           S )NNFC )unicodedata	normalizecasefoldjoinsplit)r   s    r   r   r      s:     --6688D88DJJLL!!!r    valueobjectkey	list[str]c                   g }t          | t                    r?t          |           dk    r+|                                dv r|                    |            nt          | t
                    r)| D ]%}|                    t          ||                     &nRt          | t                    r=| 	                                D ](\  }}|                    t          ||                     )|S )Nx   >   bodyr   prosestorycontent	narration	paragraph)

isinstancer	   lenr   appendlistextendprose_from_jsondictitems)r   r   resultitem	child_keys        r   r-   r-      s    F% 
<u:: 4
 "
 "
 MM%   	E4	 	  < 	6 	6DMM/$445555	6	E4	 	  <${{}} 	< 	<OItMM/$	::;;;;Mr   pathr   c                    | j                                         dk    r|                     d          S t          j        |                     d                    }d                    t          |                    S )Nz.txtutf-8encodingz

)suffixr   	read_textjsonloadsr   r-   )r3   objs     r   load_corpus_textr=   ,   sf    {''~~w~///
*T^^W^55
6
6C;;s++,,,r   tokenssizeintset[tuple[str, ...]]c                     t                     k     rt                      S  fdt          t                     z
  dz             D             S )Nc                D    h | ]}t          ||z                      S  )tuple).0ir?   r>   s     r   	<setcomp>zngrams.<locals>.<setcomp>6   s.    MMM!E&1t8$%%MMMr   r   )r)   setrange)r>   r?   s   ``r   ngramsrK   3   sS    
6{{TuuMMMMMuS[[45G!5K/L/LMMMMr   c                 
   t                                           s9t          t          j        ddt          t                     d                     dS t                                           } |                     dd                                          }	                                }d t          |          D             d	 t          |          D             d
 t          |          D             d t          |          D             d t          |          D             d t          |          D             d}g d}ddgddgddgdgg dg dd}fd|D             }fd|                                D             }}|D ]}	|                    |	d          }g }
dD ]a}t          t          j        dt          j        |           d|                    }|r%|
                    |t%          |          d            bt'                    	                                }t)          |d!          }g }t          j        t,                              d"                    }|                    d#g           D ]}t3          |d$                   }|                                s|                    |d%         d&d'           Kt5          |          }t'          |          	                                }t)          |d!          }||z  }|                    |d%         t          |          t7          |                    d                    t%          |          d( t;          |          d d)         D             t%          |          d*k    d+           d&d&d,t<          t%          |          cxk    o
t>          k    nc t%          |          d,tA          !                                          t%          |           d,tE          |#                                           |d-| |d-| |d-|
 |
d-tI          d. |D                       |d-d/}tI          d0 |#                                D                       }|rd1nd2|t          t                     t7          |           t%          |           t%          |          |d3}tJ          &                    t          j        |d4d5          d6z   d"           t          t          j        |d7         ||d8         t%          |          t          tJ                    d9d4:                     |rd*nd;S )<Nblockedcandidate_missing)statusreasonr3      r5   strict)errorsc                H    g | ]\  }}t          j        d |          |dz    S )z^\s*#{1,6}\s+\Sr   rematchrF   rG   lines      r   
<listcomp>zmain.<locals>.<listcomp>D   sC     #
 #
 #
ax*D11#
E#
 #
 #
r   c                H    g | ]\  }}t          j        d |          |dz    S )u   ^\s*(?:(?:CHƯƠNG|Chương|PHẦN|Phần|CẢNH|Cảnh|SCENE|Scene|ACT|Act)(?:\s+\d+|\s*[:.-])|(?:\*\s*\*\s*\*|---+|===+))\s*$r   rU   rX   s      r   rZ   zmain.<locals>.<listcomp>H   sK     
 
 
axG 
E
 
 
r   c                r    g | ]4\  }}t          j        d |          st          j        d|          /|dz   5S )z^\s*[\[\]{}]\s*,?\s*$z^\s*"[^"\n]+"\s*:r   rU   rX   s      r   rZ   zmain.<locals>.<listcomp>P   sZ     !
 !
 !
ax0$77!
 x,d33!
E!
 !
 !
r   c                ^    g | ]*\  }}t          j        d |t           j                  %|dz   +S )uU   ^\s*(?:Ghi chú|Lưu ý|Meta|Narration|Tóm tắt|Từ khóa|Word count|SHA-?256)\s*:r   )rV   rW   IrX   s      r   rZ   zmain.<locals>.<listcomp>U   sM     
 
 
axh 
E
 
 
r   c                ^    g | ]*\  }}t          j        d |t           j                  %|dz   +S )z)\b(?:TODO|TBD|PLACEHOLDER)\b|<[^>]{2,50}>r   rV   searchr^   rX   s      r   rZ   zmain.<locals>.<listcomp>]   sG     
 
 
ayEtRTRR
E
 
 
r   c                ^    g | ]*\  }}t          j        d |t           j                  %|dz   +S )u+   Gác Mái Audio|like video|đăng ký kênhr   r`   rX   s      r   rZ   zmain.<locals>.<listcomp>a   sG     
 
 
ayGrtTT
E
 
 
r   )markdown_heading_linesscene_marker_linesjson_structure_linesmeta_note_linesplaceholder_lineschannel_cta_lines)u   Tống Vãn Ninhu   Cố Trạch Xuyênu   Lục Mạn Thanhu   Tần Nhược Lanu   Tống Nghi Lanu   áo cướiu
   lụa ngàu   chỉ đỏu	   đỏ sonu   Túc Vũu   xưởng mayu   phục chế)u	   xin phépu   quyền từ chốiu   tự chọnu   quyết định)u   mùa mưa sauu
   cầu hônu   chiếc chìa)wedding_dress
red_threadworkshoprestorationconsentendingc                    g | ]}|v|	S rD   rD   )rF   namer   s     r   rZ   zmain.<locals>.<listcomp>p   s#    IIIdD8H8HT8H8H8Hr   c                P    g | ]"\  }}t          fd |D                        |#S )c              3  h   K   | ],}|                                                                  v V  -d S r   )r   )rF   aliasr   s     r   	<genexpr>z"main.<locals>.<listcomp>.<genexpr>s   s:      LL55>>##t}}6LLLLLLr   )any)rF   r   aliasesr   s      r   rZ   zmain.<locals>.<listcomp>q   sP       WLLLLGLLLLL  r   r   )Ninhu   XuyênThanhLanz(?<!\w)z(?!\w))tokencount   r6   corpussource_pathprojectT)r   source_missingc                8    g | ]}d                      |          S )r   )r   rF   r   s     r   rZ   zmain.<locals>.<listcomp>   s"    HHH5sxxHHHr      r   )r   r~   source_sha256shared_12gramssampleverified)r   r   )r   findingsc              3  B   K   | ]}|                     d d          V  dS )r   FN)get)rF   r1   s     r   rt   zmain.<locals>.<genexpr>   s0      PPDDHHZ77PPPPPPr   )utf8_strict
word_count	non_emptynarration_hygienerequired_namesrequired_conceptsname_calling_ruleoriginality_12gramc              3  &   K   | ]}|d          V  dS )r   NrD   r   s     r   rt   zmain.<locals>.<genexpr>   s'      BB5$BBBBBBr   	completedfailed)rO   r   candidate_pathcandidate_sha256r   words_whitespacechecksF)ensure_asciiindent
rO   r   )rO   r   r   r   report)r   r   )'	CANDIDATEexistsprintr:   dumpsr	   
read_bytesdecode
splitlinesr   	enumerater/   replacer+   rV   finditerescaper*   r)   r   rK   r;   CORPUSr9   r   r   r=   r   encodesorted	MIN_WORDS	MAX_WORDSboolstripru   valuesallREPORT
write_text)rawlineswordsr   r   r   missing_namesmissing_conceptsstrippedrp   prohibited_single_name_callsrz   matchescandidate_tokenscandidate_ngramsoriginalityr}   r1   sourceold_text
old_tokens
old_ngramsoverlapr   r   r   r   s                             @r   mainr   9   s    djI9LVYZcVdVdeeffgggq



 
 C::gh://DOOEJJLLE#
 #
!*5!1!1#
 #
 #

 
!*5!1!1
 
 
!
 !
!*5!1!1!
 !
 !


 
!*5!1!1
 
 

 
!*5!1!1
 
 

 
!*5!1!1
 
 
="# "#HH ~~N'6$k20&'YYYBBB  JIIInIIIM    1 7 7 9 9  
 H / /##D#..#% 3 Y Yr{#Fbi.>.>#F#F#FQQRR 	Y(//%#g,,0W0WXXX ,,...33KZ(('(::;;F

8R((  d=)**}} 	4	?dSSTTT#F++x((..00
J++
"Z/Iv;;)(//'*B*BCC!'llHHF7OOBQB4GHHHG)
 
 	 	 	 	 %)488#,E

#G#G#G#Gi#G#G#G#GRUV[R\R\]]"&tzz||"4"4s3xxHH 1 1222 
 
 ,9'8mTT.>*>L\]]884
 

 PPKPPPPP#
 
 F& BB&--//BBBBBH!)7++xi..(--SJJ F djeAFFFMX_```	$*""#56JJf++        11ar   __main__)r   r   r   r	   )r   r	   r   r	   )r   )r   r   r   r	   r   r   )r3   r   r   r	   )r>   r   r?   r@   r   rA   )r   r@   )
__future__r   r   r:   rV   sysr   pathlibr   __file__resolveparentsROOTr   r   r   r   r   r   r   r-   r=   rK   r   __name__exitrD   r   r   <module>r      s^   " " " " " "   				 



          tH~~'*;;		/	/	-	-		, , , ," " " "
     - - - -N N N NA  A  A  A H zCHTTVV r   