
    Pw_j-D                       d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	m
Z
 d dlmZ  ed          ZdZdZedz  ez  d	z  Zed
z  Zedz  Zedz  Zedz  Zedz  Zedz  Z ed          ZdZdZd9dZd:dZd;dZd<dZd=d Zd>d!Zd?d"Z d@d%Z!dAd*Z"dBdCd.Z#dDd2Z$dEd4Z%dFd6Z&d>d7Z'e(d8k    r e'             dS dS )G    )annotationsN)Counter)PathzQ/data/video-pipeline/HaTramAudio/project/020-Chiec-Nhan-Chi-Deo-Khi-Co-Nguoi-Nhinzrun-20260721T124541Z-88551d8aLevyz
work/storyzcandidate.txtzstory/story-canon.txtzscript/canon.txtzstory/spoken-narration.txtzscript/promotion-report.jsonzlog/story-qa.jsonzlog/story.jsonz(/data/video-pipeline/HaTramAudio/projecti%  i$,  returnstrc                     t           j                             t           j        j                                                                      dd          S )Nz+00:00Z)datetimenowtimezoneutc	isoformatreplace      work/verify_and_promote_story.pyr   r      s<      !2!677AACCKKHVYZZZr   databytesc                N    t          j        |                                           S N)hashlibsha256	hexdigest)r   s    r   	sha_bytesr   !   s    >$))+++r   pathr   c                D    t          |                                           S r   )r   
read_bytes)r   s    r   shar   %   s    T__&&'''r   Nonec                   | j                             dd           |                     d| j        z   dt	          j                     dz             }|                    d          5 }|                    |           |                                 t	          j	        |
                                           d d d            n# 1 swxY w Y   t	          j        ||            d S )NT)parentsexist_ok.z.tmpwb)parentmkdir	with_namenameosgetpidopenwriteflushfsyncfilenor   )r   r   tmphandles       r   atomic_bytesr3   )   s    KdT222
..ty+@ry{{+@+@+@@
A
AC	$ "6T
!!!" " " " " " " " " " " " " " " JsDs   'ACC
Cvaluedictc                |    t          | t          j        |dd          dz                       d                     d S )NF   ensure_asciiindent
utf-8)r3   jsondumpsencode)r   r4   s     r   atomic_jsonr@   3   s=    
5uQGGG$NVVW^__`````r   c                    t          j        t          dz                      d                    t          j        t
          t          d} t          fd|                                 D                       rt          d          t          j        t          dz                      d                    }|
                    d          d	k    s<|
                    d
          t
          k    s|
                    d          t          k    rt          d          t          j        t          dz                      d                    }|
                    di           
                    d          dk    rt          d          d S )Nz.ownership-lock.jsonr<   encoding)
project_idrun_idownerc              3  P   K   | ] \  }}                     |          |k    V  !d S r   )get).0kvlocks      r   	<genexpr>zguard.<locals>.<genexpr>:   s6      
9
91488A;;!
9
9
9
9
9
9r   zownership lock mismatchscript/gate-leases/story.jsonstatusactiverE   rF   z!Story lease is not active/currentscript/project-manifest.jsonstepsstorypendingzStory Gate is not pending)r=   loadsROOT	read_textr)   RUN_IDOWNERanyitemsRuntimeErrorrH   )expectedleasemanifestrL   s      @r   guardr`   7   sX   :t44???QQRRD"i6EJJH

9
9
9
9(8(8
9
9
999 64555J>>IISZI[[\\Eyyh&&%))H*=*=*G*G599U\K]K]afKfKf>???z4"@@KKU\K]]^^H||GR  $$W--::6777 ;:r   c                |    |                                  }|j        |j        |j        |j        t          |           dS )N)deviceinodesizemtime_nsr   )statst_devst_inost_sizest_mtime_nsr   )r   rf   s     r   fingerprintrk   D   s<    99;;D+$d))  r   text	list[str]c                f    t          j        d|                                 t           j                  S )Nu   [0-9A-Za-zÀ-ỹĐđ]+)refindallcasefoldUNICODE)rl   s    r   tokensrs   O   s!    :/"*MMMr   r[   nintset[tuple[str, ...]]c                     t                     k     rt                      S  fdt          t                     z
  dz             D             S )Nc                D    h | ]}t          ||z                      S r   tuple)rI   ir[   rt   s     r   	<setcomp>zngrams.<locals>.<setcomp>V   s-    CCCAE%!A#,CCCr      )lensetrange)r[   rt   s   ``r   ngramsr   S   sP    
5zzA~~uuCCCCCs5zzA~/A)B)BCCCCr   patternflags	list[int]c                    t          j        | |          fdt          |                                d          D             S )Nc                D    g | ]\  }}                     |          |S r   )search)rI   r{   lineregexs      r   
<listcomp>z line_numbers.<locals>.<listcomp>[   s.    SSS'!TT@R@RSASSSr   r}   )ro   compile	enumerate
splitlines)r   rl   r   r   s      @r   line_numbersr   Y   sB    Jw&&ESSSSYt'8'8!<<SSSSr   	text_foldalternativesboolc                :     t           fd|D                       S )Nc              3  D   K   | ]}|                                 v V  d S r   )rq   )rI   itemr   s     r   rM   z!phrase_present.<locals>.<genexpr>_   s0      EEt}})+EEEEEEr   )rZ   )r   r   s   ` r   phrase_presentr   ^   s&    EEEEEEEEEEr   dict[str, bool]c                   |                                  }d                    |                                 d d                                                    }|}d                    |                                 dd                                                     }|                    d          }|dk    r
||d          n|}i dt	          |g d          dt	          |g d	          d
t	          |g d          dt	          |g d          dt	          |ddg          ot	          |g d          dt	          |g d          dt	          |g d          dt	          |g d          dt	          |g d          ot	          |g d          dt	          |g d          ot	          |ddg          dt	          |g d           ot	          |d!d"g          d#t	          |d$g          ot	          |d%g          d&t	          |g d'          ot	          |d(d)g          d*t	          |g d+           d,t	          |g d-          ot	          |g d.          d/t	          |g d0          ot	          |d1d2g          S )3N i  i|u"   ngày thứ một trăm hai mươir   gate0_hook_early)u   bảng tênu	   ấn tínu
   máy ảnh   chiếc nhẫngate1_narrator_early)u   tôi là tạ minh daou   tạ minh daozminh daogate1_place_early)u   trùng khánhu	   nhà tổu   triển lãmgate1_goal_early)
   minh châuu
   thay chỗu   một trăm hai mươi ngàyu	   120 ngàylegal_marriage_before_loveu   đăng ký kết hônu   giấy chứng nhận)u
   chưa yêuu   không phải tình yêuu   không phải vì yêuu   chưa có tình cảmindependent_counsel)u   luật sư riêngu   luật sư độc lậpu   nhược hàseparate_rooms)u   phòng riêngu
   hai phòngu   phòng của tôiu   phòng của anh	ring_rule)u(   chỉ đeo nhẫn khi có người nhìnu"   đeo nhẫn khi có người nhìnu   không còn ai nhìnboth_replacements_midpoint)u   hai lá thư từ chốiu   hai bức thư từ chốiu!   hai tờ giấy nằm cạnh nhau)u   cũng là người thay thếu   cũng tự nguyện thayu   anh tự nguyện thaynameplate_climax)u   tháo bảng tênu   bảng tên nàng dâuu   tên thật của tôiu%   tháo tấm thẻ giấy khỏi khungu   tạ minh dao hiện rõu   bảng tên đã viếtritual_seal_returned)u   trả ấn tínu   đặt ấn tínu   ấn tín xuốngu   chiếc ấn đá từ hộpu   trả quyền đại diệnu$   đặt xuống cạnh chiếc nhẫnyounger_pair_speaksr   u	   gia hạofemale_lead_choice)u
   tôi hỏiu   tôi muốnu   tôi chọnr   u   đeo nhẫnno_divorce_ending)u   chúng tôi ly hônu   chúng tôi đã ly hônu   nộp đơn ly hônu   ký đơn ly hônu   chấm dứt hôn nhânprivate_ring_coda)u   bữa sángu   không có kháchu   không ai nhìn)u   nhẫn vẫnr   u   hai chiếc nhẫntwo_lights_coda)u   hai ngọn đènu   ngọn đèn còn lạiu   gần như cùng lúcu   hai vùng sángu   bật ngọn đèn phía mình)rq   joinsplitrfindr   )rl   foldfirstopeninglastclimax_markerclimaxs          r   semantic_checksr   b   sJ   ==??DHHTZZ\\$3$'((1133EG88DJJLL'((1133DJJCDDM%2a%7%7T-..!!TFN52n2n2noow8o8o8o p p 	^G5c5c5cdd 	N74}4}4}~~	
 	%nT<SUl;m&n&n  'g  tB  CG  If  If  If  tg  tg 	~d4t4t4tuu 	./w/w/wxx 	^D  +U  +U  +U  V  V 	%nT  <[  <[  <[  '\  '\  'N  ao  pt  vM  vM  vM  aN  aN 	N6  4f  4f  4f  g  g  {  lz  {A  D^  `y  Cz  l{  l{ 	v  8T  8T  8T  !U  !U  !y  Zh  io  rO  Qw  qx  Zy  Zy 	~d\NCCkW[^i]jHkHk 	nT3_3_3_``  ]estx  |L  N[  {\  f]  f] 	  7q  7q  7q  "r  "r  r 	^D2i2i2ijj  ~o}  C  E}  E}  E}  p~  p~  	>$0x0x0xyy  J  M  NR  Uf  hH  TI  J  J! r   
list[Path]c                 @   t          t                              d                    } g }t                      }| D ]c}t          |j        v s|                                s%t          |          }||v r9|                    |           |	                    |           d|S )Nz*/story/story-canon.txt)
sortedCORPUS_ROOTglobr   rV   r"   is_filer   addappend)pathsresultseenr   digests        r   corpus_filesr   }   s    ;##$=>>??EF55D  4<t||~~TT>>dMr   c                   345 t                       t                                          st          d          t	          t                    } t          j        d           t	          t                    }| |k    rt          d          t                                          }|                    d          5t          j
        d5          st          d          5                                }5                                }g }t          |          |                    d          |                    d          5                    d	          t!          d
5          t!          d5          t!          d5t"          j                  t!          d5t"          j                  t!          d5t"          j                  5                    d          t#          j        d5t"          j                  d}t(          t          |          cxk    rt*          k    s'n |                    dt          |          d           |                                D ](\  }}|dk    r|r|                    d|z   |d           )t1          j        t4          dz                      d                    }	g }
g }|	d                                         D ]8}|
                    |d                    |                    |d                    95}t=          t?          |
          t          d          D ]>}t#          j         dt#          j!        |           ddt          |          z  |          }?i }t=          t?          |                    D ]@}5fdt#          j"        dt#          j!        |           d|          D             }|r|||<   A|r|                    d |d           g d!}5fd"|D             }|r|                    d#|d           d#                    t#          j        d$5                    }t          |                                          }|tI          d%t          |                    z  }d&|cxk    rd'k    sn |                    d(|d           tK          5          }|                                D ]"\  }}|s|                    d)|z   d*d           #tM          5          3tO          3d+          }tQ          3fd,tS          tI          d-t          3          d.z
                      D                       }d/ |                                D             }|r |                    d0|d d1         d           g }d-}d2}tU                      D ]}tM          |                    d                    } tO          | d+          }!t          ||!z            }"tO          3d3          }#tO          | d3          }$t          |#|$z            }%|%rt          |#|$z            |%z  nd2}&||"z  }tI          ||&          }|                    |j+        j+        j,        t[          |          |"|&d4           |r|                    d5|d           |d6k    r|                    d7|d           i d8| d9t4          j,        d:t\          d;t_          |          d<t          |          d=dd>|dt          |          d?|d(|d@|dA|dB|dC|dDt          |          ||t          |          |dEdF|}'|r3ta          t1          j1        |'d*dGH                     te          d%          t                       t_          |          4t4          dIz  }(t1          j        |(                    d                    })t1          j        t4          dJz                      d                    }*t[          |(          }+|*3                    dK          },|+|,k    rt          dL          |)3                    dMi           3                    dN          dOk    s)|)3                    dPi           3                    dN          rt          dQ          |)3                    dR          d 4fvrt          dS          th          tj          tl          fD ]}to          ||           tq          4fdTth          tj          tl          fD                       rt          dU          g dV}-dW |-D             }.ts                      }/i dXdGd9t4          j,        d:t\          dYdZd8dd[4d;4dR4d\4dt          |          d]tu          t          |          d^z  dG          d_d`dadbdcdddddddddddddddddddddddddedfd(|dC|dD|'dD         |.|/dg}0i dXdGd9t4          j,        d:t\          dYdZd8ddhtw          t                    d;4di|d=ddjtw          th                    dR4dktw          tj                    dltw          tl                    d\4dmddntw          tx                    do|.dp|/i}1dGt4          j,        t\          dZd44tw          th                    4t          |          tw          tz                    tw          tx                    d`db|/dq}2t}          tx          |0           t}          tz          |1           t}          t~          |2           4|)dR<   t          |          |)d<   d`dbddGdr|)ds<   |/|)dt<   t}          |(|)           ta          t1          j1        dZdt          |          |4|ddud*v                     d S )wNzcandidate missingZ   z3candidate changed during 90-second stability windowr<   NFCzcandidate is not UTF-8 NFCs   ﻿s   
	z[ \t]+$z^\s*#{1,6}\s+u\   ^\s*(?:chương|phần|cảnh|hồi|chapter|part|scene)(?:\s+(?:\d+|[ivxlcdm]+)\b|\s*[:.\-])uH   ^\s*[\[(](?:nhạc|sfx|hiệu ứng|chuyển cảnh|pause|music|sound)\buI   (?:đăng ký|subscribe|nhấn chuông)\s+kênh|hãy\s+(?:like|chia sẻ)z```z\b(?:TODO|TBD|PLACEHOLDER)\b)
word_countbom
crlf_count	tab_counttrailing_space_linesmarkdown_heading_linesnamed_section_linesstage_direction_lineschannel_cta_linesfenced_code_countplaceholder_hitsr   )checkr4   zhygiene.script/ledger.jsonrB   identity_registryallowed_formsforbidden_one_syllable_aliasesT)keyreversez(?<!\w)z(?!\w)r   c                h    g | ].}                     d d|                                          dz   /S )r;   r   r}   )countstart)rI   mrl   s     r   r   zmain.<locals>.<listcomp>   s6    uuuq

4AGGII..2uuur   z identity.forbidden_short_aliases)u   Hứa Tịnh Nghiu   Trình Duy Khau   Vân Đìnhu   khách sạnu   cổ phầnu   quyền biểu quyếtu   ủy quyềnu   ghế chủ tịchu   đại hội cổ đôngu   bán tài sảnu   bán thương hiệuu   thương vụ thâu tómc                    i | ]d}|                                                                  v *|                                                     |                                           eS r   )rq   r   )rI   termrl   s     r   
<dictcomp>zmain.<locals>.<dictcomp>   s      B  B  BT^b^k^k^m^mquq~q~  rA  rA  _A  _AD$--////@@  _A  _A  _Ar   revision_1_residueu   [“"]([^”"\n]+)[”"]r}   gffffff?g?dialogue_ratioz	semantic.F   c              3  L   K   | ]}t          ||d z                      V  dS )r   Nry   )rI   r{   candidate_tokenss     r   rM   zmain.<locals>.<genexpr>   s9      oo!e$4QqtV$<==oooooor   r      c                N    g | ]"\  }}|d k    d                     |          |f#S )r}   r   )r   )rI   rJ   rK   s      r   r   zmain.<locals>.<listcomp>   s7    VVVTQPQTUPUPU!a(PUPUPUr   z&originality.internal_12gram_repetition   g           )projectr   exact_12gramjaccard_5gramzoriginality.corpus_exact_12gramg{Gz?zoriginality.max_5gram_jaccardverifiedrD   rE   candidate_sha256candidate_bytesstable_secondsrk   dialogue_wordshygiene
alias_hitsrevision_1_residue_hitsr   originality)corpus_countexact_12gram_overlap_totalmax_5gram_jaccardinternal_repeated_12gram_countcorpusfailuresr7   r8   rQ   rN   manifest_sha256_at_acquirez.manifest changed since Story lease acquisitionrR   rS   rT   gate_eventsz'Story Gate changed before canon bindingcanon_sha256z(manifest already binds a different canonc              3  >   K   | ]}t          |          k    V  d S r   )r   )rI   r   
canon_hashs     r   rM   zmain.<locals>.<genexpr>  s.      
P
Pt3t99
"
P
P
P
P
P
Pr   zraw-byte promotion mismatch)zscript/creative-options.jsonzscript/story-brief.jsonzscript/outline.jsonr   z$script/story-authority-revision.jsonc                >    i | ]}|t          t          |z            S r   )r   rV   )rI   rels     r   r   zmain.<locals>.<dictcomp>"  s$    HHHS__HHHr   schema_versionrO   	completedsource_canon_sha256spoken_narration_sha256estimated_minutes_at_231_wpm   story_family_primaryu   Cưới trước yêu saustory_family_secondaryu   Hào môn / liên hôngatespassedabsent)global_coherence_text_review
gate0_hookgate1_comprehensiondialogueidentity_aliastimeline_knowledge_propsconsent_power_balancenarration_hygieneoriginality_internal_12gramoriginality_corpus_12gramfamily_primary_payofffamily_secondary_payoffthird_family_drift)authority_sha256completed_atcandidate_pathcandidate_fingerprint
canon_pathscript_canon_pathspoken_narration_pathraw_byte_identicalstory_qa_pathr  r  )r  rD   rE   rO   r   r  r   artifact_pathartifact_sha256r   promotion_report_pathr!  r  r  r  )primary	secondarylockedrevisionstory_family
updated_at)rO   r   r   r   r   r   manifest_canon_bound)r9   )@r`   	CANDIDATEr   r\   rk   timesleepr   decodeunicodedatais_normalizedr   r   r~   
startswithr   r   ro   Irp   	MIN_WORDS	MAX_WORDSr   r[   r=   rU   rV   rW   valuesextendr   r   subescapefinditerr   maxr   rs   r   r   r   r   r&   r)   r   rX   r   printr>   
SystemExitrH   CANONSCRIPT_CANON	NARRATIONr3   rZ   r   roundr   
QA_RECEIPT	PROMOTIONr@   STORY_RECEIPT)6beforeafterrawwordslinesr   r   r   r4   ledgerallowed_namesforbidden_aliasesinfomaskedr)   r   aliashitsresidue_termsresidue_hitsdialogue_textr   r   semanticr
  cand12internal_countsinternal_repeatscorpus_resultsexact_overlap_totalmax_jaccardr   
old_tokensold12exactcand5old5unionjaccardr   manifest_pathr_   r^   current_manifest_hashexpected_manifest_hashauthority_filesauthority_hashesr  qa	promotionrS   r   r   rl   s6                                                      @@@r   mainri     s   	GGG 0.///##FJrNNN	""EPQQQ



 
 C::gD$UD11 97888JJLLEOOEH%jj~~o..ii((ZZ%% ,Z > >"./?"F"F+  -L  NR  TV  TX   Y   Y!-.y{  BD  BF  "G  "G)*vx|  A  C  D  D!ZZ..J'FbdSS G E

////i////,UDDEEEmmoo I I
U,5OOj3&6GGHHHZ 44???QQRRFM*+2244 I IT/2333  &F!GHHHHFs=))sDAAA U U:29T??:::C#d))OVTTJ-..// % %uuuuBKHkSUS\]bScScHkHkHkms<t<tuuu 	% $Ju \"DzZZ[[[  M
 B  B  B  B]  B  B  BL P"6NNOOO HHRZ(CTJJKKM,,..//N#c!SZZ&8&88N>))))T))))"2^LLMMMt$$H~~'' J JV 	JOOkC&7%HHIIId||$b))FoooouSQRTWXhTiTijlTlMmMmGnGnoooooOVV_5J5J5L5LVVV m"JUefigifiUjkklllNK 
J 
JDNNGN<<==
z2&&FUN##'++j!$$EDL!!/4=#edl##e++#u$+w//$+*<*ASQUYYhm  AH  I  I  	J  	J  	J  	J d"CNabbcccT"AKXXYYYLdi 	& 	IcNN	
 	3s88 	" 	u 	c%jj 	. 	. 	7 	j 	"< 	8 	//*=!,.12B.C.C$
 
, 	H-F0  djeA>>>???mm 
GGG3J99Mz-1171CCDDHJ>>IISZI[[\\E.."YY'CDD 666KLLL||GR  $$W--::hll=Z\>]>]>a>abi>j>j:DEEE||N##D*+===EFFF i0    T3

P
P
P
P|Y/O
P
P
PPP :8999 g  g  gOHHHHHI"
!"
di"
 	&"
 	+	"

 	D"
 	z"
 	J"
 	
"
 	":"
 	c%jj"
 	'c%jj3.>(B(B"
 	 ;"
 	!":"
 	,4"#+ &(0%-!)+3)1%-'/"*
 
"
: 	.;"
< 	8="
> 	vm,?"
@ -!C"
 "
 "
BF!di 	& 	+	
 	D 	#i.. 	J 	  	" 	c%jj 	
 	S.. 	 Y 	": 	d  	Z!" 	,#$ 		% I* i)"U%%jj!$YZ ;":! E" 
B	9%%%u%%%  *H^ ZZH\.-	   H^ 'H\x(((	$*SQVZZky  LV  mx  RV  W  W  fk  l  l  l  m  m  m  m  mr   __main__)r   r   )r   r   r   r   )r   r   r   r   )r   r   r   r   r   r    )r   r   r4   r5   r   r    )r   r    )r   r   r   r5   )rl   r   r   rm   )r[   rm   rt   ru   r   rv   )r   )r   r   rl   r   r   ru   r   r   )r   r   r   rm   r   r   )rl   r   r   r   )r   r   ))
__future__r   r   r   r=   r*   ro   r-  r0  collectionsr   pathlibr   rV   rX   rY   r,  r>  r?  r@  rC  rB  rD  r   r4  r5  r   r   r   r3   r@   r`   rk   rs   r   r   r   r   r   ri  __name__r   r   r   <module>ro     sj   " " " " " "    				 				                 t_``	(<&(?:	&&((//	11	''
''d=>>		[ [ [ [, , , ,( ( ( (   a a a a
8 
8 
8 
8   N N N ND D D DT T T T T
F F F F   6   om om om omd zDFFFFF r   