
    _jC                       d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	m
Z
 d dlmZ  ed          ZdZdZedz  ez  d	z  Zed
z  Zedz  Zedz  Zedz  Zedz  Zedz  Z ed          ZdZdZd9dZd:dZd;dZd<dZd=d Zd>d!Zd?d"Z d@d%Z!dAd*Z"dBdCd.Z#dDd2Z$dEd4Z%dFd6Z&d>d7Z'e(d8k    r e'             dS dS )G    )annotationsN)Counter)PathzD/data/video-pipeline/HaTramAudio/project/025-Nguoi-Giu-Cho-Cuoi-Cungzrun-20260721T232244Z-a0837ee4Levyz
work/storyzcandidate.txtzstory/story-canon.txtzscript/canon.txtzstory/spoken-narration.txtzscript/promotion-report.jsonzlog/story-qa.jsonzlog/story.jsonz(/data/video-pipeline/HaTramAudio/projecti%  i$,  returnstrc                     t           j                             t           j        j                                                                      dd          S )Nz+00:00Z)datetimenowtimezoneutc	isoformatreplace      work/verify_and_promote_story.pyr   r      s<      !2!677AACCKKHVYZZZr   databytesc                N    t          j        |                                           S N)hashlibsha256	hexdigest)r   s    r   	sha_bytesr   !   s    >$))+++r   pathr   c                D    t          |                                           S r   )r   
read_bytes)r   s    r   shar   %   s    T__&&'''r   Nonec                   | j                             dd           |                     d| j        z   dt	          j                     dz             }|                    d          5 }|                    |           |                                 t	          j	        |
                                           d d d            n# 1 swxY w Y   t	          j        ||            d S )NT)parentsexist_ok.z.tmpwb)parentmkdir	with_namenameosgetpidopenwriteflushfsyncfilenor   )r   r   tmphandles       r   atomic_bytesr3   )   s    KdT222
..ty+@ry{{+@+@+@@
A
AC	$ "6T
!!!" " " " " " " " " " " " " " " JsDs   'ACC
Cvaluedictc                |    t          | t          j        |dd          dz                       d                     d S )NF   ensure_asciiindent
utf-8)r3   jsondumpsencode)r   r4   s     r   atomic_jsonr@   3   s=    
5uQGGG$NVVW^__`````r   c                    t          j        t          dz                      d                    t          j        t
          t          d} t          fd|                                 D                       rt          d          t          j        t          dz                      d                    }|
                    d          d	k    s<|
                    d
          t
          k    s|
                    d          t          k    rt          d          t          j        t          dz                      d                    }|
                    di           
                    d          dk    rt          d          d S )Nz.ownership-lock.jsonr<   encoding)
project_idrun_idownerc              3  P   K   | ] \  }}                     |          |k    V  !d S r   )get).0kvlocks      r   	<genexpr>zguard.<locals>.<genexpr>:   s6      
9
91488A;;!
9
9
9
9
9
9r   zownership lock mismatchscript/gate-leases/story.jsonstatusactiverE   rF   z!Story lease is not active/currentscript/project-manifest.jsonstepsstorypendingzStory Gate is not pending)r=   loadsROOT	read_textr)   RUN_IDOWNERanyitemsRuntimeErrorrH   )expectedleasemanifestrL   s      @r   guardr`   7   sX   :t44???QQRRD"i6EJJH

9
9
9
9(8(8
9
9
999 64555J>>IISZI[[\\Eyyh&&%))H*=*=*G*G599U\K]K]afKfKf>???z4"@@KKU\K]]^^H||GR  $$W--::6777 ;:r   c                |    |                                  }|j        |j        |j        |j        t          |           dS )N)deviceinodesizemtime_nsr   )statst_devst_inost_sizest_mtime_nsr   )r   rf   s     r   fingerprintrk   D   s<    99;;D+$d))  r   text	list[str]c                f    t          j        d|                                 t           j                  S )Nu   [0-9A-Za-zÀ-ỹĐđ]+)refindallcasefoldUNICODE)rl   s    r   tokensrs   O   s!    :/"*MMMr   r[   nintset[tuple[str, ...]]c                     t                     k     rt                      S  fdt          t                     z
  dz             D             S )Nc                D    h | ]}t          ||z                      S r   tuple)rI   ir[   rt   s     r   	<setcomp>zngrams.<locals>.<setcomp>V   s-    CCCAE%!A#,CCCr      )lensetrange)r[   rt   s   ``r   ngramsr   S   sP    
5zzA~~uuCCCCCs5zzA~/A)B)BCCCCr   patternflags	list[int]c                    t          j        | |          fdt          |                                d          D             S )Nc                D    g | ]\  }}                     |          |S r   )search)rI   r{   lineregexs      r   
<listcomp>z line_numbers.<locals>.<listcomp>[   s.    SSS'!TT@R@RSASSSr   r}   )ro   compile	enumerate
splitlines)r   rl   r   r   s      @r   line_numbersr   Y   sB    Jw&&ESSSSYt'8'8!<<SSSSr   	text_foldalternativesboolc                :     t           fd|D                       S )Nc              3  D   K   | ]}|                                 v V  d S r   )rq   )rI   itemr   s     r   rM   z!phrase_present.<locals>.<genexpr>_   s0      EEt}})+EEEEEEr   )rZ   )r   r   s   ` r   phrase_presentr   ^   s&    EEEEEEEEEEr   dict[str, bool]c                B   |                                  }d                    |                                 d d                                                    }d                    |                                 dd                                                     }|                    d          }|dk    r
||d          n|}i dt	          |g d          dt	          |g d	          d
t	          |g d          dt	          |g d          dt	          |dg          ot	          |dg          dt	          |dg          ot	          |dg          dt	          |ddg          dt	          |ddg          dt	          |ddg          ot	          |ddg          dt	          |d d!g          ot	          |d"g          d#t	          |d$g          ot	          |d%d&g          d't	          |d(d)g          d*t	          |d+d,g          ot	          |d-g          d.t	          |d/d0g          ot	          |d1d2g          d3t	          |d4d5g          d6t	          |g d7          d8t	          |d9d:g          ot	          |d;d<g           d=t	          |d>d?g          iS )@N i  iPu)   tĩnh an đặt một chiếc giỏ thépr   gate0_hook_early)   bốn giờ mười bảyu   váy cướiu   ảnhu   một lần sửa quá khứgate1_narrator_early)u   tôi là giang vãn đườngu   giang vãn đườngu   vãn đườnggate1_place_early)u   hạnh phúc lýu   thành đôu   tiệm sửa áo cướigate1_goal_early)u   hợp đồngu   giữ tiệmu   giao xong chiếc váyactivation_ruler   u   bốn giờ ba mươithirteen_minute_ruleu   mười ba phútu*   một hành động hoặc một câu nóifour_use_limitu   tối đa bốn lầnu   lần thứ tư	cost_ruleu   mất một ký ức chungu'   mất vĩnh viễn một ký ức chungfirst_cost_verifiedu   khăn tay màu xámu
   khăn xámu   cơn mưa đầu tiênu	   trú mưasecond_cost_verifiedu   tạp dề xámu   tạp dề màu xanhu   bữa sáng đầu tiênthird_cost_verifiedu   dải ruy băngu#   ngày chúng tôi bắt đầu yêuu#   ngày hai người xác nhận yêumidpoint_self_revealu*   chính tôi đã mang chúng đến quầyu&   chính tôi đã dùng tiệm ba lầnfemale_lead_stops_machineu   lấy áo rau   nút trắng vẫn chưa sángu   không giao dịchtruth_and_accountabilityu   tôi làm vì tôi sợu   tôi làm vậy vì tôi sợu   không thể trả ký ứcu   ngừng sửa thêmno_instant_forgivenessu   anh chưa thể tha thứu   không tha thứ ngayconsensual_rebuild)u   mỗi tuần một lầnu   tiếp tục gặp anhu   người đang đi cạnh tôino_memory_resetu   không ký ức nào quay lạiu(   những khoảng trống ấy vẫn đauu   mọi ký ức trở lạiu#   ký ức đã được khôi phụcblank_fourth_receipt_codau"   ô phiếu thứ tư vẫn trốngu   ô thứ tư để trống)rq   joinsplitrfindr   )rl   foldfirstlastclimax_markerclimaxs         r   semantic_checksr   b   s]   ==??DHHTZZ\\$3$'((1133E88DJJLL())2244DJJJKKM%2a%7%7T-..!!TFN5  3I  3I  3I  J  Ju  7B  7B  7B  !C  !C 	^E3r3r3rss 	N52m2m2mnn	
 	>$1L0MNN  CSabf  jA  iB  TC  TC 	t6H5I J J  !T~^b  fR  eS  PT  PT 	.0GIZ/[\\ 	^D+GIr*stt 	~d5JL4YZZ   ]_mnr  vN  P[  u\  `]  `] 	t6GI_5` a a  !Yftuy  }W  |X  gY  gY 	~d5F4GHH   r^\`  dI  Kp  cq  Nr  Nr 	t6b  eM  6N  !O  !O 	$^F^Mm<n%o%o  &c  uC  DJ  Ma  Lb  uc  uc 	#N6<UWv;w$x$x  %L  ~L  MS  Vs  uJ  UK  ~L  ~L 	!.:UWn9o"p"p  	nT  4L  4L  4L  M  M!" 	>$1RT~0  A  A  e  JX  Y]  `|  ~c  _d  Je  Je  Fe#$ 	$^D;_a}:~%%%  r   
list[Path]c                 @   t          t                              d                    } g }t                      }| D ]c}t          |j        v s|                                s%t          |          }||v r9|                    |           |	                    |           d|S )Nz*/story/story-canon.txt)
sortedCORPUS_ROOTglobr   rV   r"   is_filer   addappend)pathsresultseenr   digests        r   corpus_filesr   ~   s    ;##$=>>??EF55D  4<t||~~TT>>dMr   c                   456 t                       t                                          st          d          t	          t                    } t          j        d           t	          t                    }| |k    rt          d          t                                          }|                    d          6t          j
        d6          st          d          6                                }6                                }g }t          |          |                    d          |                    d          6                    d	          t!          d
6          t!          d6          t!          d6t"          j                  t!          d6t"          j                  t!          d6t"          j                  6                    d          t#          j        d6t"          j                  d}t(          t          |          cxk    rt*          k    s'n |                    dt          |          d           |                                D ](\  }}|dk    r|r|                    d|z   |d           )t1          j        t4          dz                      d                    }	g }
g }|	d                                         D ]P\  }}|
                    |           |
                    |d                    |                    |d                    Q6}t;          t=          |
          t          d          D ]>}t#          j        dt#          j         |           ddt          |          z  |          }?i }t;          t=          |                    D ]@}6fdt#          j!        dt#          j         |           d|          D             }|r|||<   A|r|                    d |d           g d!}6fd"|D             }|r|                    d#|d           d"                    t#          j        d$6                    }t          |                                          }|tG          d%t          |                    z  }d&|cxk    rd'k    sn |                    d(|d           tI          6          }|                                D ]"\  }}|s|                    d)|z   d*d           #tK          6          4tM          4d+          }tO          4fd,tQ          tG          d-t          4          d.z
                      D                       }d/ |                                D             }|r |                    d0|d d1         d           g }d-}d2}tS                      D ]} tK          |                     d                    }!tM          |!d+          }"t          ||"z            }#tM          4d3          }$tM          |!d3          }%t          |$|%z            }&|&rt          |$|%z            |&z  nd2}'||#z  }tG          ||'          }|                    | j*        j*        j+        tY          |           |#|'d4           |r|                    d5|d           |d6k    r|                    d7|d           i d8| d9t4          j+        d:tZ          d;t]          |          d<t          |          d=dd>|dt          |          d?|d(|d@|dA|dB|dC|dDt          |          ||t          |          |dEdF|}(|r3t_          t1          j0        |(d*dGH                     tc          d%          t                       t]          |          5t4          dIz  })t1          j        |)                    d                    }*t1          j        t4          dJz                      d                    }+tY          |)          },|+2                    dK          }-|,|-k    rt          dL          |*2                    dMi           2                    dN          dOk    s)|*2                    dPi           2                    dN          rt          dQ          |*2                    dR          d 5fvrt          dS          tf          th          tj          fD ]} tm          | |           to          5fdTtf          th          tj          fD                       rt          dU          g dV}.dW |.D             }/tq                      }0i dXdGd9t4          j+        d:tZ          dYdZd8dd[5d;5dR5d\5dt          |          d]ts          t          |          d^z  dG          d_d`dad dbdcdcdcdcdcdcdcdcdcdcdcdcddded(|dC|dD|(dD         |/|0df}1i dXdGd9t4          j+        d:tZ          dYdZd8ddgtu          t                    d;5dh|d=dditu          tf                    dR5djtu          th                    dktu          tj                    d\5dlddmtu          tv                    dn|/do|0i}2dGt4          j+        tZ          dZd55tu          tf                    5t          |          tu          tx                    tu          tv                    d`d |0dp}3t{          tv          |1           t{          tx          |2           t{          t|          |3           5|*dR<   t          |          |*d<   dqdrddGds|*dt<   |0|*du<   t{          |)|*           t_          t1          j0        dZdt          |          |5|ddvd*w                     d S )xNzcandidate missingZ   z3candidate changed during 90-second stability windowr<   NFCzcandidate is not UTF-8 NFCs   ﻿s   
	z[ \t]+$z^\s*#{1,6}\s+u\   ^\s*(?:chương|phần|cảnh|hồi|chapter|part|scene)(?:\s+(?:\d+|[ivxlcdm]+)\b|\s*[:.\-])uH   ^\s*[\[(](?:nhạc|sfx|hiệu ứng|chuyển cảnh|pause|music|sound)\buI   (?:đăng ký|subscribe|nhấn chuông)\s+kênh|hãy\s+(?:like|chia sẻ)z```z\b(?:TODO|TBD|PLACEHOLDER)\b)
word_countbom
crlf_count	tab_counttrailing_space_linesmarkdown_heading_linesnamed_section_linesstage_direction_lineschannel_cta_linesfenced_code_countplaceholder_hitsr   )checkr4   zhygiene.script/ledger.jsonrB   identity_registryaliases_allowed	forbiddenT)keyreversez(?<!\w)z(?!\w)r   c                h    g | ].}                     d d|                                          dz   /S )r;   r   r}   )countstart)rI   mrl   s     r   r   zmain.<locals>.<listcomp>   s6    uuuq

4AGGII..2uuur   z identity.forbidden_short_aliases)u   Tống Vãn Hàu   Lục Thừa Anu   ga Bắc Hàu   chuyến tàu cuốiu   nhà ga cũu   kho đồ thất lạcu    chìa khóa không có ổ khóau   Tạ Minh Daou   Phó Cảnh Tuyênu   Vân Đìnhu   quyền biểu quyếtc                    i | ]d}|                                                                  v *|                                                     |                                           eS r   )rq   r   )rI   termrl   s     r   
<dictcomp>zmain.<locals>.<dictcomp>   s      B  B  BT^b^k^k^m^mquq~q~  rA  rA  _A  _AD$--////@@  _A  _A  _Ar   revision_1_residueu   [“"]([^”"\n]+)[”"]r}   gffffff?g?dialogue_ratioz	semantic.F   c              3  L   K   | ]}t          ||d z                      V  dS )r   Nry   )rI   r{   candidate_tokenss     r   rM   zmain.<locals>.<genexpr>   s9      oo!e$4QqtV$<==oooooor   r      c                N    g | ]"\  }}|d k    d                     |          |f#S )r}   r   )r   )rI   rJ   rK   s      r   r   zmain.<locals>.<listcomp>   s7    VVVTQPQTUPUPU!a(PUPUPUr   z&originality.internal_12gram_repetition   g           )projectr   exact_12gramjaccard_5gramzoriginality.corpus_exact_12gramg{Gz?zoriginality.max_5gram_jaccardverifiedrD   rE   candidate_sha256candidate_bytesstable_secondsrk   dialogue_wordshygiene
alias_hitsrevision_1_residue_hitsr   originality)corpus_countexact_12gram_overlap_totalmax_5gram_jaccardinternal_repeated_12gram_countcorpusfailuresr7   r8   rQ   rN   manifest_sha256_at_acquirez.manifest changed since Story lease acquisitionrR   rS   rT   gate_eventsz'Story Gate changed before canon bindingcanon_sha256z(manifest already binds a different canonc              3  >   K   | ]}t          |          k    V  d S r   )r   )rI   r   
canon_hashs     r   rM   zmain.<locals>.<genexpr>   s.      
P
Pt3t99
"
P
P
P
P
P
Pr   zraw-byte promotion mismatch)zscript/creative-options.jsonzscript/story-brief.jsonzscript/outline.jsonr   z$script/story-authority-revision.jsonc                >    i | ]}|t          t          |z            S r   )r   rV   )rI   rels     r   r   zmain.<locals>.<dictcomp>$  s$    HHHS__HHHr   schema_versionrO   	completedsource_canon_sha256spoken_narration_sha256estimated_minutes_at_231_wpm   story_family_primaryzHigh-conceptstory_family_secondarygatespassedabsent)global_coherence_text_review
gate0_hookgate1_comprehensiondialogueidentity_aliastimeline_knowledge_propsconsent_power_balancenarration_hygieneoriginality_internal_12gramoriginality_corpus_12gramfamily_primary_payoff#high_concept_rule_cost_limit_payoffthird_family_drift)authority_sha256completed_atcandidate_pathcandidate_fingerprint
canon_pathscript_canon_pathspoken_narration_pathraw_byte_identicalstory_qa_pathr  r  )r  rD   rE   rO   r   r  r   artifact_pathartifact_sha256r   promotion_report_pathr!  r  r  r  u   Cưới trước yêu sauu   Hào môn / liên hôn)primary	secondarylockedrevisionstory_family
updated_at)rO   r   r   r   r   r   manifest_canon_bound)r9   )?r`   	CANDIDATEr   r\   rk   timesleepr   decodeunicodedatais_normalizedr   r   r~   
startswithr   r   ro   Irp   	MIN_WORDS	MAX_WORDSr   r[   r=   rU   rV   rW   extendr   r   subescapefinditerr   maxr   rs   r   r   r   r   r&   r)   r   rX   r   printr>   
SystemExitrH   CANONSCRIPT_CANON	NARRATIONr3   rZ   r   roundr   
QA_RECEIPT	PROMOTIONr@   STORY_RECEIPT)7beforeafterrawwordslinesr   r   r   r4   ledgerallowed_namesforbidden_aliases	full_nameinfomaskedr)   r   aliashitsresidue_termsresidue_hitsdialogue_textr   r   semanticr
  cand12internal_countsinternal_repeatscorpus_resultsexact_overlap_totalmax_jaccardr   
old_tokensold12exactcand5old5unionjaccardr   manifest_pathr_   r^   current_manifest_hashexpected_manifest_hashauthority_filesauthority_hashesr  qa	promotionrS   r   r   rl   s7                                                       @@@r   mainri     s   	GGG 0.///##FJrNNN	""EPQQQ



 
 C::gD$UD11 97888JJLLEOOEH%jj~~o..ii((ZZ%% ,Z > >"./?"F"F+  -L  NR  TV  TX   Y   Y!-.y{  BD  BF  "G  "G)*vx|  A  C  D  D!ZZ..J'FbdSS G E

////i////,UDDEEEmmoo I I
U,5OOj3&6GGHHHZ 44???QQRRFM!"56<<>> 4 4	4Y'''T"34555  k!23333Fs=))sDAAA U U:29T??:::C#d))OVTTJ-..// % %uuuuBKHkSUS\]bScScHkHkHkms<t<tuuu 	% $Ju \"DzZZ[[[  M
 B  B  B  B]  B  B  BL P"6NNOOO HHRZ(CTJJKKM,,..//N#c!SZZ&8&88N>))))T))))"2^LLMMMt$$H~~'' J JV 	JOOkC&7%HHIIId||$b))FoooouSQRTWXhTiTijlTlMmMmGnGnoooooOVV_5J5J5L5LVVV m"JUefigifiUjkklllNK 
J 
JDNNGN<<==
z2&&FUN##'++j!$$EDL!!/4=#edl##e++#u$+w//$+*<*ASQUYYhm  AH  I  I  	J  	J  	J  	J d"CNabbcccT"AKXXYYYLdi 	& 	IcNN	
 	3s88 	" 	u 	c%jj 	. 	. 	7 	j 	"< 	8 	//*=!,.12B.C.C$
 
, 	H-F0  djeA>>>???mm 
GGG3J99Mz-1171CCDDHJ>>IISZI[[\\E.."YY'CDD 666KLLL||GR  $$W--::hll=Z\>]>]>a>abi>j>j:DEEE||N##D*+===EFFF i0    T3

P
P
P
P|Y/O
P
P
PPP :8999 g  g  gOHHHHHI"
!"
di"
 	&"
 	+	"

 	D"
 	z"
 	J"
 	
"
 	":"
 	c%jj"
 	'c%jj3.>(B(B"
 	"
 	!$"
 	,4"#+ &(0%-!)+3)1%-3;"*
 
"
: 	.;"
< 	8="
> 	vm,?"
@ -!C"
 "
 "
BF!di 	& 	+	
 	D 	#i.. 	J 	  	" 	c%jj 	
 	S.. 	 Y 	": 	d  	Z!" 	,#$ 		% I* i)"U%%jj!$YZ ."&! E" 
B	9%%%u%%%  *H^ ZZH\.-	   H^ 'H\x(((	$*SQVZZky  LV  mx  RV  W  W  fk  l  l  l  m  m  m  m  mr   __main__)r   r   )r   r   r   r   )r   r   r   r   )r   r   r   r   r   r    )r   r   r4   r5   r   r    )r   r    )r   r   r   r5   )rl   r   r   rm   )r[   rm   rt   ru   r   rv   )r   )r   r   rl   r   r   ru   r   r   )r   r   r   rm   r   r   )rl   r   r   r   )r   r   ))
__future__r   r   r   r=   r*   ro   r-  r0  collectionsr   pathlibr   rV   rX   rY   r,  r=  r>  r?  rB  rA  rC  r   r4  r5  r   r   r   r3   r@   r`   rk   rs   r   r   r   r   r   ri  __name__r   r   r   <module>ro     sj   " " " " " "    				 				                 tRSS	(<&(?:	&&((//	11	''
''d=>>		[ [ [ [, , , ,( ( ( (   a a a a
8 
8 
8 
8   N N N ND D D DT T T T T
F F F F   8   pm pm pm pmf zDFFFFF r   