
    `j1B                       d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	m
Z
 d dlmZ  ed          ZdZdZedz  ez  d	z  Zed
z  Zedz  Zedz  Zedz  Zedz  Zedz  Z ed          ZdZdZd9dZd:dZd;dZd<dZd=d Zd>d!Zd?d"Z d@d%Z!dAd*Z"dBdCd.Z#dDd2Z$dEd4Z%dFd6Z&d>d7Z'e(d8k    r e'             dS dS )G    )annotationsN)Counter)PathzO/data/video-pipeline/HaTramAudio/project/029-Chiec-O-De-Lai-O-Tram-Xe-Buyt-Cuoizrun-20260722T162951Z-87a939adLevyz
work/storyzcandidate.txtzstory/story-canon.txtzscript/canon.txtzstory/spoken-narration.txtzscript/promotion-report.jsonzlog/story-qa.jsonzlog/story.jsonz(/data/video-pipeline/HaTramAudio/projecti%  i$,  returnstrc                     t           j                             t           j        j                                                                      dd          S )Nz+00:00Z)datetimenowtimezoneutc	isoformatreplace      work/verify_and_promote_story.pyr   r      s<      !2!677AACCKKHVYZZZr   databytesc                N    t          j        |                                           S N)hashlibsha256	hexdigest)r   s    r   	sha_bytesr   !   s    >$))+++r   pathr   c                D    t          |                                           S r   )r   
read_bytes)r   s    r   shar   %   s    T__&&'''r   Nonec                   | j                             dd           |                     d| j        z   dt	          j                     dz             }|                    d          5 }|                    |           |                                 t	          j	        |
                                           d d d            n# 1 swxY w Y   t	          j        ||            d S )NT)parentsexist_ok.z.tmpwb)parentmkdir	with_namenameosgetpidopenwriteflushfsyncfilenor   )r   r   tmphandles       r   atomic_bytesr3   )   s    KdT222
..ty+@ry{{+@+@+@@
A
AC	$ "6T
!!!" " " " " " " " " " " " " " " JsDs   'ACC
Cvaluedictc                |    t          | t          j        |dd          dz                       d                     d S )NF   ensure_asciiindent
utf-8)r3   jsondumpsencode)r   r4   s     r   atomic_jsonr@   3   s=    
5uQGGG$NVVW^__`````r   c                    t          j        t          dz                      d                    t          j        t
          t          d} t          fd|                                 D                       rt          d          t          j        t          dz                      d                    }|
                    d          d	k    s<|
                    d
          t
          k    s|
                    d          t          k    rt          d          t          j        t          dz                      d                    }|
                    di           
                    d          dk    rt          d          d S )Nz.ownership-lock.jsonr<   encoding)
project_idrun_idownerc              3  P   K   | ] \  }}                     |          |k    V  !d S r   )get).0kvlocks      r   	<genexpr>zguard.<locals>.<genexpr>:   s6      
9
91488A;;!
9
9
9
9
9
9r   zownership lock mismatchscript/gate-leases/story.jsonstatusactiverE   rF   z!Story lease is not active/currentscript/project-manifest.jsonstepsstorypendingzStory Gate is not pending)r=   loadsROOT	read_textr)   RUN_IDOWNERanyitemsRuntimeErrorrH   )expectedleasemanifestrL   s      @r   guardr`   7   sX   :t44???QQRRD"i6EJJH

9
9
9
9(8(8
9
9
999 64555J>>IISZI[[\\Eyyh&&%))H*=*=*G*G599U\K]K]afKfKf>???z4"@@KKU\K]]^^H||GR  $$W--::6777 ;:r   c                |    |                                  }|j        |j        |j        |j        t          |           dS )N)deviceinodesizemtime_nsr   )statst_devst_inost_sizest_mtime_nsr   )r   rf   s     r   fingerprintrk   D   s<    99;;D+$d))  r   text	list[str]c                f    t          j        d|                                 t           j                  S )Nu   [0-9A-Za-zÀ-ỹĐđ]+)refindallcasefoldUNICODE)rl   s    r   tokensrs   O   s!    :/"*MMMr   r[   nintset[tuple[str, ...]]c                     t                     k     rt                      S  fdt          t                     z
  dz             D             S )Nc                D    h | ]}t          ||z                      S r   tuple)rI   ir[   rt   s     r   	<setcomp>zngrams.<locals>.<setcomp>V   s-    CCCAE%!A#,CCCr      )lensetrange)r[   rt   s   ``r   ngramsr   S   sP    
5zzA~~uuCCCCCs5zzA~/A)B)BCCCCr   patternflags	list[int]c                    t          j        | |          fdt          |                                d          D             S )Nc                D    g | ]\  }}                     |          |S r   )search)rI   r{   lineregexs      r   
<listcomp>z line_numbers.<locals>.<listcomp>[   s.    SSS'!TT@R@RSASSSr   r}   )ro   compile	enumerate
splitlines)r   rl   r   r   s      @r   line_numbersr   Y   sB    Jw&&ESSSSYt'8'8!<<SSSSr   	text_foldalternativesboolc                :     t           fd|D                       S )Nc              3  D   K   | ]}|                                 v V  d S r   )rq   )rI   itemr   s     r   rM   z!phrase_present.<locals>.<genexpr>_   s0      EEt}})+EEEEEEr   )rZ   )r   r   s   ` r   phrase_presentr   ^   s&    EEEEEEEEEEr   dict[str, bool]c                   |                                  }d                    |                                 d d                                                    }d                    |                                 dd                                                     }|                    d          }|dk    r
||d          n|}i dt	          |dg          ot	          |dg          d	t	          |d
dg          dt	          |ddg          ot	          |dg          dt	          |dg          ot	          |dg          dt	          |dg          ot	          |dg          dt	          |dg          dt	          |dg          ot	          |d
g          dt	          |dg          ot	          |dg          dt	          |dg          ot	          |dg          d t	          |d!g          ot	          |d"g          d#t	          |d$g          ot	          |d%g          d&t	          |dg          ot	          |d'g          d(t	          |d)g          ot	          |d*g          d+t	          |dg          ot	          |d
g          d,t	          |d-g          ot	          |d.g          d/t	          |d0g          ot	          |d1g          d2t	          |d3g          ot	          |d4g          S )5N i  i$u9   đồng hồ trên cánh gà bắt đầu đếm ngượcr   gate0_hook_refusalu"   tôi sẽ không đọc bản nàyu   để họ tự nói tên mìnhgate1_narrator_earlyu   tống vãn nghiu	   vãn nghigate1_place_and_stakesu   lễ tốt nghiệpu   hội trườngu
   danh sáchaccessibility_agencyu   để tôi tự nóiu   hà dịch thanhconsent_contractu$   bạn có quyền không kể lý doconsentmultiple_valid_pathsu3   một hệ thống có nhiều đường hợp lệmidpoint_empty_seatu   ghế số hai mươi bảymutual_vulnerabilityu   tôi ghi nhầm giờu   mẹ tôi mổ cột sốngfemale_accountabilityu   tôi đã saiu*   không trao cho tôi quyền đại diệnconsent_withdrawal_respectedu!   mười hai người rút consentu   không bị đưa trở lạimature_redesignu   từng người tự nói tênu   không kể lý lịchfemale_climax_choiceu$   tôi từ chối dùng giọng mìnhmale_climax_costu   anh tắt đồng hồu    chịu toàn bộ trách nhiệmself_naming_payoffprocedural_consequenceu'   nhà trường lập biểu mẫu mớiu'   không ai phải nộp lý do cá nhânno_instant_romanceu   một tuần sauu    người tôi muốn tìm hiểuself_naming_codau   tự cầm micrôu   tôi ở đây)rq   joinsplitrfindr   )rl   foldfirstlastclimax_markerclimaxs         r   semantic_checksr   b   sq   ==??DHHTZZ\\$3$'((1133E88DJJLL())2244DJJZ[[M%2a%7%7T-..!!TFnU5Y4Z[[  [`not  xY  wZ  a[  a[u7I;6W X X 	!.9NPa8b"c"c  #Nhvw|  @L  M  iN  iN 	t6L5M N N }Sabfi{h|S}S}	
 	N42X1YZZ  A_mnru~t  `A  `A 	t6k5l m m 	~d5R4STT   DYghl  pB  oC  ZD  ZD 	t6M5N O O  !JTbcg  kH  jI  UJ  UJ 	 6G!H!H  "R^\`  dP  cQ  NR  NR 	't>a=b(c(c  )`hvw{  ^  ~_  i`  i` 	>$1Q0RSS  IXfgk  oG  nH  YI  YI 	v8\7] ^ ^  !dcqrx  |b  {c  dd  dd 	N64L3MNN  PSabh  lN  kO  TP  TP 	nV6H5IJJ{~^dgyfzO{O{ 	!.8a7b"c"c  #jhvw{  h  ~i  ij  ij  	nT4F3GHH  H^\`  dF  cG  NH  NH!" 	N42E1FGGtN[_brasLtLt# r   
list[Path]c                    t          t          t                              d                    t          t                              d                    z            } g }t                      }| D ]c}t          |j        v s|                                s%t          |          }||v r9|                    |           |	                    |           d|S )Nz*/story/story-canon.txtz*/story/canon.txt)
sortedr   CORPUS_ROOTglobrV   r"   is_filer   addappend)pathsresultseenr   digests        r   corpus_filesr   }   s    3{''(ABBCCc+JZJZ[nJoJoFpFppqqEF55D  4<t||~~TT>>dMr   c                   456 t                       t                                          st          d          t	          t                    } t          j        d           t	          t                    }| |k    rt          d          t                                          }|                    d          6t          j
        d6          st          d          6                                }6                                }g }t          |          |                    d          |                    d          6                    d	          t!          d
6          t!          d6          t!          d6t"          j                  t!          d6t"          j                  t!          d6t"          j                  6                    d          t#          j        d6t"          j                  d}t(          t          |          cxk    rt*          k    s'n |                    dt          |          d           |                                D ](\  }}|dk    r|r|                    d|z   |d           )t1          j        t4          dz                      d                    }	g }
g }|	d                                         D ]P\  }}|
                    |           |
                    |d                    |                    |d                    Q6}t;          t=          |
          t          d          D ]>}t#          j        dt#          j         |           ddt          |          z  |          }?i }t;          t=          |                    D ]@}6fdt#          j!        dt#          j         |           d|          D             }|r|||<   A|r|                    d |d           |	d!         }6fd"|D             }|r|                    d#|d           d"                    t#          j        d$6                    }t          |                                          }|tG          d%t          |                    z  }d&|cxk    rd'k    sn |                    d(|d           tI          6          }|                                D ]"\  }}|s|                    d)|z   d*d           #tK          6          4tM          4d+          }tO          4fd,tQ          tG          d-t          4          d.z
                      D                       }d/ |                                D             }|r |                    d0|d d1         d           g }d-}d2}tS                      D ]} tK          |                     d                    }!tM          |!d+          }"t          ||"z            }#tM          4d3          }$tM          |!d3          }%t          |$|%z            }&|&rt          |$|%z            |&z  nd2}'||#z  }tG          ||'          }|                    | j*        j*        j+        tY          |           |#|'d4           |r|                    d5|d           |d6k    r|                    d7|d           i d8| d9t4          j+        d:tZ          d;t]          |          d<t          |          d=dd>|dt          |          d?|d(|d@|dA|dB|dC|dDt          |          ||t          |          |dEdF|}(|r3t_          t1          j0        |(d*dGH                     tc          d%          t                       t]          |          5t4          dIz  })t1          j        |)                    d                    }*t1          j        t4          dJz                      d                    }+tY          |)          },|+2                    dK          }-|,|-k    rt          dL          |*2                    dMi           2                    dN          dOk    s)|*2                    dPi           2                    dN          rt          dQ          |*2                    dR          d 5fvrt          dS          tf          th          tj          fD ]} tm          | |           to          5fdTtf          th          tj          fD                       rt          dU          g dV}.dW |.D             }/tq                      }0i dXdGd9t4          j+        d:tZ          dYdZd8dd[5d;5dR5d\5dt          |          d]ts          t          |          d^z  dG          d_d`dadbdcdddddddddddddddddddddddddedfd(|dC|dD|(dD         |/|0dg}1i dXdGd9t4          j+        d:tZ          dYdZd8ddhtu          t                    d;5di|d=ddjtu          tf                    dR5dktu          th                    dltu          tj                    d\5dmddntu          tv                    do|/dp|0i}2dGt4          j+        tZ          dZd55tu          tf                    5t          |          tu          tx                    tu          tv                    d`db|0dq}3t{          tv          |1           t{          tx          |2           t{          t|          |3           5|*dR<   t          |          |*d<   d`dbddrds|*dt<   |0|*du<   t{          |)|*           t_          t1          j0        dZdt          |          |5|ddvd*w                     d S )xNzcandidate missingZ   z3candidate changed during 90-second stability windowr<   NFCzcandidate is not UTF-8 NFCs   ﻿s   
	z[ \t]+$z^\s*#{1,6}\s+u\   ^\s*(?:chương|phần|cảnh|hồi|chapter|part|scene)(?:\s+(?:\d+|[ivxlcdm]+)\b|\s*[:.\-])uH   ^\s*[\[(](?:nhạc|sfx|hiệu ứng|chuyển cảnh|pause|music|sound)\buI   (?:đăng ký|subscribe|nhấn chuông)\s+kênh|hãy\s+(?:like|chia sẻ)z```z\b(?:TODO|TBD|PLACEHOLDER)\b)
word_countbom
crlf_count	tab_counttrailing_space_linesmarkdown_heading_linesnamed_section_linesstage_direction_lineschannel_cta_linesfenced_code_countplaceholder_hitsr   )checkr4   zhygiene.script/ledger.jsonrB   identity_registryaliases_allowed	forbiddenT)keyreversez(?<!\w)z(?!\w)r   c                h    g | ].}                     d d|                                          dz   /S )r;   r   r}   )countstart)rI   mrl   s     r   r   zmain.<locals>.<listcomp>   s6    uuuq

4AGGII..2uuur   z identity.forbidden_short_aliasesforbidden_residuec                    i | ]d}|                                                                  v *|                                                     |                                           eS r   )rq   r   )rI   termrl   s     r   
<dictcomp>zmain.<locals>.<dictcomp>   s      B  B  BT^b^k^k^m^mquq~q~  rA  rA  _A  _AD$--////@@  _A  _A  _Ar   revision_1_residueu   [“"]([^”"\n]+)[”"]r}   gffffff?g?dialogue_ratioz	semantic.F   c              3  L   K   | ]}t          ||d z                      V  dS )r   Nry   )rI   r{   candidate_tokenss     r   rM   zmain.<locals>.<genexpr>   s9      oo!e$4QqtV$<==oooooor   r      c                N    g | ]"\  }}|d k    d                     |          |f#S )r}   r   )r   )rI   rJ   rK   s      r   r   zmain.<locals>.<listcomp>   s7    VVVTQPQTUPUPU!a(PUPUPUr   z&originality.internal_12gram_repetition   g           )projectr   exact_12gramjaccard_5gramzoriginality.corpus_exact_12gramg{Gz?zoriginality.max_5gram_jaccardverifiedrD   rE   candidate_sha256candidate_bytesstable_secondsrk   dialogue_wordshygiene
alias_hitsrevision_1_residue_hitsr   originality)corpus_countexact_12gram_overlap_totalmax_5gram_jaccardinternal_repeated_12gram_countcorpusfailuresr7   r8   rQ   rN   manifest_sha256_at_acquirez.manifest changed since Story lease acquisitionrR   rS   rT   gate_eventsz'Story Gate changed before canon bindingcanon_sha256z(manifest already binds a different canonc              3  >   K   | ]}t          |          k    V  d S r   )r   )rI   r   
canon_hashs     r   rM   zmain.<locals>.<genexpr>  s.      
P
Pt3t99
"
P
P
P
P
P
Pr   zraw-byte promotion mismatch)zscript/creative-options.jsonzscript/story-brief.jsonzscript/outline.jsonr   z$script/story-authority-revision.jsonc                >    i | ]}|t          t          |z            S r   )r   rV   )rI   rels     r   r   zmain.<locals>.<dictcomp>  s$    HHHS__HHHr   schema_versionrO   	completedsource_canon_sha256spoken_narration_sha256estimated_minutes_at_231_wpm   story_family_primaryu)   Ngôn tình học đường / thanh xuânstory_family_secondaryu   Chữa lànhgatespassedabsent)global_coherence_text_review
gate0_hookgate1_comprehensiondialogueidentity_aliastimeline_knowledge_propsconsent_power_balancenarration_hygieneoriginality_internal_12gramoriginality_corpus_12gramfamily_primary_payoffsecondary_healing_payoffthird_family_drift)authority_sha256completed_atcandidate_pathcandidate_fingerprint
canon_pathscript_canon_pathspoken_narration_pathraw_byte_identicalstory_qa_pathr  r  )r  rD   rE   rO   r   r  r   artifact_pathartifact_sha256r   promotion_report_pathr!  r  r  r     )primary	secondarylockedrevisionstory_family
updated_at)rO   r   r   r   r   r   manifest_canon_bound)r9   )?r`   	CANDIDATEr   r\   rk   timesleepr   decodeunicodedatais_normalizedr   r   r~   
startswithr   r   ro   Irp   	MIN_WORDS	MAX_WORDSr   r[   r=   rU   rV   rW   extendr   r   subescapefinditerr   maxr   rs   r   r   r   r   r&   r)   r   rX   r   printr>   
SystemExitrH   CANONSCRIPT_CANON	NARRATIONr3   rZ   r   roundr   
QA_RECEIPT	PROMOTIONr@   STORY_RECEIPT)7beforeafterrawwordslinesr   r   r   r4   ledgerallowed_namesforbidden_aliases	full_nameinfomaskedr)   r   aliashitsresidue_termsresidue_hitsdialogue_textr   r   semanticr
  cand12internal_countsinternal_repeatscorpus_resultsexact_overlap_totalmax_jaccardr   
old_tokensold12exactcand5old5unionjaccardr   manifest_pathr_   r^   current_manifest_hashexpected_manifest_hashauthority_filesauthority_hashesr  qa	promotionrS   r   r   rl   s7                                                       @@@r   mainrj     s   	GGG 0.///##FJrNNN	""EPQQQ



 
 C::gD$UD11 97888JJLLEOOEH%jj~~o..ii((ZZ%% ,Z > >"./?"F"F+  -L  NR  TV  TX   Y   Y!-.y{  BD  BF  "G  "G)*vx|  A  C  D  D!ZZ..J'FbdSS G E

////i////,UDDEEEmmoo I I
U,5OOj3&6GGHHHZ 44???QQRRFM!"56<<>> 4 4	4Y'''T"34555  k!23333Fs=))sDAAA U U:29T??:::C#d))OVTTJ-..// % %uuuuBKHkSUS\]bScScHkHkHkms<t<tuuu 	% $Ju \"DzZZ[[[./M B  B  B  B]  B  B  BL P"6NNOOO HHRZ(CTJJKKM,,..//N#c!SZZ&8&88N>))))T))))"2^LLMMMt$$H~~'' J JV 	JOOkC&7%HHIIId||$b))FoooouSQRTWXhTiTijlTlMmMmGnGnoooooOVV_5J5J5L5LVVV m"JUefigifiUjkklllNK 
J 
JDNNGN<<==
z2&&FUN##'++j!$$EDL!!/4=#edl##e++#u$+w//$+*<*ASQUYYhm  AH  I  I  	J  	J  	J  	J d"CNabbcccT"AKXXYYYLdi 	& 	IcNN	
 	3s88 	" 	u 	c%jj 	. 	. 	7 	j 	"< 	8 	//*=!,.12B.C.C$
 
, 	H-F0  djeA>>>???mm 
GGG3J99Mz-1171CCDDHJ>>IISZI[[\\E.."YY'CDD 666KLLL||GR  $$W--::hll=Z\>]>]>a>abi>j>j:DEEE||N##D*+===EFFF i0    T3

P
P
P
P|Y/O
P
P
PPP :8999 g  g  gOHHHHHI"
!"
di"
 	&"
 	+	"

 	D"
 	z"
 	J"
 	
"
 	":"
 	c%jj"
 	'c%jj3.>(B(B"
 	 K"
 	!."
 	,4"#+ &(0%-!)+3)1%-(0"*
 
"
: 	.;"
< 	8="
> 	vm,?"
@ -!C"
 "
 "
BF!di 	& 	+	
 	D 	#i.. 	J 	  	" 	c%jj 	
 	S.. 	 Y 	": 	d  	Z!" 	,#$ 		% I* i)"U%%jj!$YZ K"0! E" 
B	9%%%u%%%  *H^ ZZH\>#	   H^ 'H\x(((	$*SQVZZky  LV  mx  RV  W  W  fk  l  l  l  m  m  m  m  mr   __main__)r   r   )r   r   r   r   )r   r   r   r   )r   r   r   r   r   r    )r   r   r4   r5   r   r    )r   r    )r   r   r   r5   )rl   r   r   rm   )r[   rm   rt   ru   r   rv   )r   )r   r   rl   r   r   ru   r   r   )r   r   r   rm   r   r   )rl   r   r   r   )r   r   ))
__future__r   r   r   r=   r*   ro   r.  r1  collectionsr   pathlibr   rV   rX   rY   r-  r>  r?  r@  rC  rB  rD  r   r5  r6  r   r   r   r3   r@   r`   rk   rs   r   r   r   r   r   rj  __name__r   r   r   <module>rp     sj   " " " " " "    				 				                 t]^^	(<&(?:	&&((//	11	''
''d=>>		[ [ [ [, , , ,( ( ( (   a a a a
8 
8 
8 
8   N N N ND D D DT T T T T
F F F F   6   lm lm lm lm^ zDFFFFF r   