
    3jL%                         d dl Z d dlZd dlZd dlZd dlZd dlmZ  ed          Zedz  Zedz  Z	edz  Z
edz  Zedz  Zd	Zd
ZdZ ej        dej                  Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zedk    r e             dS dS )    N)PathzL/data/video-pipeline/Youtube-Video-Maker/Project/10062026-001-Tran-Quoc-Tuanzscript/scenes.jsonz)subtitles/master_narration_asr_words.jsonzsubtitles/master_subtitles.srtz'subtitles/master_subtitles_mapping.jsonz*logs/step11_subtitle_mapping_manifest.jsonN   g      @gffffff?z\w+c                 R    t          t          j        d          | z   d           d S )Nz%Y-%m-%d %H:%M:%S T)flush)printtimestrftime)msgs    q/data/video-pipeline/Youtube-Video-Maker/Project/10062026-001-Tran-Quoc-Tuan/scripts_run_step_11_map_subtitles.pylogr      s+    	$-,
-
-
34@@@@@@    c                     |                                                      dd          } d                    d t          j        d|           D                       } t          j        dd|           S )Nu   đd c              3   J   K   | ]}t          j        |          d k    |V  dS )MnN)unicodedatacategory).0chs     r   	<genexpr>znorm_token.<locals>.<genexpr>   s9      gg"kFZ[]F^F^bfFfFf2FfFfFfFfggr   NFDz
[^a-z0-9]+)lowerreplacejoinr   	normalizeresubtexts    r   
norm_tokenr!      s`    ::<<c**D77gg 5eT B BgggggD6-T***r   c                 J    d t                               |           D             S )Nc                     g | ]`}|                     d           t          |                     d                     |                                |                                daS )r   )r    norm
start_charend_char)groupr!   startend)r   ms     r   
<listcomp>z'tokenize_with_spans.<locals>.<listcomp>!   sh      Q  Q  QstQWWQZZAGGAJJ)?)?qwwyyfgfkfkfmfmnn  Q  Q  Qr   )TOKEN_REfinditerr   s    r   tokenize_with_spansr.       sP     Q  Q  yA  yJ  yJ  KO  yP  yP  Q  Q  Q  Qr   c                    g }|                      dg           pg D ]}|                     dg           pg D ]}|                     d          |                     d          -|                    |                     dd                                          t          |                     dd                    t	          |d                   t	          |d                   d           d |D             S )	Nsegmentswordsr(   r)   wordr   )r    r$   r(   r)   c                 "    g | ]}|d          
|S r$    r   ws     r   r+   z%collect_asr_words.<locals>.<listcomp>+   s!    ***!&	*A***r   )getappendstripr!   float)docr1   segr7   s       r   collect_asr_wordsr>   $   s-   Ewwz2&&," Y Y"%%+ 	Y 	YAuuW~~%u)=LL!%%"3"3"9"9";";ZPQPUPUV\^`PaPaEbEbmrstu|s}m~m~  HM  NO  PU  NV  HW  HW  X  X  Y  Y  Y  Y	Y +*u****r   c                    g }g }d}| D ]}|r|                     d           |dz  }|}|                    dd                                          }|                     |           |t          |          z  }|                     |d         |                    dd          ||d           d                    |          |fS )	Nr   
   	narrationr   scene_id
scene_type)rC   rD   r%   r&   )r9   r8   r:   lenr   )sceneschunksscene_rangescursorscener(   rB   s          r   build_original_textrK   .   s    FLF Q Q 	MM$aKFIIk2..4466	i   #i.. z):%))T`bdJeJeuz  IO  P  P  	Q  	Q  	Q  	Q776??L((r   c                    d | D             }d |D             }t          j        d ||d          }d gt          |           z  }d}|                                D ]\  }}}	}
}|dk    rYt	          t          ||	          t          |
|                    D ]+\  }}||         d         ||         d         d	d
||<   |dz  },h|dk    r|	|z
  ||
z
  k    rt	          t          ||	          t          |
|                    D ]e\  }}t          j        d ||         ||                                                   }|dk    r&||         d         ||         d         dd
||<   |dz  }fd t          |          D             }|st          d          t          |           D ]\  }|         rfd|D             }fd|D             }|r|d         nd }|r|d         nd }|?|=||z
  }|z
  |z  }||         d         |||         d         ||         d         z
  z  z   }n?|||         d         d|z
  z  z   }n%t          d||         d         d|z
  z  z
            }||dz   dd
|<   ||fS )Nc                     g | ]
}|d          S r4   r5   )r   ts     r   r+   z align_tokens.<locals>.<listcomp>?   s    000q6000r   c                     g | ]
}|d          S r4   r5   r6   s     r   r+   z align_tokens.<locals>.<listcomp>@   s    ---a&	---r   F)autojunkr   equalr(   r)   matched)r(   r)   sourcerA   r   g
ףp=
?fuzzyc                     g | ]	\  }}||
S r5   r5   )r   ivs      r   r+   z align_tokens.<locals>.<listcomp>Q   s!    ???41aQ?Q???r   z:No aligned tokens between original narration and ASR wordsc                      g | ]
}|k     |S r5   r5   r   xrV   s     r   r+   z align_tokens.<locals>.<listcomp>W       333qQUUUUUr   c                      g | ]
}|k    |S r5   r5   rY   s     r   r+   z align_tokens.<locals>.<listcomp>X   r[   r   g      ?        皙?interpolated)
difflibSequenceMatcherrE   get_opcodesziprangeratio	enumerateRuntimeErrormax)orig_tokens	asr_words	orig_normasr_normsmtoken_timesrR   tagi1i2j1j2oiajrf   timed_indicestokprevsnextsprev_inext_ispanfracr(   rV   s                           @r   align_tokensr   >   s   00K000I--9---H		 y(U	K	K	KB&3{+++KG!~~// ! !RR'>>eBmmU2r]];;  B,5bM',B9UW=Y^K_kt"u"uB1 I27R"8"8eBmmU2r]];; ! !B/imXb\RRXXZZD==09"g0FyY[}]bOcov&w&wKOqLG??9[#9#9???M YWXXXK(( X X3q> 	3333M3333333M333#-r",q&"4F?DJ$&D'.V9LW9UXcdjXklqXr9r1ssEE'.V1DDEE[09DFQJ<OOPPE#(WWAr   c           
          g }d}d}t          |          D ]\  }}| ||         d         |d                                                  }||         d         ||         d         z
  }	| |d         |d         dz            }
|
dv }|
dk    }|
d	k    ot          |          d
k    }t          |          t          k    }|	t          k    ot          |          dk    }|t          |          dz
  k    s|s	|s|s|s|r||         d         }|d         }|t          |           k     r,| |         dv r"|dz  }|t          |           k     r
| |         dv "t          j        dd| ||                                                   }||         d         }||         d         }||z
  t          k     r
|t          z   }|                    t          |          dz   t          |d          t          |d          |||d           |dz   }|}d |D             S )Nr   r%   r&   r)   r(   rA   z.?!;:r@   ,&      z.,?!;:z\s+    )indexr(   r)   r    start_token	end_tokenc                 "    g | ]}|d          
|S r   r5   )r   cs     r   r+   zsplit_cues.<locals>.<listcomp>   s!    )))!qy)A)))r   )
rg   r:   rE   MAX_CUE_CHARSMAX_CUE_SECONDSr   r   MIN_CUE_SECONDSr9   round)r    tokensro   cuescur_start_tokenlast_break_tokenrV   rx   snippetcue_duration	next_charpunctuation_breakscene_boundary_breakcomma_breaktoo_longtoo_slowr%   r&   cue_textr(   r)   s                        r   
split_cuesr   g   s[   DOF## ! !3vo.|<S_LMSSUU"1~e,{?/KG/TTZZ1)<<=	%0(D03&=3w<<2+=w<<=0?2Is7||r7IFa#7;LP[_gks0>J:HSYY&&4>X+E+EA SYY&&4>X+E+Evfc4
80C+DEEKKMMH09Ea.'CU{_,,o-KK#d))a-%q//RWX[]^R_R_iq  CR  ab  c  c  d  d  d!eO ))t))))r   c           	      :   t          |           D ]	\  }}|d         dk     rd|d<   |dk    r|d         | |dz
           d         k     r| |dz
           }t          |d         |d         z   dz  d          }|d         |d         k    r6t          |d         t          |d         dz   |d         d	z
                      n||d<   t          |d         d	z   d          |d<   |d         |d         k    r!t          |d         t          z   d          |d<   |dz   |d
<   | S )Nr(   r   r^   rA   r)      r   r_   g{Gz?r   )rg   r   minri   r   )r   rV   cueprevmidpoints        r   fix_overlapsr      s9   D// 
 
3w<!CLq55S\DQK$666A;Dd7mc%j8A=qAAH]`ah]ilpqxly]y]y#d5k3tG}s/BCLSWDW+X+XYYY  @HDK et!3Q77CLu:W%%s7|o=qAACJ1uGKr   c                     t          t          | dz                      }t          |d          \  }}t          |d          \  }}t          |d          \  }}|dd|dd|dd|dS )Ni  i6 i`  02d:r   03d)intr   divmod)secmshremr*   ss         r   fmt_timer      s    	U3:		BB  FAsCFAs3EAr...a...a...b....r   c           	         g }| D ]}|                     t          |d                              |                     t          |d                    dt          |d                               |                     |d                    |                     d           |                    d                    |          d	           d S )
Nr   r(   z --> r)   r    r   r@   utf-8encoding)r9   strr   
write_textr   )r   pathlinesr   s       r   	write_srtr      s    E  SW&&'''W..KKXc%j5I5IKKLLLS[!!!ROODIIe$$wO77777r   c                     t          j        t                              d                    d         } t          j        t                              d                    }t          |           \  }}t          |          }t          |          }t          dt          |           dt          |                      t          ||          \  }}t          t          |||                    }g }	|s|	                    d           t          |          D ]\  }
}|d         s|	                    d|
d	z                nv|d
         dk     s|d         |d
         k    r|	                    d|
d	z                n;|
r8|d
         ||
d	z
           d         k     r|	                    d|
d	z                nt          j                            dd           t$          j                            dd           t'          |t                     t(          j        ddddt          |          t          |          |t          |          ||d}t,                              t          j        |dd          d           |	sdnd}t(          j        d|t3          t                    t3          t,                    t          |          t          |          t          |          ||	dd}t$                              t          j        |dd          d           |	r%t5          dd                    |	          z             t          dt          j        |t          |          |dd           z              d S )!Nr   r   rF   zALIGN original_tokens=z asr_words=zno subtitle cues generatedr    zempty text at cue rA   r(   r   r)   zinvalid time at cue zoverlap at cue T)parentsexist_ok   z"script/scenes.json:scene.narrationz:subtitles/master_narration_asr_words.json:segments[].wordsF)
project_idstepsource_texttiming_sourceasr_transcript_is_display_textoriginal_tokensrk   aligned_tokens
cues_countrH   r   r   )ensure_asciiindent	completedfailed_validationzVSubtitle display text is original scene.narration; ASR words are used only for timing.)r   r   status
output_srtoutput_jsonr   r   rk   r   validation_errorsnotezValidation failed: z; zDONE )r   r   r   )r   )jsonloadsSCENES_JSON	read_textASR_JSONrK   r.   r>   r   rE   r   r   r   r9   rg   
OUTPUT_SRTparentmkdirMANIFESTr   PROJECTnameOUTPUT_JSONr   dumpsr   
SystemExitr   )rF   asr_docr    rH   rj   rk   ro   rR   r   r   rV   r   mappingr   manifests                  r   mainr      sv   Z--w-??@@JFj++W+==>>G,V44D,%d++K!'**IN[!1!1NNc)nnNNOOO'Y??K
4kBBCCD ?  !=>>>D// 	 	36{ 	$$%?!A#%?%?@@@Ew<!s5zS\99$$%AAaC%A%ABBBE 	WQUE 222$$%<qs%<%<===ED4888O$666dJ l;U*/{++^^!$ii$ G 4:gE!LLLW^___ 1J[[7JFl*oo;''$ii{++^^!.h H 
8%JJJU\]]] O.;L1M1MMNNN$*D		U\]]lqrrrrsssssr   __main__)ra   r   r   r   r   pathlibr   r   r   r   r   r   r   r   r   r   compileUNICODEr,   r   r!   r.   r>   rK   r   r   r   r   r   r   __name__r5   r   r   <module>r      s     				           
$]
^
^,,@@77
AAAA2:fbj))A A A+ + +Q Q Q+ + +) ) ) &  &  & R* * *<  / / /8 8 8<t <t <t~ zDFFFFF r   