
    +^j!                        d dl Z d dlZd dlZd dlmZmZ d dlmZ  ee                                          j	        d         Z
e
dz  Ze
dz  Zedz  Zedz  Zed	z  Zed
z  Zedz  Zedz  Zedz  Ze
dz  Ze
dz  Ze
dz  Zedz  Ze
dz  ZddhZd Zd Zd Zd Zd Zd Ze dk    rA	  e! e                      # e"$ r%Z# e$de# ej%                    e!d          dZ#[#ww xY wdS )    N)datetimetimezone)Path   logzscript/project-manifest.jsonzcompletion-report.jsonzpostiz-media.jsonzpostiz-schedule.jsonzupload-transcode.jsonzfinal-render.jsonzfootage-job.jsonzstorage-cleanup.jsonzoutput/final.mp4zoutput/footage/footage.mp4z-output/footage/footage-rejected-subtitles.mp4zfootage-rejection.jsonzoutput/final-upload.mp4cmrq10vod000hj7cbt8zvuj6acmrpr0j9u000bj7cboqonlx4bc                     |                                  s*t          d|                     t                               t	          j        |                                           S )Nzmissing )existsRuntimeErrorrelative_toPROJECTjsonloads	read_text)paths    script/cleanup-storage.pyloadr      sR    ;;== CAd&6&6w&?&?AABBB:dnn&&'''    c                    t          j                    }|                     d          5 t          fdd          D ]}|                    |           	 d d d            n# 1 swxY w Y   |                                S )Nrbc                  .                          d          S )Ni   )read)handles   r   <lambda>zdigest.<locals>.<lambda>#   s    &++o">"> r   r   )hashlibsha256openiterupdate	hexdigest)r   hasherblockr   s      @r   digestr$       s    ^F	4 !F>>>>DD 	! 	!EMM%    	!! ! ! ! ! ! ! ! ! ! ! ! ! ! ! s   ,A##A'*A'c                     dD ]!}|                      |          r
| |         c S "|                      d          pi }|                     d          p|                     d          S )N)output_sha256r   verificationr&   r   )get)rowkeyr'   s      r   receipt_hashr+   (   sr    *  773<< 	s8OOO	77>**0bLO,,J0@0@0J0JJr   c                 H   |                     d          pi }|                     d          p|                     d          pi }t          |          }|                     d          p)|                     d          p|                     d          }|s*|                     d          pi                      d          }|r|s*t          d|                     t                               t          |                     t                              t          |          |d	S )
Nr'   independent_probeprobeoutput_bytesbytesformatsizez"historical receipt incomplete for r   r0   r   )r(   r+   r   r   r   strint)r   receiptr'   r.   output_hashr/   s         r   historical_rowr8   0   s   ;;~..4"LKK+,,JG0D0DJEw''K;;~..j'++g2F2Fj,JZJZ[iJjJjL A8,,277?? ]l ][@P@PQX@Y@Y[[\\\((1122S=N=NZefffr   c                    |                      d          dus|                      d          dk    rdS |                      d          pi }t          |          t          k    rdS t                      }t          D ]|}|                     |          pg }t          |          dk    s|d                              d	          d
k    r dS |                    |d                              d                     }t          |          dk    od |vS )NverifiedTstatus	completedFpostsr   r   stateQUEUEpublishDate)r(   setTARGET_INTEGRATIONSlenadd)scheduler=   
timestampsintegrationrowss        r   validate_schedulerI   <   s    ||Jt++x||H/E/E/T/TuLL!!'RE
5zz(((uJ* 3 3yy%%+t99>>T!W[[11W<<55tAw{{=112222z??a:D
$::r   c                     t          t                    } t          t                    }t          t                    }t          t                    }t          t
                    }t          t                    }t          t                    }|                    d          }|                    d          pi                     d          pi }|                     d          }	|                     d          du o+|                     d          |                     d          k    |                    d          du o|                    d	          |	k    t          |          o|                    d	          |	k    |                    d          du oD|                    d
          dk    o+|                    d          pi                     d          du t                                          d}
t          |
                                          sAt          dd                    d |
                                D                       z             t#          t                    }t                                          j        }|dk    s|r||k    rt          d          |                    d          |k    s't)          |                    dd                    |k    rt          d          t*                                          r+t-          j        t*                                                    nd }|r)|                    d          du r|                    d
          dk    r|                    d          pi }t2                                          st4                                          st6                                          st                                          rnt#          t                    |k    rV|                    d          |k    r=t9          t-          j        ddd|                    d          dd                     dS t          d          t=          t2          |          t=          t4          |          g}t6                                          rt          t>                    }tA          t6          !                    tD                              t)          |                    dd                    |                    d          d }|                    d          dus_|                    d
          d!k    sF|d         t6                                          j        k    s|d         t#          t6                    k    rt          d"          |#                    |           t2                                          rt4                                          st          d#          t2                                          j        |d         d         k    s$t#          t2                    |d         d         k    rt          d$          t4                                          j        |d%         d         k    s$t#          t4                    |d%         d         k    rt          d&          tI          j%        tL          j'                  (                                }d%dd'tD          j)        |	|
|tA          t          !                    tD                              ||d |d(	}t*          *                    t-          j        |dd)*          d+z              t2          +                                 t4          +                                 t6                                          rt6          +                                 t                                          rt#          t                    nd }t2                                          t4                                          t6                                          t                                          |t                                          rt                                          j        nd d,}|d-          o|d.          o|d/          o|d0         o||k    }|s|,                    d1|tI          j%        tL          j'                  (                                d2           t*          *                    t-          j        |dd)*          d+z              t          d3          |,                    ddt[          d4 |D                       |tI          j%        tL          j'                  (                                d5           t*          *                    t-          j        |dd)*          d+z              d|.                    d6i           d7<   ddd8|d         d9|d7<   |d:         |d;<   t          *                    t-          j        |dd)*          d+z              t9          t-          j        dd|d         |d<         d=d                     dS )>Nr&   mediavideocanon_sha256r:   Tchecks_passedchecks_totalsource_canon_sha256r;   r<   server_verificationunder_one_gb)
completionrK   rE   	transcodeupload_copy_existszcleanup gates failed: z, c              3   $   K   | ]\  }}||V  d S )N ).0r*   values      r   	<genexpr>zmain.<locals>.<genexpr>^   s,      ?q?q
Ukp?q?q?q?q?q?q?qr   i ʚ;zupload copy size/hash mismatchr   r0   z/Postiz media receipt does not match upload copypost_deleteupload_copy_sha256freed_bytes)r:   r;   resumedr^   F)ensure_asciir   z6completed cleanup receipt conflicts with current filesr3   rejectedz1rejected footage does not match rejection receiptz7master missing before cleanup without completed receiptz.final master does not match historical receiptr   z0footage master does not match historical receiptrunning)	versionr:   r;   
project_idrP   gate_checksdeleted_candidatesretained
started_at   )r`   indent
)final_master_existsfootage_master_existsrejected_footage_existsrU   r]   upload_copy_bytesrl   rm   rn   rU   failed)r;   r\   completed_atzpost-delete verification failedc              3   &   K   | ]}|d          V  dS )r0   NrW   )rX   r)   s     r   rZ   zmain.<locals>.<genexpr>   s*      OpOpadPST[P\OpOpOpOpOpOpr   )r:   r;   r^   r\   rq   stepsstorage_cleanupzlog/storage-cleanup.json)r;   r:   r6   r^   rq   
updated_atrg   )r:   r;   r^   rg   )/r   
COMPLETIONMEDIASCHEDULE	TRANSCODEFINAL_RECEIPTFOOTAGE_RECEIPTMANIFESTr(   rI   UPLOADr   allvaluesr   joinitemsr$   statst_sizer5   RECEIPTr   r   r   FINALFOOTAGEREJECTED_FOOTAGEprintdumpsr8   REJECTION_RECEIPTr4   r   r   appendr   nowr   utc	isoformatname
write_textunlinkr    sum
setdefault)rS   rK   rE   rT   final_receiptfootage_receiptmanifestexpected_upload_hashmedia_videocanongatesupload_hashupload_bytespriorpostdeleted	rejectionrejected_rowstartedrb   upload_afterr:   s                         r   mainr   K   s	   j!!JKKEH~~HYI''M?++OH~~H$==9999W%%+0099?RKNN>**E nnZ00D8~Z^^O=\=\`j`n`no}`~`~=~:&&$.\599=R3S3SW\3\%h//`HLLAV4W4W[`4`]]:..$6  o9==;R;RVa;a  ogpgtgt  vK  hL  hL  hR  PR  gW  gW  Xf  gg  gg  ko  go$mmoo E u||~~ s3dii?q?qV[VaVaVcVc?q?q?q6q6qqrrr..K;;==(L}$$,@$KSgDgDg;<<<x  K//3{wPR7S7S3T3TXd3d3dLMMM/6~~/?/?IDJw((**+++TE U:&&$..599X3F3F+3U3Uyy''-2||~~ 	gnn&6&6 	?O?V?V?X?X 	]c]j]j]l]l 	qwx~qq  DO  rO  rO  TX  T\  T\  ]q  Tr  Tr  vA  TA  TA$*$+RVglgpgpq~gg  A  A  PU  V  V  V  W  W  W1STTTe]33^G_5]5]^G   %*++	(44W==>>w3344mmH--
 
 MM*%%T11}}X&&*44G$(8(=(=(?(?(GGGH%0@)A)AAARSSS|$$$<<>> V!1!1 VTUUUzz||wqz'222fUmmwqzRZG[6[6[KLLL||~~G!444w7ST:V^K_8_8_MNNNl8<((2244G%9GL$URY !3!3G!<!<==`kll	 G tz'aHHH4OPPP	LLNNNNN   "!!!%+]]__>6&>>>$L$||~~HXHX#3#:#:#<#<$mmoo\6<mmooOV[[]]224	 D -..  |t<S7T3T  |]ab{]|Y|  |  BF  G[  B\  |  am  q|  a|H >(4QYQ]^f^jQkQkQuQuQwQwxxyyy4:gE!LLLtSTTT<===NNCOpOphoOpOpOpLpLp  BF  X`  Xd  em  eq  Xr  Xr  X|  X|  X~  X~      @  @  @tz'aHHH4OPPP:EH$$%67-8dWq  CJ  KX  CY  #Z  #ZH$^4H\
8%JJJTQRRR	$*$+gVcNdry  {E  sF  G  G  V[  \  \  \  ]  ]  ]1r   __main__zStorage cleanup blocked: )file)&r   r   sysr   r   pathlibr   __file__resolveparentsr   LOGr|   rv   rw   rx   ry   rz   r{   r   r   r   r   r   r}   rB   r   r$   r+   r8   rI   r   __name__
SystemExit	Exceptionexcr   stderrrW   r   r   <module>r      s     



 ' ' ' ' ' ' ' '      
$x..
 
 
"
"
*1
-o33++
!!''))	))**
&
&$$
0
0LL 22 	,	,24OP ( ( (  K K K	g 	g 	g; ; ;V V Vr zj      /#//cjAAAAjmm s   &B9 9C#> CC#