
    7q]j4              
          d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
mZ  ee                                          j        d         Zej        Zedz  Zd/d
Zd0dZd1dZd2dZd3dZd4dZd5dZd6dZd7d Zd8d"Zd9d&Zd:d'Zed(k    rw	  ej          e                       dS # e!$ rTZ" e# ej$        d)d* e%e"          j         d+e" d,d*-          ej&        .            ej         d           Y dZ"["dS dZ"["ww xY wdS );    )annotationsN)Path   script/project-manifest.jsondatabytesreturnstrc                N    t          j        |                                           S )N)hashlibsha256	hexdigest)r   s    script/run-originality.py	sha_bytesr      s    >$))+++    pathr   c                   t          j                    }|                     d          5 t          fdd          D ]}|                    |           	 d d d            n# 1 swxY w Y   |                                S )Nrbc                 .                          d          S )Ni   )read)handles   r   <lambda>zsha_file.<locals>.<lambda>   s    &++o">"> r   r   )r   r   openiterupdater   )r   digestchunkr   s      @r   sha_filer      s    ^F	4 !F>>>>DD 	! 	!EMM%    	!! ! ! ! ! ! ! ! ! ! ! ! ! ! ! s   ,A##A'*A'valuedictNonec                   | j                             dd           t          j        |dd          dz                       d          }t          j        d| j         d| j         	          \  }}	 t          j	        |d
          5 }|
                    |           |                                 t          j        |                                           d d d            n# 1 swxY w Y   t          j        ||            t          j                            |          rt          j        |           d S d S # t          j                            |          rt          j        |           w w xY w)NT)parentsexist_okF   )ensure_asciiindent
utf-8.)prefixdirwb)parentmkdirjsondumpsencodetempfilemkstempnameosfdopenwriteflushfsyncfilenoreplacer   existsunlink)r   r   r   fdr5   r   s         r   atomic_jsonr@      s   KdT222Ju5;;;dBJJ7SSD'749'7'7'7T[IIIHBYr4   	&FLLLLNNNHV]]__%%%	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	
47>>$ 	IdOOOOO	 	27>>$ 	IdOOOO	s1   2D: AC#D: #C''D: *C'+D: :6E0manifestkeyc                   |                      d          pi                      |          }t          |t                    r|st          d| d          t	          |          }|                                s
t          |z  }	 |                                                    t                                                     n%# t          $ r}t          d|           |d }~ww xY w|S )Nactive_pathszactive_paths.z is missingzactive path escapes project: )
get
isinstancer
   RuntimeErrorr   is_absoluteROOTresolverelative_to
ValueError)rA   rB   r   r   excs        r   resolve_activerN   .   s    \\.))/R44S99EeS!! = =;3;;;<<<;;D d{K""4<<>>2222 K K K@3@@AAsJKKs   >C 
C$CC$objecttuple[str, str]c                   t          | t                    rHt          |                     d          t                    r | d                                         }|r|dfS t          | t                    r|                     dg           ng }d |D             }|rd                    |          dfS g }|D ]!}t          |t                    r(t          |                    d          t                    sA|d         D ]}t          |t                    r<|                                r(|                    |                                           St          |t                    rot          |                    d          t                    rG|d                                         r-|                    |d                                                    ؐ#|rd                    |          d	fS t          d
          )N	narrationtop_level_narrationpartsc                (   g | ]}t          |t                    t          |                    d           t                    ?|                    d d                                          g|                    d d                                          S )rR    )rF   r    rE   r
   strip.0items     r   
<listcomp>z&extract_json_prose.<locals>.<listcomp>B   s       dD!! '1+1F1F&L&L RVQYQYZegiQjQjQpQpQrQrb!!''))  r   z

parts_narration_lf2
paragraphstextparts_paragraph_text_lf2zno supported prose field)	rF   r    rE   r
   rW   joinlistappendrL   )r   r^   rT   
narrationsr]   part	paragraphs          r   extract_json_proserf   <   s   % /:eii.D.Dc#J#J /[!'')) 	/...&0&=&=EEIIgr"""2E   J
  >{{:&&(===J = =$%% 	Z8N8NPT-U-U 	l+ 	= 	=I)S)) =ioo.?.? =!!)//"3"34444It,, =IMM&<Q<QSV1W1W =\efl\m\s\s\u\u =!!)F"3"9"9";";<<<		=
  C{{:&&(BBB
/
0
00r   c                $   | j                                         dk    r=|                     d                                          }|st	          d          |dfS t          j        |                     d                    }t          |          S )Nz.txtr)   encodingz
empty text	utf8_text)suffixcasefold	read_textrW   rL   r0   loadsrf   )r   r^   r   s      r   extractro   W   s    {''~~w~//5577 	+\***[  Jt~~w~7788Ee$$$r   
word_countinttuple[int, int, str]c                    | j                                         }ddddd}|                    |d          }||t          |           fS )Nd   _   Z   K   znarration.txtzstory_canonical.jsonzstory_package.jsonzstory_package_candidate.jsonr   )r5   rl   rE   r
   )r   rp   r5   
prioritiesprioritys        r   candidate_scorer{   a   sU    9D " (*	 J ~~dA&&HZT**r   tuple[list[dict], list[dict]]c                    g } g }t          t                                                    D ]}|                                r/|                                t
                                          k    rGg }|dz  }|                                r	 t          j        |	                    d                    }|
                    d          pi }dD ]}|
                    |          }t          |t                    r|s/t          |          }	|	                                s||	z  }	|	                                r|	|vr|                    |	           nL# t           $ r?}
|                    |j        dt%          |
          j         d|
 d           Y d }
~
nd }
~
ww xY w|d	z  }|                                r7d
D ]4}||z  }	|	                                r|	|vr|                    |	           5|s#|                    |j        dddigd           g }g }|D ]O}		 t)          |	          \  }}t+          |                                          }|dk     r)|                    t          |	          d| d           f|                    |j        t          |	          t/          |	          |t1          |                    d                    |t+          d t5          j        d|          D                       |t7          |	|          d	           # t           $ rG}
|                    t          |	          t%          |
          j         d|
 d           Y d }
~
Id }
~
ww xY w|rIt9          |d           }|                     |           |r|                    |j        d|d           |r|                    |j        d|d           | |fS )Nr   r)   rh   rD   )spoken_narrationstory_canon	candidatestory_candidateregistry_unreadable: )projectstatusreason	artifactsrx   coverage_note_onlyr   z$no current readable prose candidates)r   r   failuresi  zprose_too_short:)r   r   c                :    g | ]}|                                 |S  )rW   rX   s     r   r[   zinventory.<locals>.<listcomp>   s*    +i+i+iT\`\f\f\h\h+iD+i+i+ir   \n\s*\n)	r   artifact_pathartifact_sha256extraction_modeextracted_prose_sha256rp   paragraph_count_prose_scorec                    | d         S )Nr   r   )rZ   s    r   r   zinventory.<locals>.<lambda>   s
    DN r   )rB    covered_with_other_current_prose)r   r   ignored_failures)sortedPROJECTS_ROOTiterdiris_dirrJ   rI   is_filer0   rn   rm   rE   rF   r
   r   rH   rb   	Exceptionr5   type__name__ro   lensplitr   r   r2   rer{   max)selectednotesr   pathsmanifest_pathproject_manifestactiverB   r   r   rM   r   r5   readabler   prosemodewordschosens                      r   	inventoryr   m   s   HE-//1122 ;j ;j~~ 	7??#4#4#F#F  "@@  "" 	CC#':m.E.Ew.E.W.W#X#X )--n==C^ + +C"JJsOOE%eS11 ! ! ;;D++-- .&~||~~ +$e*;*;T***+  C C CAVeijmenenew  cA  cA{~  cA  cA  B  B  C  C  C  C  C  C  C  CCk)	 	'w ' ' 4'<<>> 'd%&7&7LL&&& 	LLW\=Qai  lR  aS  `T  U  U  V  V  V!! 	_ 	_D_%dmmtEKKMM**4<<OOSYYB\UZB\B\$]$]^^^&|%(YY'/~~'+.7W8M8M.N.N"''*+i+iRXjRW=X=X+i+i+i'j'j#-dE::
! 
! 
 
 
 
  _ _ _TcAS>\>\WZ>\>\ ] ]^^^^^^^^_ 	j'B'BCCCFOOF### DAc  zB  C  C  D  D  D 	jLLW\=Q_ghhiiiU?s9   CE
F 5FF!A!LBL
M+$<M&&M+r^   c                    t          j        d|                                           } t          j        dd|                                           S )NNFCz\s+ )unicodedata	normalizerl   r   subrW   r^   s    r   r   r      s@     --6688D6&#t$$**,,,r   	list[str]c                ^    t          j        dt          |           t           j                  S )Nz\w+)flags)r   findallr   UNICODEr   s    r   tokensr      s!    :fiooRZ@@@@r   r   corpustuple[int, str]c                l   i }t          |          D ].\  }}|                    |g                               |           /d}d}t          |           D ]\  }}|                    |g           d d         D ]}d}	||	z   t	          |           k     rw||	z   t	          |          k     ra| ||	z            |||	z            k    rI|	dz  }	||	z   t	          |           k     r.||	z   t	          |          k     r| ||	z            |||	z            k    I|	|k    r"|	}d                    | |||	z                      }||fS )Nr   rV   rt   r   r   )	enumerate
setdefaultrb   rE   r   r`   )
r   r   	positionsindextokenlongestsampleleftrightlengths
             r   longest_exact_runr      s   &(I!&)) 6 6uUB''..u5555GF ++ A Ae]]5"--dsd3 	A 	AEF-#i..00UV^c&kk5Q5QV_`dgm`mVnrxy~  BH  zH  sI  WI  WI! -#i..00UV^c&kk5Q5QV_`dgm`mVnrxy~  BH  zH  sI  WI  WI )D,>"?@@	A F?r   c                    t          j        t                              d                    } t	          | d          }t	          | d          }t	          | d          }t	          | d          }t          j        |                    d                    }t          |          }|                    d          }|                    d          dk    s|                    d	          d
urt          d          |                    d          |k    sD|                    d|                    d                    |                                j	        k    rt          d          t                      \  }}	t          |          dk     rt          dt          |                     dd
dt          t                    dt          |          d |D             |	t          j                            t          j        j                                                  d	}
t'          ||
           t)          |          d t+          j        d|          D             }g }d}|D ]}|d         }t)          |          d t+          j        d|          D             }t/          ||z            }i }t1          dd          D ]fdt1          t3          dt                    z
  dz                       D             }fdt1          t3          dt                    z
  dz                       D             }t          ||z            |t                    <   t5                    \  }}t3          ||          }| o0t7          d  |                                D                       o|dk     }|                    |d!         |d"         |d#         |d$         t          |          |d d%         ||||d&
           d
t          |          dk    t7          d' |D                       t7          d( |D                       |dk     d)}t7          |                                          ot7          d* |D                       }|rdnd+|t          |          |t          |          t          |          t          |          t          |          dddd,t=          d- |D                       |d.d/ |                                D             ||	t          j                            t          j        j                                                  d0}t'          ||           tA          t          j!        |d         ||t          |          |d1         d2         |t          |	          t          |          t          |          d3	d45                     |rdndS )6Nr)   rh   r   producer_close_receiptoriginality_inventoryoriginality_receiptr   	completedwriter_closedTz/candidate producer-close marker is not terminalcandidate_sha256r   
byte_countz,candidate differs from producer-close marker   z)readable current prose corpus too small: zNread-only originality comparison corpus; never project authority or dependencyzkone current readable prose artifact per prior project; prefer narration, canonical, package, then candidatec                J    g | ] }d  |                                 D             !S )c                D    i | ]\  }}|                     d           ||S )_)
startswith)rY   rB   r   s      r   
<dictcomp>z#main.<locals>.<listcomp>.<dictcomp>   s0    ZZZJCcnnUXFYFYZS%ZZZr   )itemsrX   s     r   r[   zmain.<locals>.<listcomp>   s0    ooo_cZZ$**,,ZZZooor   )	r   verifiedroleproject_authorityselection_ruleentry_countentriescoverage_notesgenerated_atc                l    h | ]1}t          t          |                    d k    "t          |          2S    r   r   r   rX   s     r   	<setcomp>zmain.<locals>.<setcomp>   sB        CPVW[P\P\L]L]acLcLc	$LcLcLcr   r   r   r   c                l    h | ]1}t          t          |                    d k    "t          |          2S r   r   rY   r   s     r   r   zmain.<locals>.<setcomp>   sD     
 
 
!&sSYZ_S`S`OaOaegOgOgIeOgOgOgr         c                D    h | ]}t          ||z                      S r   tuple)rY   r   candidate_tokenssizes     r   r   zmain.<locals>.<setcomp>   sC       J   J   JPU&6uUT\7I&J K K   J   J   Jr   r   c                D    h | ]}t          ||z                      S r   r   )rY   r   corpus_tokensr   s     r   r   zmain.<locals>.<setcomp>   sA      A  A  A%U=ut|1C#DEE  A  A  Ar   c              3  "   K   | ]
}|d k    V  dS r   Nr   r   s     r   	<genexpr>zmain.<locals>.<genexpr>   s&      3\3\5EQJ3\3\3\3\3\3\r   r   r   r   r      )
r   r   r   r   duplicate_paragraphsduplicate_paragraph_samplesshared_ngramslongest_exact_token_runlongest_exact_token_sampler   c              3  .   K   | ]}|d          dk    V  dS )r   r   Nr   rX   s     r   r   zmain.<locals>.<genexpr>  s-      (`(`t.D)E)J(`(`(`(`(`(`r   c              3  x   K   | ]5}t          d  |d                                         D                       V  6dS )c              3  "   K   | ]
}|d k    V  dS r   r   r   s     r   r   z!main.<locals>.<genexpr>.<genexpr>  s&      -e-eUeqj-e-e-e-e-e-er   r   N)allvaluesrX   s     r   r   zmain.<locals>.<genexpr>  sK      ){){jn#-e-ed?F[FbFbFdFd-e-e-e*e*e){){){){){){r   )producer_closedcorpus_readableduplicate_paragraphs_zeroshared_12_to_16_grams_zero longest_exact_token_run_under_12c              3  &   K   | ]}|d          V  dS )r   Nr   rX   s     r   r   zmain.<locals>.<genexpr>  s'      +R+RD,<+R+R+R+R+R+Rr   failed)r   shared_12_to_16_grams%longest_exact_token_run_max_exclusivec              3  &   K   | ]}|d          V  dS )r   Nr   rX   s     r   r   zmain.<locals>.<genexpr>  s)      1d1dSW$7M2N1d1d1d1d1d1dr   )r   max_longest_exact_token_runc                     i | ]\  }}|||d S ))r   r   r   )rY   r5   r   s      r   r   zmain.<locals>.<dictcomp>  s&    ___u4ee<<___r   )r   r   candidate_pathr   r   inventory_pathinventory_sha256corpus_count
thresholds	aggregatechecksfindingsr   
checked_atr  r   )	r   r   r   r  r   r	  coverage_note_countr   receiptFr&   )"r0   rn   MANIFEST_PATHrm   rN   r   rE   rG   statst_sizer   r   r
   rI   datetimenowtimezoneutc	isoformatr@   r   r   r   r   ranger   r   r   r   rb   sumr   printr1   )rA   r  
close_pathr  receipt_pathclosecandidate_hashcandidate_textr   r   inventory_receiptcandidate_paragraphsr  max_runrZ   corpus_textcorpus_paragraphsr   sharedcandidate_ngramscorpus_ngramsr   r   r   r  r  r   r   r   s                             @@@r   mainr/     s   z-1171CCDDH#Hk::N*BCCJ#H.EFFN!(,ABBLJz++W+==>>En--N#--w-??Nyyk))UYY-G-Gt-S-SLMMMyy#$$66%))GUYYWcMdMd:e:eiwi|i|i~i~  jG  ;G  ;GIJJJ'kkG^
7||aUs7||UUVVV` YY H7||oognooo( )--h.?.CDDNNPP
 
  1222n-- $&HZ$H$H   HG  8n{++
 
*,(:{*K*K
 
 
  &&:=N&NOO!#"bMM 	F 	FD  J   J   J   J   JY^_bcdfijzf{f{  C  gC  FG  gG  `H  `H  ZI  ZI   J   J   J A  A  A  A  ASXY\]^`cdq`r`ruy`y|}`}Y~Y~SS  A  A  AM #$4}$D E EF3t99+,<mLLgw''++m3\3\FMMOO3\3\3\0\0\mahkmamI!/2#$56&*+C&D$'(<$=$=+?+C#'.*0 
 
 	 	 	 	  w<<1,%((`(`W_(`(`(`%`%`&)){){rz){){){&{&{,3bL F 6==??##R+R+R+R+R+R(R(RH!)7++xn--*"%j//n--$^44G/01wyzz.11d1d[c1d1d1d.d.d  FM  N  N__PVP\P\P^P^___('++H,=,ABBLLNN G  g&&&	$*(#~Ggk>RSi>j'.s>GZGZ((S5F5F	 
       11ar   __main__r  Fr   )r   r   errorr  )file)r   r   r	   r
   )r   r   r	   r
   )r   r   r   r    r	   r!   )rA   r    rB   r
   r	   r   )r   rO   r	   rP   )r   r   r	   rP   )r   r   rp   rq   r	   rr   )r	   r|   )r^   r
   r	   r
   )r^   r
   r	   r   )r   r   r   r   r	   r   )r	   rq   )'
__future__r   r  r   r0   r6   r   sysr3   r   pathlibr   __file__rJ   r#   rI   r.   r   r  r   r   r@   rN   rf   ro   r{   r   r   r   r   r/  r   exitr   rM   r!  r1   r   stderrr   r   r   <module>r9     sw   " " " " " "    				 				 



           tH~~'*55, , , ,         1 1 1 16% % % %	+ 	+ 	+ 	+? ? ? ?D- - - -
A A A A   "^  ^  ^  ^ B z   jdjH%TTRUYYM_JhJhcfJhJhiix}~~~  FI  FP  	Q  	Q  	Q  	Q s   B2 2D7A	DD