
    @(i_                    R   S r SSKJr  SSKrSSKrSSKrSSKrSSKrSSK	J
r
  SSKJr  SSKJr  SSKJr  SSKrSSKrSSKrSSKJr  SS	KJr  SS
KJr  SSKJrJr  SSKJr  SSKJ r   SSK!J"r"  SSK#J$r$  \RJ                  " \&5      r'\(       a$  SSK(J)r)J*r*  SSKJ+r+  SSK!J,r,  SSK-J.r.J/r/  SSK0J1r1J2r2  Sr3Sr4S(S jr5S)S jr6\Rn                  S*S j5       r8S+S jr9 " S S\Rt                  5      r; " S S\;5      r<S,S  jr= " S! S"\;5      r>S-S# jr? S.     S/S$ jjr@S0S% jrAS1S& jrBS2S' jrCg)3zZ
Models for sharded repodata, and to make monolithic repodata look like sharded
repodata.
    )annotationsN)defaultdict)Path)TYPE_CHECKING)urljoin)context)
SubdirData)get_session)_add_http_value_to_dictconda_http_errors)Channel)specs)	HTTPError   )shards_cache)IterableKeysView)RepodataCache)Response)RepodataDictShardsIndexDict)PackageRecordDict	ShardDict
   i   c                 P    [         R                  b  [         R                  $ [        $ )a  
If context.repodata_threads is not set, find the size of the connection pool
in a typical https:// session. This should significantly reduce dropped
connections. We match requests' default 10.

Is this shared between all sessions? Or do we get a different pool for a
different get_session(url)?

Other adapters (file://, s3://) used in conda would have different
concurrency behavior;  we are not prepared to have separate threadpools per
connection type.
)r   repodata_threadsSHARDS_CONNECTIONS_DEFAULT     T/data/cameron/miniconda/lib/python3.13/site-packages/conda_libmamba_solver/shards.py_shards_connectionsr!   9   s"     +'''%%r   c                    S HO  nU R                  U5      =n(       d  M  [        U[        5      (       a  M4  [        U5      R	                  5       X'   MQ     U $ )zA
Convert bytes checksums to hex; leave unchanged if already str.
)sha256md5)get
isinstancestrbyteshex)record	hash_type
hash_values      r    ensure_hex_hashr-   K   sL     %	I..:.j#..$)*$5$9$9$;! % Mr   c                n    [         R                  R                  U 5      n[        UR                  5      nU$ )z3
Given a dependency spec, return the package name.
)r   	MatchSpecparser'   name)specparsed_specr1   s      r    spec_to_package_namer4   V   s-     //''-K{ DKr   c              #    #    [        5       n/ U S   R                  5       QU S   R                  5       Q7 HQ  n[        U5        / UR                  SS5      Q7 H*  nX1;   a  M
  UR	                  U5        [        U5      nUv   M,     MS     g7f)z_
Return all dependency names mentioned in a shard, not including the shard's
own package name.
packagespackages.condadependsr   N)setvaluesr-   r%   addr4   )shardunique_specspackager2   r1   s        r    shard_mentioned_packagesr?   b   s     
 5LSU:&--/S%8H2I2P2P2RS 2gkk)R02D#T"'-DJ 3 Ts   B
Bc                  P   \ rS rSr% SrS\S'   S\S'   S\S'   S\S	'   \\R                  SS
 j5       5       r	\SS j5       r
SS jr\R                  SS j5       r\R                  SS j5       rSS jrSS jr\R                  SS j5       r\R                  SS j5       rSS jrSrg)	ShardBaser   z
Abstract base class for shard-like objects.

Defines the common interface for both sharded repodata (Shards)
and traditional repodata presented as shards (ShardLike).
r'   urlr   repodata_no_packageszdict[str, ShardDict | None]visited	_base_urlc                    g)zDReturn the names of all packages available in this shard collection.Nr   selfs    r    package_namesShardBase.package_names        	r   c                V    [        [        U R                  U R                  5      S5      $ )a*  
Return self.url joined with base_url from repodata, or self.url if no
base_url was present. Packages are found here.

Note base_url can be a relative or an absolute url.
The double urljoin ensures proper URL normalization:
first join with _base_url, then with "." to add trailing slash
if needed.
.)r   rC   rF   rH   s    r    base_urlShardBase.base_url   s      wtxx8#>>r   c                    XR                   ;   $ )z9Check if a package is available in this shard collection.)rJ   rI   r>   s     r    __contains__ShardBase.__contains__   s    ,,,,r   c                    g)z
Return shard URL for a given package. For monolithic repodata, should
not be fetched but is a unique identifier.

Raise KeyError if package is not in the index.
Nr   rR   s     r    	shard_urlShardBase.shard_url   s     	r   c                    g)8
Return True if the given package's shard is in memory.
Nr   rR   s     r    shard_loadedShardBase.shard_loaded       
 	r   c                    g)zF
Return a shard that is already loaded in memory and mark as visited.
Nr   rR   s     r    visit_packageShardBase.visit_package   rL   r   c                     X R                   U'   g)z+
Store new shard data in the visited dict.
NrE   rI   r>   r<   s      r    visit_shardShardBase.visit_shard   s     !&Wr   c                    g)z2
Fetch an individual shard for the given package.
Nr   rR   s     r    fetch_shardShardBase.fetch_shard   r\   r   c                    g)z"
Fetch multiple shards in one go.
Nr   )rI   r6   s     r    fetch_shardsShardBase.fetch_shards   r\   r   c                    U R                   R                  5       nUR                  0 0 S.5        U R                  R	                  5        H(  u  p#Uc  M
  S H  nX   R                  X4   5        M     M*     U$ )z:
Return monolithic repodata including all visited shards.
r6   r7   )rD   copyupdaterE   items)rI   repodata_r<   package_groups        r    build_repodataShardBase.build_repodata   sp     ,,113R2>?**,HA}!?'..u/CD "@ -
 r   r   NreturnzKeysView[str]rv   r'   r>   r'   rv   boolr>   r'   rv   r'   r>   r'   rv   r   )r>   r'   r<   r   r6   Iterable[str]rv   zdict[str, ShardDict])rv   r   )__name__
__module____qualname____firstlineno____doc____annotations__propertyabcabstractmethodrJ   rO   rS   rV   rZ   r^   rc   rf   ri   rs   __static_attributes__r   r   r    rA   rA   r   s     
H&&((N   
? 
?- 	  	 & 	  	 r   rA   c                     ^  \ rS rSrSrSSS jjrU 4S jr\SS j5       rSS jr	SS jr
SS jrSS	 jrSS
 jrSrU =r$ )	ShardLike   z:
Present a "classic" repodata.json as per-package shards.
c                   0 UE0 0 S.EU l         UR                  S0 5      UR                  S0 5      S.nX l        [        S 5      nUR	                  5        H-  u  pVUR	                  5        H  u  pxUS   n	XU	   U   U'   M     M/     [        U5      U l        0 U l         U R                   S   S   n
[        U
[        5      (       d+  [        R                  S[        U
5       35        [        5       eXl        g
! [         a
    S	U l         g
f = f)z6
url: must be unique for all ShardLike used together.
rl   r6   r7   c                     0 0 S.$ )Nrl   r   r   r   r    <lambda>$ShardLike.__init__.<locals>.<lambda>   s	    "%Kr   r1   inforO   z0repodata["info"]["base_url"] was not a str, got  N)rD   r%   rC   r   ro   dictshardsrE   r&   r'   logwarningtype	TypeErrorrF   KeyError)rI   rp   rC   all_packagesr   
group_namegroupr>   r*   r1   rO   s              r    __init__ShardLike.__init__   s   3
3
 3
! !Z4&ll+;R@
 KL!-!3!3!5J#(;;=f~4:tZ(1 $1 "6 -1L 57	 008DHh,,NtT\~N^_`k!%N 	 DN	 s   AC8 8DDc                f   > [         TU ]  5       R                  SS9u  pU SU R                   SU 3$ )Nr   )maxsplit )super__repr__splitrC   )rI   leftright	__class__s      r    r   ShardLike.__repr__   s;    g&(...:q
!E7++r   c                6    U R                   R                  5       $ N)r   keysrH   s    r    rJ   ShardLike.package_names   s    {{!!r   c                B    U R                   U     U R                   SU 3$ )W
Return shard URL for a given package.

Raise KeyError if package is not in the index.
#)r   rC   rR   s     r    rV   ShardLike.shard_url  s%     	G((1WI&&r   c                    XR                   ;   $ rY   )r   rR   s     r    rZ   ShardLike.shard_loaded  s     ++%%r   c                2    U R                  U5      nUc   eU$ z?
Return a shard that is already in memory and mark as visited.
rf   rb   s      r    r^   ShardLike.visit_package  s%       )   r   c                @    U R                   U   nX R                  U'   U$ )z
"Fetch" an individual shard.

Update self.visited with all not-None packages.

Raise KeyError if package is not in the index.
)r   rE   rb   s      r    rf   ShardLike.fetch_shard  s#     G$ %Wr   c                P    U Vs0 s H  o"U R                  U5      _M     sn$ s  snf )zS
Fetch multiple shards in one go.

Update self.visited with all not-None packages.
r   )rI   r6   r>   s      r    ri   ShardLike.fetch_shards%  s,     CKK(w))'22(KKKs   #)rF   rD   r   rC   rE   )r   )rp   r   rC   r'   ru   rz   rx   r{   r|   )r~   r   r   r   r   r   r   r   rJ   rV   rZ   r^   rf   ri   r   __classcell__)r   s   @r    r   r      sG    $ L, " "'&
L Lr   r   c                p    U(       a  UR                  S5      (       d  US-  n[        [        X5      S5      $ )zq
Return shards_base_url joined with base_url and url.
Note shards_base_url can be a relative or an absolute url.
/rN   )endswithr   )rC   shards_base_urls     r    _shards_base_urlr   .  s3    
 77<<3730#66r   c                      \ rS rSrSrSrSrSS jr\S 5       r	\S 5       r
\SS j5       rSS	 jrSS
 jrSS jrSS jrSS jrS rSrg)Shardsi8  zG
Handle repodata_shards.msgpack.zst and individual per-package shards.
r   )NNc                    Xl         X l        X0l        [        U R                  5      R
                  n[        U5      U l        US   0 0 SS.U l        0 U l	        US   S   U l
        g)z]
Args:
    shards_index: raw parsed msgpack dict
    url: URL of repodata_shards.msgpack.zst
r      )r   r6   r7   repodata_versionrO   N)shards_indexrC   r   r   r   rO   r
   sessionrD   rE   rF   )rI   r   rC   cachechannel_base_urls        r    r   Shards.__init__A  st     )! #4#7#78AA"#34 !(  !	%
! 57
 &f-j9r   c                6    U R                   R                  5       $ r   )packages_indexr   rH   s    r    rJ   Shards.package_names_  s    ""''))r   c                     U R                   S   $ )Nr   )r   rH   s    r    r   Shards.packages_indexc  s      **r   c                    U R                   S   R                  SS5      nU R                  U4nU R                  U:w  a!  X l        [	        U R                  U5      U l        U R                  $ )zi
Return self.url joined with shards_base_url.
Note shards_base_url can be a relative or an absolute url.
r   r   r   )r   r%   rC   _shards_base_url_keyr   )rI   shards_base_url_	cache_keys      r    r   Shards.shards_base_urlg  sf      ,,V4889JBOXX/0	$$	1(1%$4TXX?O$PD!$$$r   c                t    [        U R                  U   5      R                  5        S3nU R                   U 3$ )r   z.msgpack.zst)r(   r   r)   r   )rI   r>   
shard_names      r    rV   Shards.shard_urlu  s@     d11':;??AB,O
&&'
|44r   c                    XR                   ;   $ r   ra   rR   s     r    rZ   Shards.shard_loaded  s     ,,&&r   c                $    U R                   U   nU$ r   ra   rb   s      r    r^   Shards.visit_package  s     W%r   c                ,    U R                  U/5      U   $ )z
Fetch an individual shard for the given package.

Default implementation calls fetch_shards() with a single package.
Subclasses may override for more efficient single-fetch operations.

Raise KeyError if package is not in the index.
)ri   rR   s     r    rf   Shards.fetch_shard  s       '+G44r   c                   0 nS n[        [        U5      5      n0 nU H8  nXPR                  ;   a  U R                  U   X%'   M%  XTU R                  U5      '   M:     [        R
                  R                  [        5       S9 nUR                  5        VVs0 s H*  u  puXR;  d  M  UR                  X0R                  Xu5      Xu4_M,     nnn[        R
                  R                  U5       H3  n	[        R                  SX   5        X   u  puU R                  XXR5        M5     SSS5        U R                  R                  U5        U$ s  snnf ! , (       d  f       N1= f)z
Return mapping of *package names* to Shard for given packages.

If a shard is already in self.visited, it is not fetched again.
c                    U R                  U5      nUR                  5         UR                  n[        R                  " XUS9$ )N)rC   r>   compressed_shard)r%   raise_for_statuscontentr   AnnotatedRawShard)srC   package_to_fetchresponsedatas        r    fetch"Shards.fetch_shards.<locals>.fetch  s>    uuSzH%%'##D11D r   max_workersz. %sN)sortedlistrE   rV   
concurrentfuturesThreadPoolExecutorr!   ro   submitr   as_completedr   debug_process_fetch_resultrn   )
rI   r6   resultsr   urls_packagesr>   executorrC   r   futures
             r    ri   Shards.fetch_shards  s:    	 $x.)G,,&#'<<#8 9@dnnW56	   22?R?T2UYa %2$7$7$9$9LC) S||SBSNR$9  
 %,,99'B		&'/2&**6I C V 	G$ VUs%   ?EE""EAEE
Ec                *   [        X#5         UR                  5       nSSS5        [        R                  " [        R
                  " WR                  [        S95      XER                  '   U R                  R                  U5        g! , (       d  f       Nm= f)z!
Process a single fetched shard.
Nmax_output_size)r   resultmsgpackloads	zstandard
decompressr   ZSTD_MAX_SHARD_SIZEr>   r   insert)rI   r   rC   r>   r   fetch_results         r    r   Shards._process_fetch_result  sn     s,!==?L - )0  --?R)
$$% 	  . -,s   B
B)	rF   r   r   rD   r   r   r   rC   rE   N)r   r   rC   r'   r   zshards_cache.ShardCacherw   rz   rx   r{   r|   )r~   r   r   r   r   r   r   r   r   rJ   r   r   rV   rZ   r^   rf   ri   r   r   r   r   r    r   r   8  su    
 ':< * * + + % %5'	5&P/r   r   c                   [        U 5      nUR                  n0 nUR                  nUR                  nU(       a  [	        U5      US'   U(       a  [	        U5      US'   Sn[        X5         [        R                  [        R                  4nUR                  XUR                  US9n	U	R                  5         U	R                  n
SSS5        W	R                  S:X  a  UR                  R                  5       $ [         R"                  R$                  R&                  U 0nS[         R"                  R$                  R(                  4S[         R"                  R$                  R*                  4S	[         R"                  R$                  R,                  44 H  u  p[/        XX5        M     UR1                  U5        W
$ ! , (       d  f       N= f)
zn
Fetch shards index with cache.

Update cache state.

Return shards data, either newly fetched or from cache.
zIf-None-MatchzIf-Modified-Sincezrepodata_shards.msgpack.zst)headersproxiestimeoutNi0  EtagzLast-ModifiedzCache-Control)r
   stateetagmodr'   r   r   remote_connect_timeout_secsremote_read_timeout_secsr%   r  r   r   status_codecache_path_shards
read_bytescondagatewaysrp   URL_KEYETAG_KEYLAST_MODIFIED_KEYCACHE_CONTROL_KEYr   rn   )rC   r   r   r
  r  r  last_modifiedfilenamer  r   response_bytessaved_fieldsheaderkeys                 r    repodata_shardsr    s}    #GKKEG::DIIM#&t9 '*='9#$,H	3	)//,,
 %[['//7 ) 
 	!!#!)) 
* s" &&1133NN++33S9L	((112NN##55	
 
%..11CCD 	 ,D 
LL ? 
*	)s   )AF22
G c                   U R                   nUR                  n UR                  S5       n[        R                  " UR                  5       5      nUR                  R                  U5        SSS5        UR                  nUcE  [        R                  " [        [        R                  R                  R!                  5       5      5      nUR#                  S5      (       a  SnU R$                   S3nUR&                  R)                  5       (       d  SUl        SUl        O/UR/                  5       (       d  UR&                  R1                  5       nUc/   [3        X5      nUR5                  SS5        UR7                  U5        U(       a;  [>        R                  " [@        RB                  " U[D        S95      n	[G        XU5      n
U
$ g! , (       d  f       GN\= f! [        [        R                  4 a     GN|f = f! [8        [        R                  R                  R:                  4 a%    UR5                  SS5        UR=                  5          Nf = f)	a  
Check a SubdirData's URL for shards.

Return shards index bytes from cache or network.
Return None if not found; caller should fetch normal repodata.

TODO: If this function fails to retrieve the sharded repodata index file, it will
      mark it is as not supporting this feature in cache. This can problematic
      because sometimes server errors can happen which will lead it to wrongly
      assuming the channel doesn't support sharding. We need to rethink our
      logic for determining shard support.
zr+Nr   z/repodata_shards.msgpack.zstr   TFr   )$
repo_fetch
repo_cachelockjsonr   readr
  rn   FileNotFoundErrorJSONDecodeErrorr   
ShardCacher   r  r  rp   create_cache_dirshould_check_formaturl_w_subdirr  existsr  r  staler  r  set_has_formatsaver   RepodataIsEmptyrefreshr   r   r   r  r   )sdr   r   r!  
state_filer
  cache_stateshards_datashards_index_urlr   r   s              r    fetch_shards_indexr6    s     MME!!J__T"j JJz01E##E*	 # ""K}''U^^-D-D-U-U-W(XY&&x00 oo..JK++2244!K KO!!##$66AACK	%-.>K**8T: , ,3MM$$[BUV-L LEBFM] #"
 t334 8 u~~66FFG %**8U;""$%sB   G A G +G .G3  
G
G G G0/G03AIIc           	     :   U  Vs/ s H  n[        U[        5      (       d  M  UPM     n n/ nU  H5  nU H,  nXT;   d  M
  UR                  XEUR                  U5      45        M.     M7     [        R                  S[        U5      5        U (       d  [        R                  S5        U$ U S   R                  nUR                  U VVs/ s H  Gt pxUPM	     snn5      n	U H1  u  pJnU	R                  U5      =n(       d  M   UR                  X5        M3     U$ s  snf s  snnf )z
Given a list of Shards objects and a list of package names, fetch all URLs
from a shared local cache, and update Shards with those per-package shards.
Return the remaining URLs that must be fetched from the network.
z%d shards to fetchzNo sharded channels found.r   )r&   r   appendrV   r   r   lenr   retrieve_multipler%   rc   )shardedr6   	shardlikewantedr<   package_nameshared_shard_cacherq   rV   
from_cacher>   from_cache_shards               r    batch_retrieve_from_cacherB  U  s    +2S'YZ	65Ry'GSF $L$uEOOL4QRS % 
 II"CK0		./ 00#55V\6]V\]ayV\6]^J &,!	)~~i888g8 &, M1 T" 7^s   DDD
c                    [        [        5      nU  H  u  p#nX   R                  U5        M     UR                  5        H  u  p%UR	                  U5        M     g)z
Given a list of (Shards, package name, shard URL) tuples, group by Shards and call fetch_shards
with a list of all URLs for that Shard.
N)r   r   r8  ro   ri   )r=  shard_packagesr<   r>   rq   r6   s         r    batch_retrieve_from_networkrE  v  sS    
 /:$.?N#$$W- $ *//18$ 2r   c                   U  Vs/ s H  n[        U5      PM     n n[        S U  5       5      n0 n[        R                  " [	        [
        R                  R                  R                  5       5      5      n[        R                  R                  [        5       S9 nUR                  5        VVs0 s H&  u  pgUR                  [        [!        U5      U5      U_M(     nnn0 n	[        R                  R#                  U5       H^  n
X   nU
R%                  5       nU(       a  XU'   M$  UU	UR                  [!        [        U5      5      R&                  R(                  5      '   M`     [        R                  R#                  U	5       H-  n
X   nU
R%                  5       u  pU S3n[+        X5      nXU'   M/     SSS5        U$ s  snf s  snnf ! , (       d  f       U$ = f)z
Return a dict mapping of a channel URL to a `Shard` or `ShardLike` object.

Attempt to fetch the sharded index first and then fall back to retrieving
a traditional `repodata.json` file.
c              3     #    U  H:  nUR                  S [        R                  5        H  nU[        U5      4v   M     M<     g7f)TN)urlsr   subdirsr   ).0channelchannel_urls      r    	<genexpr>!fetch_channels.<locals>.<genexpr>  s=      G"<<goo>K 
gk*+> 	,s   AAr   z/repodata.jsonN)r   r   r   r'  r   r  r  rp   r(  r   r   r   r!   ro   r   r6  r	   r   r   r   fetch_latest_parsedr   )channelscurl_to_channelchannel_datar   r   rL  rK  r   futures_non_shardedr   foundrepodata_jsonrq   rC   s                  r    fetch_channelsrW    s    %--Hq
HH-   N *,L ##D)@)@)Q)Q)S$TUE 
			.	.;N;P	.	QU] +9*>*>*@
*@& OO.
70CUK[X*@ 	 
 ! ((55g>F!/KMMOE,1[)  	 $OO"7;#78CCWW ? !((556IJF-5K%}}M
 !M0Cm1E(-% K+ 
R@ k .,
 
R	Q@ s$   F;G.-G CG G
G)rv   int)r*   r   )r2   r'   rv   r'   )r<   r   rv   r}   rw   )r   r   rv   r(   r   )r1  r	   r   zshards_cache.ShardCache | Nonerv   zShards | None)r;  zlist[Shards]r6   z	list[str])r=  zlist[tuple[Shards, str, str]])rP  zIterable[Channel | str]rv   zdict[str, ShardBase])Dr   
__future__r   r   concurrent.futuresr   	functoolsr#  loggingcollectionsr   pathlibr   typingr   urllib.parser   conda.gateways.repodatar  r   r   conda.base.contextr   conda.core.subdir_datar	   !conda.gateways.connection.sessionr
   r   r   conda.models.channelr   libmambapy.bindingsr   requestsr   r   r   	getLoggerr~   r   collections.abcr   r   r   r   #conda_libmamba_solver.shards_typingr   r   shards_typingr   r   r   r  r!   r-   r   r4   r?   ABCrA   r   r   r   r  r6  rB  rE  rW  r   r   r    <module>rm     s  
 # 
     #         & - 9 ) %  ! 25!Q;   &$   Z Zz\L	 \L~7V/Y V/r3n =AKK9KK\B%"=r   