a b3j?@sdZddlmZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl mZmZmZddlmZddlmZddlmZddlmZmZmZdd lmZdd lmZdd lm Z m!Z!dd l"m#Z#dd l$m%Z%ddl&m'Z'ddl(m)Z)ddl*m+Z+ddl,m-Z-ddl.m/Z/ddl0m1Z1ddl2m3Z3ddl4m5Z5m6Z6m7Z7e 8e9Z:ee;e;fZZ?ddd d!d"Z@Gd#d$d$e>ZAdd%dd&d'd(ZBdd%dd&d)d*ZCd+dd,d-d.ZDGd/d0d0ZEGd1d2d2eZFd2d2d3d4d5ZGeGd6d7d8d9d:ZHed;d<Gd=d6d6ZIGd>d?d?eZJdRd@dAdBddCdDdEZKdSddFd6dGdHdIZLd@d%dJdKdLdMZMGdNdOdOeZNGdPdQdQZOdS)TzO The main purpose of this module is to expose LinkCollector.collect_sources(). ) annotationsN)IterableMutableMappingSequence) dataclass) HTMLParser)Values)Callable NamedTupleProtocol)requests)Response) RetryErrorSSLError)NetworkConnectionError)Link) SearchScope) PipSession)raise_for_status)is_archive_fileredact_auth_from_url) url_to_path)vcs)CandidatesFromPage LinkSource build_sourcestr str | NoneurlreturncCs6tjD]*}||r|t|dvr|SqdS)zgLook for VCS schemes in the URL. Returns the matched VCS scheme, or None if there's no match. z+:N)rschemeslower startswithlen)r!schemer(z/www/wwwroot/dpstar/app/297b3aabda72fedb274352021c2dd8b5_venv/lib/python3.9/site-packages/pip/_internal/index/collector.py_match_vcs_scheme.s  r*cs&eZdZddddfdd ZZS)_NotAPIContentrNone) content_type request_descr"cst||||_||_dSN)super__init__r-r.)selfr-r. __class__r(r)r1:sz_NotAPIContent.__init__)__name__ __module__ __qualname__r1 __classcell__r(r(r3r)r+9sr+r r,)responser"cCs6|jdd}|}|dr$dSt||jjdS)z Check the Content-Type header to ensure the response contains a Simple API Response. Raises `_NotAPIContent` if the content type is not a valid content-type. Content-TypeUnknown)z text/htmlz#application/vnd.pypi.simple.v1+html#application/vnd.pypi.simple.v1+jsonN)headersgetr$r%r+requestmethod)r9r-content_type_lr(r(r)_ensure_api_header@srBc@s eZdZdS)_NotHTTPN)r5r6r7r(r(r(r)rCVsrCr)r!sessionr"cCsFtj|\}}}}}|dvr$t|j|dd}t|t|dS)z Send a HEAD request to the URL, and ensure the response contains a simple API Response. Raises `_NotHTTP` if the URL is not available for a HEAD request, or `_NotAPIContent` if the content type is not a valid content type. >httpshttpT)allow_redirectsN)urllibparseurlsplitrCheadrrB)r!rDr'netlocpathqueryfragmentrespr(r(r)_ensure_api_responseZs rQcCsxtt|jrt||dtdt||j|dgdddd}t |t |tdt||j d d |S) aYAccess an Simple API response with GET, and return the response. This consists of three parts: 1. If the URL looks suspiciously like an archive, send a HEAD first to check the Content-Type is HTML or Simple API, to avoid downloading a large file. Raise `_NotHTTP` if the content type cannot be determined, or `_NotAPIContent` if it is not HTML or a Simple API. 2. Actually perform the request. Raise HTTP exceptions on network failures. 3. Check the Content-Type header to make sure we got a Simple API response, and raise `_NotAPIContent` otherwise. rDzGetting page %sz, )r<z*application/vnd.pypi.simple.v1+html; q=0.1ztext/html; q=0.01z max-age=0)Acceptz Cache-Control)r=zFetched page %s as %sr:r;) rrfilenamerQloggerdebugrr>joinrrBr=)r!rDrPr(r(r)_get_simple_responsels&   rXResponseHeaders)r=r"cCs<|r8d|vr8tj}|d|d<|d}|r8t|SdS)z=Determine if we have any encoding information in our headers.r:z content-typecharsetN)emailmessageMessage get_paramr)r=mrZr(r(r)_get_encoding_from_headerss    r`c@s:eZdZdddddZdddd d Zd d d dZdS)CacheablePageContent IndexContentr,pager"cCs|js J||_dSr/)cache_link_parsingrdr2rdr(r(r)r1s zCacheablePageContent.__init__objectbool)otherr"cCst|t|o|jj|jjkSr/) isinstancetyperdr!)r2rir(r(r)__eq__szCacheablePageContent.__eq__intr"cCs t|jjSr/)hashrdr!r2r(r(r)__hash__szCacheablePageContent.__hash__N)r5r6r7r1rlrqr(r(r(r)rasrac@seZdZdddddZdS) ParseLinksrbIterable[Link]rccCsdSr/r(rfr(r(r)__call__zParseLinks.__call__N)r5r6r7rtr(r(r(r)rrsrr)fnr"cs>tjdddfdd tdddfdd }|S) z Given a function that parses an Iterable[Link] from an IndexContent, cache the function's result (keyed by CacheablePageContent), unless the IndexContent `page` has `page.cache_link_parsing == False`. raz list[Link])cacheable_pager"cst|jSr/)listrd)rw)rvr(r)wrappersz*with_cached_index_content..wrapperrbrccs|jrt|St|Sr/)rerarx)rdrvryr(r)wrapper_wrappers z2with_cached_index_content..wrapper_wrapper) functoolscachewraps)rvr{r(rzr)with_cached_index_contents rrbrsrcc cs|j}|drTt|j}|dgD]"}t||j }|durHq,|Vq,dSt |j }|j pfd}| |j ||j }|jp|}|jD]$} tj| ||d}|durq|VqdS)z\ Parse a Simple API's Index Content, and yield its anchor elements as Link objects. r<filesNzutf-8)page_urlbase_url)r-r$r%jsonloadscontentr>r from_jsonr!HTMLLinkParserencodingfeeddecoderanchorsZ from_element) rdrAdatafilelinkparserrr!ranchorr(r(r) parse_linkss&       rT)frozenc@sLeZdZUdZded<ded<ded<ded<d Zd ed <dd d dZdS)rbaRepresents one response (or page), along with its URL. :param encoding: the encoding to decode the given content. :param url: the URL from which the HTML was downloaded. :param cache_link_parsing: whether links parsed from this page's url should be cached. PyPI index urls should have this set to False, for example. bytesrrr-rrr!TrhrerncCs t|jSr/)rr!rpr(r(r)__str__szIndexContent.__str__N)r5r6r7__doc____annotations__rerr(r(r(r)rbs   csJeZdZdZdddfdd Zddddd d Zdd d d dZZS)rzf HTMLParser that keeps the first base HREF and a list of all anchor elements' attributes. rr,r cs$tjdd||_d|_g|_dS)NT)convert_charrefs)r0r1r!rr)r2r!r3r(r)r1szHTMLLinkParser.__init__zlist[tuple[str, str | None]])tagattrsr"cCsH|dkr,|jdur,||}|durD||_n|dkrD|jt|dS)Nbasea)rget_hrefrappenddict)r2rrhrefr(r(r)handle_starttags  zHTMLLinkParser.handle_starttagr)rr"cCs"|D]\}}|dkr|SqdS)Nrr()r2rnamevaluer(r(r)r!s  zHTMLLinkParser.get_href)r5r6r7rr1rrr8r(r(r3r)r srrzstr | ExceptionzCallable[..., None] | None)rreasonmethr"cCs|durtj}|d||dS)Nz%Could not fetch URL %s: %s - skipping)rUrV)rrrr(r(r)_handle_get_simple_fail(srrh)r9rer"cCs&t|j}t|j|jd||j|dS)Nr:)rr!re)r`r=rbrr!)r9rerr(r(r)_make_index_content2s rIndexContent | None)rrDr"c Cs|jddd}t|}|r0td||dS|drvtjt |rv| ds\|d7}t j |d}td|zt||d }WnLtytd |Yn<ty}z"td ||j|jWYd}~n d}~0ty}zt||WYd}~nd}~0ty<}zt||WYd}~nd}~0ty}z,d }|t|7}t||tjd WYd}~nld}~0tjy}zt|d|WYd}~n6d}~0tjyt|dYn0t||jdSdS)N#rrzICannot look at %s URL %s because it does not support lookup as web pages.zfile:/z index.htmlz# file: URL is directory, getting %srRz`Skipping page %s because it looks like an archive, and cannot be checked by a HTTP HEAD request.zSkipping page %s because the %s request got Content-Type: %s. The only supported Content-Types are application/vnd.pypi.simple.v1+json, application/vnd.pypi.simple.v1+html, and text/htmlz4There was a problem confirming the ssl certificate: )rzconnection error: z timed out)re)r!splitr*rUwarningr%osrMisdirrendswithrHrIurljoinrVrXrCr+r.r-rrrrrinfor ConnectionErrorTimeoutrre)rrDr! vcs_schemerPexcrr(r(r)_get_index_content?sT     $$rc@seZdZUded<ded<dS)CollectedSourceszSequence[LinkSource | None] find_links index_urlsN)r5r6r7rr(r(r(r)r{s rc@speZdZdZddddddZeddd d dd d d ZeddddZdddddZ ddddddZ dS) LinkCollectorz Responsible for collecting Link objects from all configured locations, making network requests as needed. The class's main method is its collect_sources() method. rrr,)rD search_scoper"cCs||_||_dSr/)rrD)r2rDrr(r(r)r1szLinkCollector.__init__Frrh)rDoptionssuppress_no_indexr"cCsd|jg|j}|jr8|s8tdddd|Dg}|jp@g}tj|||jd}t ||d}|S)z :param session: The Session to use to make requests. :param suppress_no_index: Whether to ignore the --no-index option when constructing the SearchScope object. zIgnoring indexes: %s,css|]}t|VqdSr/r).0r!r(r(r) ruz'LinkCollector.create..)rrno_index)rDr) index_urlextra_index_urlsrrUrVrWrrcreater)clsrDrrrrrlink_collectorr(r(r)rs$   zLinkCollector.createz list[str]rncCs|jjSr/)rrrpr(r(r)rszLinkCollector.find_linksrr)locationr"cCst||jdS)z> Fetch an HTML page containing package links. rR)rrD)r2rr(r(r)fetch_responseszLinkCollector.fetch_responserrr) project_namecandidates_from_pager"cstfddjD}tfddjD}ttj rddt ||D}t |ddg|}t d|tt|t|d S) Nc 3s&|]}t|jjdddVqdS)Frpage_validator expand_dirrerNrrDis_secure_originrlocrrr2r(r)rs z0LinkCollector.collect_sources..c 3s&|]}t|jjdddVqdS)TrNrrrr(r)rs cSs*g|]"}|dur|jdurd|jqS)Nz* )r)rsr(r(r) sz1LinkCollector.collect_sources..z' location(s) to search for versions of : )rr) collections OrderedDictrget_index_urls_locationsvaluesrrU isEnabledForloggingDEBUG itertoolschainr&rVrWrrx)r2rrindex_url_sourcesfind_links_sourceslinesr(rr)collect_sourcess*     zLinkCollector.collect_sourcesN)F) r5r6r7rr1 classmethodrpropertyrrrr(r(r(r)rs!r)N)T)Pr __future__rr email.messager[r|rrrr urllib.parserHcollections.abcrrrZ dataclassesr html.parserroptparsertypingr r r pip._vendorr Zpip._vendor.requestsr Zpip._vendor.requests.exceptionsrrpip._internal.exceptionsrpip._internal.models.linkr!pip._internal.models.search_scoperpip._internal.network.sessionrpip._internal.network.utilsrpip._internal.utils.filetypesrpip._internal.utils.miscrpip._internal.utils.urlsrpip._internal.vcsrsourcesrrr getLoggerr5rUrrYr* Exceptionr+rBrCrQrXr`rarrrrrbrrrrrrr(r(r(r)sf                  ?    <