Ë
    \›¾j÷e  ã                   ó*  — d Z ddlmZmZ ddlZddlZddlZddlZddlZddl	m
Z
 ddlZddlZddlZddlZddlmZ ddlmZ ddlmZmZmZmZmZmZmZ ddlmZmZ dd	lmZ dd
l m!Z!  ejD                  d«      Z# e$ e
jJ                  «       «      Z&dZ'dZ( ejR                  de(dz   «      Z* ejR                  de!«      Z+ ejR                  dd«      Z, ejR                  dejZ                  j]                  e&d«      «      Z/dZ0 G d„ de1«      Z2 G d„ de3«      Z4 G d„ de3«      Z5 edg d¢«      Z6d„ Z7d„ Z8d „ Z9d!„ Z:d"„ Z;d6d#„Z<d7d$„Z=d8d&„Z>d'„ Z?d(„ Z@d9d)„ZAd*„ ZBd+„ ZCd,„ ZDd-„ ZEe/e*de+ddfd.„ZFe/dfd/„ZGd0„ ZHe/e*de+dfd1„ZId2„ ZJe/e+e,dd%fd3„ZKd4e/di dde*de+e,dd%fd5„ZLy):z(
Common utilities for Stanza resources.
é    )ÚdefaultdictÚ
namedtupleN)ÚPath)Útqdm)Ú
make_table)ÚTOKENIZEÚMWTÚPOSÚLEMMAÚDEPPARSEÚNERÚ	SENTIMENT)ÚPIPELINE_NAMESÚPROCESSOR_VARIANTS)ÚPACKAGES)Ú__resources_version__Ústanzaz:https://nlp.stanford.edu/software/stanza/stanza-resources/z?https://raw.githubusercontent.com/stanfordnlp/stanza-resources/ÚSTANZA_RESOURCES_URLÚmainÚSTANZA_RESOURCES_VERSIONÚSTANZA_MODEL_URLÚdefaultÚSTANZA_RESOURCES_DIRÚstanza_resources)ÚpretrainÚforward_charlmÚbackward_charlmc                   ó   ‡ — e Zd Zˆ fd„Zˆ xZS )ÚResourcesFileNotFoundErrorc                 ó:   •— t         ‰| �  d|› d�«       || _        y )NzResources file not found at: z"  Try to download the model again.)ÚsuperÚ__init__Úresources_filepath)Úselfr#   Ú	__class__s     €úa/home/spazio/kingdom_calls/work5/transenv/lib/python3.12/site-packages/stanza/resources/common.pyr"   z#ResourcesFileNotFoundError.__init__-   s%   ø€ Ü‰ÑÐ8Ð9KÐ8LÐLnÐoÔpØ"4ˆÕó    ©Ú__name__Ú
__module__Ú__qualname__r"   Ú__classcell__©r%   s   @r&   r   r   ,   s   ø„ ÷5ð 5r'   r   c                   ó   ‡ — e Zd Zˆ fd„Zˆ xZS )ÚUnknownLanguageErrorc                 ó8   •— t         ‰| �  d|› �«       || _        y )NzUnknown language requested: )r!   r"   Úunknown_language©r$   Úunknownr%   s     €r&   r"   zUnknownLanguageError.__init__2   s    ø€ Ü‰ÑÐ7¸°yÐAÔBØ 'ˆÕr'   r(   r-   s   @r&   r/   r/   1   s   ø„ ÷(ð (r'   r/   c                   ó   ‡ — e Zd Zˆ fd„Zˆ xZS )ÚUnknownProcessorErrorc                 ó8   •— t         ‰| �  d|› �«       || _        y )Nz"Unknown processor type requested: )r!   r"   Úunknown_processorr2   s     €r&   r"   zUnknownProcessorError.__init__7   s    ø€ Ü‰ÑÐ=¸g¸YÐGÔHØ!(ˆÕr'   r(   r-   s   @r&   r5   r5   6   s   ø„ ÷)ð )r'   r5   ÚModelSpecification©Ú	processorÚpackageÚdependenciesc                 ó<   — t        | «      j                  dd¬«       y)z/
    Create dir in case it does not exist.
    T)ÚparentsÚexist_okN)r   Úmkdir)Úpaths    r&   Ú
ensure_dirrB   =   s   € ô 	ˆƒJ×Ñ˜T¨DÐÕ1r'   c                 óö   — 	 t        | d«      5 }|j                  «       }ddd«       t	        j
                  «      j                  «       S # 1 sw Y   Œ,xY w# t        $ r}|j                  s| |_        ‚ d}~ww xY w)z&
    Get the MD5 value of a path.
    ÚrbN)ÚopenÚreadÚOSErrorÚfilenameÚhashlibÚmd5Ú	hexdigest)rA   ÚfinÚdataÚes       r&   Úget_md5rO   C   sm   € ðÜ�$˜Óð 	 Ø—8‘8“:ˆD÷	ô �;‰;�tÓ×&Ñ&Ó(Ð(÷	ð 	ûäò Ø�zŠzØˆAŒJØûðús-   ‚A ŽA
ŸA Á
AÁA Á	A8ÁA3Á3A8c                 óî   — t         j                  d| › d|› d�«       t        j                  t        j
                  j                  | |«      «      5 }|j                  | «       ddd«       y# 1 sw Y   yxY w)zD
    Fully unzip a file `filename` that's in a directory `dir`.
    zUnzip: ú/z...N)ÚloggerÚdebugÚzipfileÚZipFileÚosrA   ÚjoinÚ
extractall)rA   rH   Úfs      r&   ÚunziprZ   P   s]   € ô ‡L�L�7˜4˜&  ( ¨3Ð/Ô0Ü	�‰œŸ™Ÿ™ d¨HÓ5Ó	6ð ¸!Ø	�‰�TÔ÷÷ ñ ús   ÁA+Á+A4c                 óà   — t        j                  | d«      }t        |j                  «      dkD  sJ d| › d�«       ‚t        j
                  j                  |j                  d   j                  «      S )z:
    Get the root directory from a archived zip file.
    Úrr   zZip file at fz( seems to be corrupted. Please check it.)rT   rU   ÚlenÚfilelistrV   rA   ÚdirnamerH   )rH   Úzfs     r&   Úget_root_from_zipfilera   X   sb   € ô 
�‰˜ 3Ó	'€BÜˆr�{‰{Ó˜aÒð KØ
˜�zÐ!IÐJóKÐä�7‰7�?‰?˜2Ÿ;™; q™>×2Ñ2Ó3Ð3r'   c                 ó`   — t         j                  j                  | «      xr t        | «      |k(  S )zN
    Check if the file at `path` exists and match the provided md5 value.
    )rV   rA   ÚexistsrO   )rA   rJ   s     r&   Úfile_existsrd   a   s%   € ô �7‰7�>‰>˜$ÓÒ8¤G¨D£M°SÑ$8Ð8r'   c                 ó  — t         j                  j                  | «      st        t        j
                  d| «      ‚|rCt        | «      }||k7  r2||k(  rt        j                  d| ||«       y t        d| ›d|›d|›�«      ‚y y )NzCannot find expected filez?Found a possibly older version of file %s, md5 %s instead of %szmd5 for z is z, expected )
rV   rA   rc   ÚFileNotFoundErrorÚerrnoÚENOENTrO   rR   rS   Ú
ValueError)rA   rJ   Úalternate_md5Úfile_md5s       r&   Úassert_file_existsrl   g   st   € Ü�7‰7�>‰>˜$ÔÜ¤§¡Ð.IÈ4ÓPÐPÙ
Ü˜4“=ˆØ�sŠ?Ø˜=Ò(Ü—‘Ð^Ð`dÐfsÐuxÕyå ÂDÊ(ÑTWÐ!XÓYÐYð	 ð r'   c                 ó8  — t         j                  dv }t        j                  | d|¬«      }|r|j	                  «        t        |d«      5 }t        |j                  j                  d«      «      }d}d| z   }	t        |dd| |	¬	«      5 }
|j                  |¬
«      D ]@  }|sŒ|j                  |«       |j                  «        |
j                  t        |«      «       ŒB 	 ddd«       ddd«       |j                  S # 1 sw Y   ŒxY w# 1 sw Y   |j                  S xY w)z<
    Download a URL into a file as specified by `path`.
    )r   é
   é   T)ÚstreamÚproxiesÚwbzcontent-lengthi   zDownloading ÚB)ÚtotalÚunitÚ
unit_scaleÚdisableÚdesc)Ú
chunk_sizeN)rR   ÚlevelÚrequestsÚgetÚraise_for_statusrE   ÚintÚheadersr   Úiter_contentÚwriteÚflushÚupdater]   Ústatus_code)ÚurlrA   rq   r}   Úverboser\   rY   Ú	file_sizeÚdefault_chunk_sizerx   ÚpbarÚchunks               r&   Údownload_filer‹   r   sþ   € ô �l‰l˜kÐ)€GÜ�‰�S ¨wÔ7€AÙØ	×ÑÔÜ	ˆd�DÓ	ð 
,˜QÜ˜Ÿ	™	Ÿ™Ð&6Ó7Ó8ˆ	Ø#ÐØ Ñ#ˆÜ˜	¨¸Ø�K dô,ð 	,Ø/3ØŸ™Ð3E˜ÓFò ,�ÚØ—G‘G˜E”NØ—G‘G”IØ—K‘K¤ E£
Õ+ñ	,÷	,÷	
,ð �=‰=Ð÷	,ð 	,ú÷	
,ð �=‰=Ðús*   Á	=DÂC9Â=C9ÃDÃ9D	Ã>DÄDTc                 ób  — t        |«      j                  }t        |«       t        ||«      r4|rt        j                  d|› �«       yt        j                  d|› �«       yt        j                  |¬«      5 }t        j                  j                  |t        j                  j                  |«      d   «      }	t        | |	||«       t        j                  |	|«       ddd«       t        |||«       |rt        j                  d|› �«       yt        j                  d|› �«       y# 1 sw Y   ŒJxY w)a  
    A complete wrapper over download_file() that also make sure the directory of
    `path` exists, and that a file matching the md5 value does not exist.

    alternate_md5 allows for an alternate md5 that is acceptable (such as if an older version of a file is okay)
    zFile exists: N)ÚdiréÿÿÿÿzDownloaded file to )r   ÚparentrB   rd   rR   ÚinforS   ÚtempfileÚTemporaryDirectoryrV   rA   rW   Úsplitr‹   Úreplacerl   )
r…   rA   rq   rJ   r}   Úlog_inforj   ÚbasedirÚtempÚtemppaths
             r&   Úrequest_filer™   ‡   sû   € ô �4‹j×Ñ€GÜˆwÔÜ�4˜ÔÙÜ�K‰K˜-¨ vÐ.Ô/ð 	ô �L‰L˜=¨¨Ð/Ô0Øô 
×	$Ñ	$¨Ô	1ð #°TÜ—7‘7—<‘< ¤b§g¡g§m¡m°DÓ&9¸"Ñ&=Ó>ˆÜ�c˜8 WÐ.>Ô?Ü
�
‰
�8˜TÔ"÷#ô �t˜S -Ô0ÙÜ�‰Ð)¨$¨Ð0Õ1ä�‰Ð*¨4¨&Ð1Õ2÷#ð #ús   Á7A%D%Ä%D.c                 óº   — g }t         D ]#  }| D ]  }|d   |k(  sŒ|j                  |«       Œ Œ% | D ]'  }|D ]  }|d   |d   k(  sŒ Œ |j                  «       Œ) |S )Nr   )r   Úappend)Úprocessor_listÚsorted_listr:   Úitems       r&   Úsort_processorsrŸ   ¥   sŠ   € Ø€KÜ#ò )ˆ	Ø"ò 	)ˆDØ�A‰w˜)Ó#Ø×"Ñ" 4Õ(ñ	)ð)ð $ò %ˆ	Øò 	%ˆDØ˜‰|˜t A™wÓ&Ùð	%ð ×Ñ˜tÕ$ð%ð Ðr'   c                 ó2  — | t            }|||   t           v r6t        ||   t           |   v r!t        j	                  d||«       || t        <   y|||   t            v r<t        ||   v r0|||   t           v r!t        j	                  d||«       || t        <   yyyy)a  Add mwt if tokenize is passed without mwt.

    If tokenize is in the list, but mwt is not, and there is a corresponding
    tokenize and mwt pair in the resources file, mwt is added so no missing
    mwt errors are raised.

    TODO: how does this handle EWT in English?
    z8Language %s package %s expects mwt, which has been addedN)r   r   r	   rR   Úwarning)Ú
processorsÚ	resourcesÚlangÚvalues       r&   Úadd_mwtr¦   ·   s©   € ð ”xÑ €EØ�	˜$‘¤Ñ)Ñ)¬c°Y¸t±_ÄXÑ5NÈuÑ5UÑ.UÜ�‰ÐQÐSWÐY^Ô_Øˆ
”3ŠØ
�9˜T‘?¤8Ñ,Ñ
,´¸	À$¹Ñ1GÈEÐU^Ð_cÑUdÔehÑUiÑLiÜ�‰ÐQÐSWÐY^Ô_Øˆ
”3Šð MjÐ1GÐ
,r'   c                 óà  — t        t        «      }|�rÜt        j                  d«       |rt        |v rt
        |vrt        || |«       |j                  «       D �]”  \  }}t        |t        «      st        d«      ‚t        |t        t        t        f«      st        d«      ‚t        |t        «      r|g}|t        vr|r|t        vrt        |«      ‚|D �]  }	|| |   v r;|	| |   |   v r1t        j                  d|› d|	› d�«       ||   j                  |	«       ŒF|	| |   t            v rb|| |   t            |	   v rQt        j                  d|› d| |   t            |	   |   › d�«       ||   j                  | |   t            |	   |   «       Œ¶|	| |   t            v r}d| |   t            |	   v rl|| |   t            |	   d   v rXt        j                  d|› d| |   t            |	   d   |   › d�«       ||   j                  | |   t            |	   d   |   «       �ŒA|	t"        |   v r8t        j                  d|› d|	› d|	› d	|› d
�	«       ||   j                  |	«       �Œ„|t$        k(  r7|	dk(  r2t        j                  d|› d|	› d�«       ||   j                  |	«       �ŒÄ|| |   vr1t        j                  |› d|	› d�«       ||   j                  |	«       �Œüt        j'                  d|› d|	› d�«       �Œ �Œ— |�r%t        j                  d«       t         | |   v ro|| |   t            v ra| |   t            |   j                  «       D ]@  \  }}	|dk7  sŒ||vsŒt        j                  d|› d|	› d�«       ||   j                  |	«       ŒB n–d}
t        D ]p  }|| |   vrŒ|| |   |   v sŒd}
||vr1t        j                  d|› d|› d�«       ||   j                  |«       ŒMt        j                  |› d|› d|› d||   › d�«       Œr |
st        j'                  d|› d�«       |j                  «       D ���	cg c]#  \  }}||D �	cg c]  }	t)        ||	d¬«      ‘Œ c}	g‘Œ% }}}}	t+        |«      }|S c c}	w c c}	}}w )aà  
    Given a parsed resources file, language, and possible package
    and/or processors, expands the package to the list of processors

    Returns a list of processors
    Each item in the list of processors is a pair:
      name, then a list of ModelSpecification
    so, for example:
      [['pos', [ModelSpecification(processor='pos', package='gsd', dependencies=None)]],
       ['depparse', [ModelSpecification(processor='depparse', package='gsd', dependencies=None)]]]
    z$Processing parameter "processors"...zProcessor names must be stringsz Processor values must be stringszFound ú: ú.Úoptionalz. Using external z variant for the z processor.Úidentityz. Using identity lemmatizer.z: is not officially supported by Stanza, loading it anyway.zCan not find z' from official model list. Ignoring it.z!Processing parameter "package"...FTz is overwritten by zCan not find package: Nr9   )r   ÚlistrR   rS   r   r	   r¦   ÚitemsÚ
isinstanceÚstrri   Útupler   ÚPRETRAIN_NAMESr5   r›   r   r   r   r¡   r8   rŸ   )r£   r¤   r;   r¢   Úallow_pretrainÚmaybe_add_mwtrœ   ÚkeyÚplistr¥   Úflags              r&   Úmaintain_processor_listr·   È   s=  € ô !¤Ó&€NâÜ�‰Ð;Ô=ÙœX¨Ñ3¼À:Ñ8MÜ�J 	¨4Ô0Ø$×*Ñ*Ó,ó 4	‰JˆC�Ü˜c¤3Ô'Ü Ð!BÓCÐCÜ˜e¤e¬T´3Ð%7Ô8Ü Ð!CÓDÐDÜ˜%¤Ô%Ø˜�Øœ.Ñ(Ù%¨´NÑ)BÜ/°Ó4Ð4Øó *�à˜) D™/Ñ)¨e°yÀ±ÀsÑ7KÑ.KÜ—L‘L 6¨#¨¨b°°°qÐ!9Ô:Ø" 3Ñ'×.Ñ.¨uÕ5à˜i¨™o¬hÑ7Ñ7¸CÀ9ÈTÁ?ÔS[ÑC\Ð]bÑCcÑ<cÜ—L‘LØ    R¨	°$©¼Ñ(AÀ%Ñ(HÈÑ(MÐ'NÈaÐPôð # 3Ñ'×.Ñ.¨y¸©¼xÑ/HÈÑ/OÐPSÑ/TÕUà˜i¨™o¬hÑ7Ñ7¸JÈ)ÐTXÉ/ÔZbÑJcÐdiÑJjÑ<jÐorÐvð  AEñ  wFô  GOñ  wPð  QVñ  wWð  Xbñ  wcñ  pcÜ—L‘LØ    R¨	°$©¼Ñ(AÀ%Ñ(HÈÑ(TÐUXÑ(YÐ'ZÐZ[Ð\ôð # 3Ñ'×.Ñ.¨y¸©¼xÑ/HÈÑ/OÐPZÑ/[Ð\_Ñ/`ÖaàÔ0°Ñ5Ñ5Ü—L‘LØ    R¨ wð /*Ø*/¨Ð0AÀ#ÀÀkðSôð # 3Ñ'×.Ñ.¨uÖ5àœE’\ e¨zÒ&9Ü—L‘LØ    R¨ wÐ.JÐKôð # 3Ñ'×.Ñ.¨uÖ5à 	¨$¡Ñ/Ü—L‘LØ˜%˜r % ð )-ð .ôð # 3Ñ'×.Ñ.¨uÖ5ô —N‘NØ'¨ u¨B¨u¨gð 6'ð (öòO*ð4	òl Ü�‰Ð8Ô:Ü�y ‘Ñ&¨7°iÀ±oÄhÑ6OÑ+OØ'¨™o¬hÑ7¸Ñ@×FÑFÓHò 6‘
��UØ˜*Ó$¨°NÒ)BÜ—L‘L 6¨#¨¨b°°°qÐ!9Ô:Ø" 3Ñ'×.Ñ.¨uÕ5ñ6ð
 ˆDÜ%ò �Ø˜i¨™oÑ-¨xØ˜i¨™o¨cÑ2Ò2Ø�DØ .Ñ0ÜŸ™ v¨c¨U°"°W°I¸QÐ%?Ô@Ø& sÑ+×2Ñ2°7Õ;äŸ™Ø"˜e 2 g YÐ.AØ"˜e 2 j°¡oÐ%6°að9õðñ œŸ™Ð+AÀ'ÀÈ!Ð)LÔNð IW÷  I]ñ  I]ó  I_÷  `ð  `ñ  {EÐz}ð  @E�sÐnsÖtÐejÔ/¸#ÀuÐ[_Ö`ÒtÒuð  `€Nò  `Ü$ ^Ó4€NØÐùò uùô  `s   Ð+Q)Ð8Q$ÑQ)Ñ$Q)c                 ó  — | |   }|D ]÷  }|\  }}g }|D ]Û  }t        |j                  t        |   v |t        k(  xr |j                  dk(  g«      s�|j	                  |i «      j	                  |j                  i «      j	                  dg «      }	|	D �
cg c]  }
|
d   |
d   f‘Œ }	}
|j                  t        |	«      ¬«      }t        j                  d|	||j                  «       |j                  |«       ŒÝ t        |«      |d<   Œù |S c c}
w )a¹  
    Expand the processor_list as given in maintain_processor_list to have the dependencies

    Still a list of model types to ModelSpecifications
    the dependencies are tuples: name and package
    for example:
    [['pos', (ModelSpecification(processor='pos', package='gsd', dependencies=(('pretrain', 'gsd'),)),)],
     ['depparse', (ModelSpecification(processor='depparse', package='gsd', dependencies=(('pretrain', 'gsd'),)),)]]
    r«   r<   Úmodelr;   )r<   z/Found dependencies %s for processor %s model %sé   )
Úanyr;   r   r   r|   Ú_replacer°   rR   rS   r›   )r£   r¤   rœ   Úlang_resourcesrž   r:   Úmodel_specsÚnew_model_specsÚ
model_specr<   Ú
dependencys              r&   Úadd_dependenciesrÂ   *  s-  € ð ˜t‘_€NØò )ˆØ!%Ñˆ	�;ØˆØ%ò 
	/ˆJäØ×&Ñ&Ô*<¸YÑ*GÐGØ¤Ñ&ÒK¨:×+=Ñ+=ÀÑ+Kðô ð  .×1Ñ1°)¸RÓ@×DÑDÀZ×EWÑEWÐY[Ó\×`Ñ`ÐaoÐqsÓt�Ø_kÖlÐQ[ ¨GÑ!4°jÀÑ6KÒ LÐl�ÐlØ'×0Ñ0¼eÀLÓ>QÐ0ÓR�
Ü—‘ÐNÐP\Ð^gÐis×i{Ñi{Ô|Ø×"Ñ" :Õ.ð
	/ô ˜Ó(ˆˆQŠð)ð Ðùò  ms   ÂDc           	      ót  — g }g }| D ]X  }|\  }}|D ]L  }|j                   }|j                  }|j                  ||g«       |sŒ1||D �	cg c]  }	t        |	«      ‘Œ c}	z  }ŒN ŒZ t	        |«      D �cg c]  }t        |«      ‘Œ }}|D ]!  \  }}t        j                  d|› d|› d�«       Œ# ||z  }|S c c}	w c c}w )zœ
    The flattened processor list is just a list of types & packages

    For example:
      [['pos', 'gsd'], ['depparse', 'gsd'], ['pretrain', 'gsd']]
    zFind dependency r¨   r©   )r;   r<   r›   r°   Úsetr¬   rR   rS   )
rœ   Úflattened_processor_listÚdependencies_listrž   r:   r¾   rÀ   r;   r<   rÁ   s
             r&   Úflatten_processor_listrÇ   F  sð   € ð  "ÐØÐØò XˆØ!%Ñˆ	�;Ø%ò 	XˆJØ ×(Ñ(ˆGØ%×2Ñ2ˆLØ$×+Ñ+¨Y¸Ð,@ÔAÚØ!È,Ö%W¸J¤e¨JÕ&7Ò%WÑWÑ!ñ	XðXô 14Ð4EÓ0FÖG¨œ˜d�ÐGÐÐGØ/ò AÑˆ	�7Ü�‰Ð'¨	 {°"°W°I¸QÐ?Õ@ðAàÐ 1Ñ1ÐØ#Ð#ùò &XùÚGs   ÁB0
Á/B5c                 óP  — |dk(  rd} n|dk(  rd} | €8t         j                  dk(  rt         j                  d«       t         j                  S | j                  «       } g d¢}| |vr!t	        d| › dd	j                  |«      › d
�«      ‚t         j                  | «       t         j                  S )NFÚERRORTÚINFOr   )ÚDEBUGrÊ   ÚWARNINGÚWARNrÉ   ÚCRITICALÚFATALz)Unrecognized logging level for pipeline: z. Must be one of ú, r©   )rR   rz   ÚsetLevelÚupperri   rW   )Úlogging_levelr†   Ú
all_levelss      r&   Úset_logging_levelrÕ   ]  s¨   € à�%ÒØ‰Ø	�DŠØˆàÐô �<‰<˜1ÒÜ�O‰O˜FÔ#Ü�|‰|Ðð "×'Ñ'Ó)€MÚS€JØ˜JÑ&ÜØ7ØˆoÐ.¨t¯y©y¸Ó/DÐ.EÀQðHó
ð 	
ô ‡O�O�MÔ"Ü�<‰<Ðr'   c                 ó�  ‡	— t        | t        «      r| j                  «       j                  «       } n$| �"t	        dt        | «      j                  › d�«      ‚t        |t        «      r|j                  «       }n$|�"t	        dt        |«      j                  › d�«      ‚t        |t        t        t        f«      rÐ|€t        d„ «      }nat        |t        «      r|Š	t        ˆ	fd„«      }n@t        |t        «      rt        d„ |«      }n"t	        dt        |«      j                  › d�«      ‚t        |t        «      r:|j                  d«      D �cg c]   }|j                  «       j                  «       ‘Œ" }}|D �ci c]  }|||   “Œ
 }}d }nÔt        |t        «      r |j                  «       D ���ci c]‚  \  }}|j                  «       j                  «       t        |t        t        f«      r,|D �cg c]   }|j                  «       j                  «       ‘Œ" c}n|j                  «       j                  «       “Œ„ }}}}n$|�"t	        d	t        |«      j                  › d�«      ‚t        |t        «      r|j                  «       j                  «       }n$|�"t	        d
t        |«      j                  › d�«      ‚| |||fS c c}w c c}w c c}w c c}}}w )Nz,The parameter 'lang' should be str, but got z	 instead.z1The parameter 'model_dir' should be str, but got c                   ó   — y©Nr   © rÙ   r'   r&   ú<lambda>z-process_pipeline_parameters.<locals>.<lambda>�  ó   � r'   c                  ó   •— ‰ S )NrÙ   )r   s   €r&   rÚ   z-process_pipeline_parameters.<locals>.<lambda>“  s   ø€ ¨'€ r'   c                   ó   — yrØ   rÙ   rÙ   r'   r&   rÚ   z-process_pipeline_parameters.<locals>.<lambda>—  rÛ   r'   z>The parameter 'package' should be None, str, or dict, but got ú,z:The parameter 'processors' should be dict or str, but got zSThe parameter 'package' should be str, or a dict if 'processors' is a str, but got )r®   r¯   ÚstripÚlowerÚ	TypeErrorÚtyper)   r¬   r°   r   Údictr“   r­   )
r¤   Ú	model_dirr;   r¢   Úxr:   ÚkÚvÚv_ir   s
            @r&   Úprocess_pipeline_parametersré   w  s®  ø€ ä�$œÔØ�z‰z‹|×!Ñ!Ó#‰Ø	Ð	ÜðÜ˜D“z×*Ñ*Ð+¨9ð6ó
ð 	
ô
 �)œSÔ!Ø—O‘OÓ%‰	Ø	Ð	ÜðÜ˜I“×/Ñ/Ð0°	ð;ó
ð 	
ô
 �*œs¤D¬%Ð0Ô1ð ˆ?ä!Ñ"3Ó4‰GÜ˜¤Ô%àˆGÜ!£/Ó2‰GÜ˜¤Ô&ô "Ñ"3°WÓ=‰GäðÜ ›=×1Ñ1Ð2°)ð=óð ô �j¤#Ô&Ø5?×5EÑ5EÀcÓ5JÖK°˜!Ÿ'™'›)Ÿ/™/Õ+ÐKˆJÐKà;Eö
Ø.7ˆI�w˜yÑ)Ñ)ð
ˆ
ð 
ð ‰Ü	�J¤Ô	%ð #×(Ñ(Ó*÷
ð 
á��1ð �G‰G‹I�O‰OÓÄjÐQRÔUZÔ\`ÐTaÔFbÀÖ B¸ §¡£×!2Ñ!2Õ!4Ó BÐhi×hoÑhoÓhq×hwÑhwÓhyÑyð
ˆ
ó 
ð 
Ð	ÜðÜ˜JÓ'×0Ñ0Ð1°ð<ó
ð 	
ô
 �'œ3ÔØ—-‘-“/×'Ñ'Ó)‰Ø	Ð	ÜðÜ˜G“}×-Ñ-Ð.¨ið9ó
ð 	
ð
 �˜G ZÐ/Ð/ùò1 Lùò
ùò !Cùô
s$   Å%J2Å3J7Æ)?KÇ(%J<È#KÊ<Kc                 óú   — |t         k(  r|�	t        |z   }|j                  «       dv rt        }|› d|› d�}t        j                  d|«       |€ t        j                  j                  | d«      }t        |||d¬«       y)	z=
    Downloads resources.json to obtain latest packages.
    N)ÚstanfordÚstanfordnlpz/resources_z.jsonz!Downloading resource file from %súresources.jsonT)r}   )
ÚDEFAULT_RESOURCES_URLÚSTANZA_RESOURCES_GITHUBrà   ÚSTANFORDNLP_RESOURCES_URLrR   rS   rV   rA   rW   r™   )rä   Úresources_urlÚresources_branchÚresources_versionr#   rq   s         r&   Údownload_resources_jsonrô   ¸  sˆ   € ð Ô-Ò-Ð2BÐ2NÜ/Ð2BÑBˆà×ÑÓÐ ;Ñ;Ü1ˆØ$�o [Ð1BÐ0CÀ5ÐI€MÜ
‡L�LÐ4°mÔDØÐ!ÜŸW™WŸ\™\¨)Ð5EÓFÐäØØØØö	r'   c                 ó  — |€ t         j                  j                  | d«      }t         j                  j                  |«      st	        |«      ‚t        |d¬«      5 }t        j                  |«      }ddd«       |S # 1 sw Y   S xY w)zA
    Unpack the resources json file from the given model_dir
    Nrí   zutf-8)Úencoding)rV   rA   rW   rc   r   rE   ÚjsonÚload)rä   r#   rL   r£   s       r&   Úload_resources_jsonrù   Ó  ss   € ð Ð!ÜŸW™WŸ\™\¨)Ð5EÓFÐÜ�7‰7�>‰>Ð,Ô-Ü(Ð);Ó<Ð<Ü	Ð ¨7Ô	3ð #°sÜ—I‘I˜c“Nˆ	÷#àÐ÷#àÐús   ÁA:Á:Bc                 ó@   — || vry| |   }d|v r|d   }| |   }d|v rŒ|S )zi
    Get the resources for a lang from an already loaded resources json, following 'alias' if needed
    NÚaliasrÙ   )r£   r¤   r½   s      r&   Úget_language_resourcesrü   ß  sE   € ð �9ÑØà˜t‘_€NØ
�^Ñ
#Ø˜gÑ&ˆØ" 4™ˆð �^Ò
#ð Ðr'   c                 ó°   — t        | |||d|¬«       t        | «      }|D �cg c]  }t        ||   t        «      s	d||   vr|‘Œ  }}t	        |«      }|S c c}w )z<
    List the non-alias languages in the resources file
    N©r#   rq   rû   )rô   rù   r®   r¯   Úsorted)rä   rñ   rò   ró   rq   r£   r¤   Ú	languagess           r&   Úlist_available_languagesr  í  s|   € ô ˜I }Ð6FÐHYÐnrð  }Dõ  EÜ# IÓ.€Ið #,ö ]˜$Ü" 9¨T¡?´CÔ8¸WÈIÐVZÉOÑ=[ò ð ]€Ið ]ä�yÓ!€IØÐùò]s   ¡#Ac                 ó6   — |j                  «       dk(  r| d   S |S )za
    Returns the url in the resources dict if model_url is default, or returns the model_url
    r   r…   )rà   )r£   Ú	model_urls     r&   Úexpand_model_urlr  þ  s"   € ð  )Ÿ™Ó0°IÒ=ˆ9�UÑÐLÀ9ÐLr'   c                 ó$  — |j                  |i «      j                  d|«      }t        ddg| «      }	|rt        j                  }
nt        j                  }
 |
d|› d|› d|	› �«       t        ||«      }| D ]w  \  }}	 t        |j                  |||› d|› d�¬	«      t        j                  j                  ||||› d�«      |||   |   |   d
   |||   |   |   j                  dd «      ¬«       Œy y # t        $ r}t        d|› d|› d�«      |‚d }~ww xY w)NÚ	lang_nameÚ	ProcessorÚPackagez4Downloading these customized packages for language: ú (z)...
rQ   z.pt©ró   r¤   rH   rJ   rj   )rJ   r•   rj   z@Cannot find the following processor and model name combination: rÐ   z;. Please check if you have provided the correct model name.)r|   r   rR   r�   rS   r  r™   ÚformatrV   rA   rW   ÚKeyErrorri   )Údownload_listr£   r¤   rä   ró   r  rq   r•   r  Údownload_tableÚlog_msgr…   r´   r¥   rN   s                  r&   Údownload_modelsr    sP  € ð —‘˜d BÓ'×+Ñ+¨K¸Ó>€IÜ ¨iÐ 8¸-ÓH€NÙÜ—+‘+‰ä—,‘,ˆÙØ
>Øˆ&��9�+˜V NÐ#3ð	5ôô
 ˜9 iÓ
0€Cð $ò ‰
ˆˆUð	ÜØ—
‘
Ð->ÀTÐWZÐV[Ð[\Ð]bÐ\cÐcfÐTg�
ÓhÜ—‘—‘˜Y¨¨c°e°W¸C°=ÓAØØ˜d‘O CÑ(¨Ñ/°Ñ6Ø!Ø'¨™o¨cÑ2°5Ñ9×=Ñ=¸oÈtÓTöñøô ò 	ÜØRØ�%�r˜%˜Ð [ð]óð ðûð	ús   Á:A1C.Ã.	DÃ7D
Ä
DÚenc           
      óì  — t        ||«       t        | |||«      \  } }}}|s=t        j                  j	                  t        j                  j                  |d«      «      s(|st        j                  d«       t        ||||d |
¬«       t        |«      }| |vrt        | «      ‚d||    v r*t        j                  d| › d||    d   › d�«       ||    d   } |j                  | i «      j                  d| «      }t        ||	«      }|dk(  r�|�t        |«      d	k(  r�t        j                  d
| › d|› d�«       t        |j!                  || d¬«      t        j                  j                  || d«      |
||    d   ¬«       t#        t        j                  j                  || «      d«       n;t%        || ||d¬«      }t'        || |«      }t)        |«      }t+        ||| |||	|
d¬«       t        j                  d|› �«       y )Nrí   zZAsked to skip downloading resources.json, but the file does not exist.  Downloading anywayrþ   rû   ú"z" is an alias for "r  r   r   z+Downloading default packages for language: r	  z) ...zdefault.zipr
  Údefault_md5)rJ   T)r²   )r  r£   r¤   rä   ró   r  rq   r•   z)Finished downloading models and saved to )rÕ   ré   rV   rA   rc   rW   rR   r¡   rô   rù   r/   r�   r|   r  r]   r™   r  rZ   r·   rÂ   rÇ   r  )r¤   rä   r;   r¢   rÓ   r†   rñ   rò   ró   r  rq   Údownload_jsonr£   r  r…   r  s                   r&   Údownloadr  +  s÷  € ô �m WÔ-ä+FØˆi˜ *ó,Ñ(€Dˆ)�W˜jñ œBŸG™GŸN™N¬2¯7©7¯<©<¸	ÐCSÓ+TÔUÙÜ�N‰NÐwÔxÜ 	¨=Ð:JÐL]Ðrvð  AHõ  	Iä# IÓ.€IØ�9ÑÜ" 4Ó(Ð(Ø�)˜D‘/Ñ!Ü�‰�a˜�vÐ0°¸4±ÀÑ1IÐ0JÈ!ÐLÔMØ˜‰˜wÑ'ˆØ—‘˜d BÓ'×+Ñ+¨K¸Ó>€IÜ
˜9 iÓ
0€Cð �)Ò Ð!3´s¸:³È!Ò7KÜ�‰Ø9¸$¸¸rÀ)ÀÈEÐRô	
ô 	Ø�J‰JÐ):ÀÐP]ˆJÓ^Ü�G‰G�L‰L˜ D¨KÓ9ØØ˜$‘ Ñ.õ		
ô 	Œb�g‰g�l‰l˜9 dÓ+¨]Õ;ô 0°	¸4ÀÈ*ÐeiÔjˆÜ(¨°D¸-ÓHˆÜ.¨}Ó=ˆÜ mØ"+Ø!Ø"+Ø*;Ø"+Ø 'Ø!%õ	'ô ‡K�KÐ;¸I¸;ÐGÕHr'   )NN)F)NNFTN)FT)MÚ__doc__Úcollectionsr   r   rg   rI   r÷   ÚloggingrV   Úpathlibr   r{   Úshutilr‘   rT   Ú	tqdm.autor   Ústanza.utils.helper_funcr   Ústanza.pipeline._constantsr   r	   r
   r   r   r   r   Ústanza.pipeline.registryr   r   Ú!stanza.resources.default_packagesr   Ústanza._versionr   Ú	getLoggerrR   r¯   ÚhomeÚHOME_DIRrð   rï   Úgetenvrî   ÚDEFAULT_RESOURCES_VERSIONÚDEFAULT_MODEL_URLrA   rW   ÚDEFAULT_MODEL_DIRr±   rf   r   ri   r/   r5   r8   rB   rO   rZ   ra   rd   rl   r‹   r™   rŸ   r¦   r·   rÂ   rÇ   rÕ   ré   rô   rù   rü   r  r  r  r  rÙ   r'   r&   ú<module>r)     sý  ðñ÷ 0Û Û Û Û Û 	Ý Û Û Û Û å å /ß Z× ZÑ Zß GÝ 6Ý 1à	ˆ×	Ñ	˜8Ó	$€ñ ˆyˆt�y‰y‹{Ó€ØXÐ Ø[Ð Ø!˜Ÿ	™	Ð"8Ð:QÐTZÑ:ZÓ[Ð Ø%˜BŸI™IØØóÐ ð �B—I‘IÐ0°)Ó<Ð Ø�B—I‘IØØ‡G�G‡L�L�Ð-Ó.óÐ ð
 C€ô5Ð!2ô 5ô
(˜:ô (ô
)˜Jô )ñ
  Ð 4Ò6^Ó_Ð ò2ò)òò4ò9ó	Zóó*3ò<ò$ ó"`òDò8$ò.ò4?0ðB '8Ø*?Ø-1Ø.GØ/3Ø$(óð6 #4Èó 
òð (9Ø+@Ø.2Ø/HØ%)ó	ò"Mð 0Ø&?Ø/Ø Ø!ó$ðP Ø#ØØØØØ+ØØ3Ø#ØØô@Ir'   