Ë
    \›¾jU   ã                   ó®   — d Z ddlmZ ddlZddlZddlmZ ddlmZ ddl	m
Z
 ddlmZmZ ddl­ dd	lmZmZmZ  ej&                  d
«      Z G d„ d«      Zy)z*
Class for running multilingual pipelines
é    )ÚOrderedDictN)ÚUnion)ÚDocument)Údefault_device)ÚPipelineÚDownloadMethod)Ú*)ÚDEFAULT_MODEL_DIRÚget_language_resourcesÚload_resources_jsonÚstanzac                   ó†   — e Zd ZdZedddddddej                  df
dededed	e	d
e	de
de
dededeeef   fd„Zd„ Zd„ Zd„ Zy)ÚMultilingualPipelinea‰  
    Pipeline for handling multilingual data. Takes in text, detects language, and routes request to pipeline for that
    language.

    You can specify options to individual language pipelines with the lang_configs field.
    For example, if you want English pipelines to have NER, but want to turn that off for French, you can do:
        lang_configs = {"en": {"processors": "tokenize,pos,lemma,depparse,ner"},
                        "fr": {"processors": "tokenize,pos,lemma,depparse"}}
        pipeline = MultilingualPipeline(lang_configs=lang_configs)

    You can also pass in a defaultdict created in such a way that it provides default parameters for each language.
    For example, in order to only get tokenization for each language:
    (remembering that the Pipeline will automagically add MWT to a language which uses MWT):
        from collections import defaultdict
        lang_configs = defaultdict(lambda: dict(processors="tokenize"))
        pipeline = MultilingualPipeline(lang_configs=lang_configs)

    download_method can be set as in Pipeline to turn off downloading
      of the .json config or turn off downloading of everything
    Né@   é
   FÚ	model_dirÚlang_id_configÚlang_configsÚld_batch_sizeÚmax_cache_sizeÚuse_gpuÚrestrictÚdeviceÚdownload_methodÚ
processorsc                 óú  — || _         |€i nt        j                  |«      | _        |€i nt        j                  |«      | _        || _        t        «       | _        |
€d | _        nSt        |
t        «      r3|
j                  d«      D �cg c]  }|j                  «       ‘Œ c}| _        nt        |
«      | _        |	| _        d| j                  vr| j                  | j                  d<   | j                  D ]&  }d| j                  |   vsŒ|| j                  |   d<   Œ( |rqd| j                  vrct        | j                  j!                  «       «      }|dk(  rt"        j%                  d«       n%t"        j'                  d|«       || j                  d<   |€|�|dk(  rt)        «       }nd	}|| _        t-        d| j                   d
d| j*                  dœ| j                  ¤Ž| _        t1        | j                   «      | _        y c c}w )Nú,r   ÚlangÚlangid_lang_subsetr   z`MultilingualPipeline asked to restrict to lang_configs, but lang_configs was empty.  Ignoring...z&Restricting MultilingualPipeline to %sTÚcpuÚmultilingualÚlangid)Údirr   r   r   © )r   ÚcopyÚdeepcopyr   r   r   r   Úpipeline_cacheÚdefault_processorsÚ
isinstanceÚstrÚsplitÚstripÚlistr   ÚsortedÚkeysÚloggerÚwarningÚdebugr   r   r   Úlang_id_pipeliner   Ú	resources)Úselfr   r   r   r   r   r   r   r   r   r   Úxr   Úknown_langss                 úf/home/spazio/kingdom_calls/work5/transenv/lib/python3.12/site-packages/stanza/pipeline/multilingual.pyÚ__init__zMultilingualPipeline.__init__(   sá  € ð #ˆŒØ$2Ð$:™bÄÇÁÈnÓ@]ˆÔØ".Ð"6™B¼D¿M¹MÈ,Ó<WˆÔØ,ˆÔô *›mˆÔØÐØ&*ˆDÕ#Ü˜
¤CÔ(Ø:D×:JÑ:JÈ3Ó:OÖ&P°Q q§w¡w¥yÒ&PˆDÕ#ä&*¨:Ó&6ˆDÔ#à.ˆÔØ D×$7Ñ$7Ñ7Ø59×5IÑ5IˆD×ÑÐ 1Ñ2ð
 ×%Ñ%ò 	7ˆDØ˜T×.Ñ.¨tÑ4Ò4Ø26�×!Ñ! $Ñ'¨Ò/ð	7ñ Ð,°D×4GÑ4GÑGÜ  ×!2Ñ!2×!7Ñ!7Ó!9Ó:ˆKØ˜aÒÜ—‘ð   Bõ  Cä—‘ÐEÀ{ÔSØ<G�×#Ñ#Ð$8Ñ9ð ˆ>Øˆ '¨T¢/Ü'Ó)‘à�ØˆŒô !)ð !T¨T¯^©^À.Ð]eØ04·±ñ!TØ?C×?RÑ?Rñ!TˆÔô -¨T¯^©^Ó<ˆ�ùòI 'Qs   ÂG8c                 ób  — || j                   v r| j                   j                  |d¬«       	 | j                  |   }d|vr||d<   d|vr| j                  |d<   d|vr…| j
                  ryt        | j                  |«      }| j
                  D �cg c]	  }||v sŒ|‘Œ }}|| j
                  k7  r"t        j                  d| j
                  ||«       dj                  |«      |d<   d|vr| j                  |d<   || j                   vr†t        j                  d	|«       t        | j                   «      | j                  k(  r| j                   j                  d
¬«       t!        dd| j"                  i| j                  |   ¤Ž| j                   |<   yy# t        $ r d|i}|| j                  |<   Y �Œlw xY wc c}w )zÓ
        Do any necessary updates to the pipeline cache for this language. This includes building a new
        pipeline for the lang, and possibly clearing out a language with the old last access date.
        T)Úlastr   r   r   zENot all requested processors %s available for %s.  Loading %s insteadr   r   z4Loading unknown language in MultilingualPipeline: %sFr#   Nr$   )r'   Úmove_to_endr   ÚKeyErrorr   r(   r   r4   r0   ÚinfoÚjoinr   r2   Úlenr   Úpopitemr   r   )r5   r   Úlang_configÚlang_resourcesr6   Úlang_processorss         r8   Ú_update_pipeline_cachez+MultilingualPipeline._update_pipeline_cacheg   sÅ  € ð �4×&Ñ&Ñ&Ø×Ñ×+Ñ+¨D°tÐ+Ô<ð	2Ø×+Ñ+¨DÑ1ˆKð ˜Ñ$Ø"&ˆK˜Ñà KÑ/Ø-1×-AÑ-AˆKÐ)Ñ*à˜{Ñ*Ø×&Ò&Ü!7¸¿¹ÈÓ!M�Ø.2×.EÑ.EÖ"]¨ÈÈnÒI\¢1Ð"]�Ð"]Ø" d×&=Ñ&=Ò=Ü—K‘KÐ gÐim÷  jAñ  jAð  CGð  IXô  YØ,/¯H©H°_Ó,E�˜LÑ)à˜;Ñ&Ø$(§K¡KˆK˜Ñ!ð �t×*Ñ*Ñ*Ü�L‰LÐOÐQUÔVä�4×&Ñ&Ó'¨4×+>Ñ+>Ò>Ø×#Ñ#×+Ñ+°Ð+Ô7Ü(0Ñ(_°T·^±^Ð(_Àt×GXÑGXÐY]ÑG^Ñ(_ˆD×Ñ Ò%ð +øô1 ò 	2Ø! 4˜.ˆKØ&1ˆD×Ñ˜dÔ#ð	2üò #^s   ­F	 Â	F,ÂF,Æ	F)Æ(F)c                 óB  — t        |t        «       }|r|g}n|}|r-t        |d   t        «      r|D �cg c]  }t        g |¬«      ‘Œ }}| j                  j                  |«      }i }t        |«      D ]a  \  }}t        j                  d||j                  «       |j                  |vrg ||j                  <   ||j                     j                  |«       Œc |j                  «       D ]+  }| j                  |«        | j                  |   ||   «       Œ- |r|d   S |S c c}w )zz
        Run language detection on a string, a Document, or a list of either, route to language specific pipeline
        r   )ÚtextzLanguage for document %d: %s)r)   r-   r*   r   r3   ÚprocessÚ	enumerater0   r2   r   Úappendr/   rE   r'   )	r5   ÚdocÚsingleton_inputÚdocsrG   Údocs_w_langidÚlang_batchesÚdoc_idxr   s	            r8   rH   zMultilingualPipeline.process”   s&  € ô )¨¬dÓ3Ð3ˆÙØ�5‰DàˆDá”J˜t A™w¬Ô,Ø8<Ö=°”H˜R dÖ+Ð=ˆDÐ=ð ×-Ñ-×5Ñ5°dÓ;ˆð ˆÜ% mÓ4ò 	/‰LˆG�SÜ�L‰LÐ7¸À#Ç(Á(ÔKØ�x‰x˜|Ñ+Ø)+�˜SŸX™XÑ&Ø˜Ÿ™Ñ"×)Ñ)¨#Õ.ð		/ð !×%Ñ%Ó'ò 	:ˆDØ×'Ñ'¨Ô-Ø%ˆD×Ñ Ñ% l°4Ñ&8Õ9ð	:ñ
 Ø  Ñ#Ð#à Ð ùò- >s   ³Dc                 ó(   — | j                  |«      }|S )N)rH   )r5   rK   s     r8   Ú__call__zMultilingualPipeline.__call__¹   s   € Ø�l‰l˜3ÓˆØˆ
ó    )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r
   r   ÚDOWNLOAD_RESOURCESr*   ÚdictÚintÚboolr   r-   r9   rE   rH   rR   r$   rS   r8   r   r      s®   „ ñð, #4Ø(,Ø&*Ø&(Ø')Ø!%Ø"'Ø#Ø3A×3TÑ3Tà04ñ==Øð==à!%ð==ð  $ð==ð !$ð	==ð
 "%ð==ð ð==ð  ð==ð ð==ð #1ð==ð # 3¨ 9Ñ-ó==ò~+`òZ#!óJrS   r   )rW   Úcollectionsr   r%   ÚloggingÚtypingr   Ústanza.models.common.docr   Ústanza.models.common.utilsr   Ústanza.pipeline.corer   r   Ústanza.pipeline._constantsÚstanza.resources.commonr
   r   r   Ú	getLoggerr0   r   r$   rS   r8   ú<module>re      sH   ðñõ $Û Û Ý å -Ý 5ß 9Ü (ß bÑ bà	ˆ×	Ñ	˜8Ó	$€÷iò irS   