_j ddlZddlZddlZddlZddlZddlZddlZddlm Z m Z ddl m Z ddl mZddlmZmZddlmZddlmZddlmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&ddl'Z'ddl(Z(dd l)m*Z*dd l+m,Z,m-Z-m.Z.m/Z/dd l0m1Z1d d l2m3Z3m4Z4m5Z5d dl6m7Z7d dl8m9Z9m:Z:d dl;mZ>m?Z?m@Z@d dlAmBZBmCZCd dlDmEZEd dlFmGZGmHZHmIZId dlJmKZKmLZLmMZMmNZNmOZOd dlPmQZQd dlRmSZSd dlTmUZUd dlVmWZWd dlXmYZYmZZZd dl[m\Z\m]Z]d dl5m^Z^m_Z_m`Z`maZambZbmcZcmdZdmeZemfZfd dlgmhZhd dlimjZjmkZkeeUgeUfZleemjdz Zoe5jeoZqeemjd z Zre#d!ZsGd"d#Ztd$ed%geSffd&Zud'ZvGd(d%Zwd)exd$exfd*ZyeGd+d,ZzGd-d.e{Z|d/eeYd$eeYfd0Z}d1ee$e~eUefesgeUfd2eed3eeUfd4e"exexexfd$dfd5ZGd6d7ZGd8d9ZeZy):N) ExitStackcontextmanager)deepcopy) dataclass)chaincycle)Path) default_timer)AnyCallableDictIterableIteratorListNoReturnOptionalPatternSequenceSetTupleTypeVarUnioncastoverload)Pool)ConfigCupyOps Optimizerget_current_ops)convert_recursive)abouttyutil)Literal)ErrorsWarnings) GIT_VERSION)TOKENIZER_INFIXESTOKENIZER_PREFIXESTOKENIZER_SUFFIXES)BASE_EXCEPTIONS URL_MATCH) load_lookups) analyze_pipesprint_pipe_analysisvalidate_attrs) ConfigSchemaConfigSchemaInitConfigSchemaNlpConfigSchemaPretrainvalidate_init_settings)Scorer Tokenizer)Doc) Underscore)Examplevalidate_examples) init_tok2vec init_vocab) _DEFAULT_EMPTY_PIPESCONFIG_SECTION_ORDERSimpleFrozenDictSimpleFrozenList_pipecombine_score_weights raise_errorregistrywarn_if_jupyter_cupy) BaseVectors)Vocab create_vocabzdefault_config.cfgzdefault_config_pretraining.cfg _AnyContextcFeZdZUdZeeZeed<eZ e e e e fed<eZeeee efed<eZeeee efed<eZeeee efed<dZeeed <eZeeed <iZe e efed <iZe eee geffed <e Z!e"e ed <ddddZ#y) BaseDefaultszLanguage data defaults, available via Language.Defaults. Can be overwritten by language subclasses by defining their own subclasses of Language.Defaults.  section_orderconfigtokenizer_exceptionsprefixessuffixesinfixesN token_match url_matchsyntax_iteratorslex_attr_getters stop_wordsltrT) directionhas_case has_letters)$__name__ __module__ __qualname____doc__rrArQ__annotations__r,rRr strrdictr*rSrrrrr+rTr)rUrVr r-rWrXrYintr setrZrwriting_systemX/home/spazio/kingdom_calls/work5/transenv/lib/python3.12/site-packages/spacy/language.pyrNrNXs *>?FF?2A$sDJ/A8JHhxc7l 345J8JHhxc7l 345J7HGXhuS'\23 4H&*K(#*$-Ix!-,.d3=).8:d3# 445:5JC #(d4PNrjrNreturnLanguagecdddtfd}|S)zRegistered function to create a tokenizer. Returns a factory that takes the nlp object and returns a Tokenizer instance using the language detaults. nlprmrlc |jj}|jj}|jj}|rt j |j nd}|rt j|j nd}|rt j|jnd}t|j|jj||||jj|jjS)N)rules prefix_search suffix_searchinfix_finditerrVrW)DefaultsrSrTrUr$compile_prefix_regexsearchcompile_suffix_regexcompile_infix_regexfinditerr9vocabrRrVrW)rorSrTrUrrrsrts rktokenizer_factoryz+create_tokenizer..tokenizer_factoryps<<((<<((,,&&FN11(;BBTX FN11(;BBTX GN11':CCTX II,,33'') 00ll,,  rjr8)r|s rkcreate_tokenizerr}ks  z i " rjc`tjjd|t||}|S)Nz+Loading lookups from spacy-lookups-data: %s)langtables)r$loggerdebugr.)rrlookupss rkload_lookups_datars(KKCVLV4G Nrjc eZdZUdZeZdZeee d<e Z e e jZiZeedfe d< ddiddd d d eeefd ed eeefdeedgeegeffdeedgefdeddfdZfdZedZedeeeffdZej>deeefddfdZede fdZ!e!j>de ddfdZ!ede"efdZ#ede"efdZ$ede"e%ee&ffdZ'ede"efdZ(ede"e%ee&ffdZ)ede"efd Z*edeeeffd!Z+edeee"effd"Z,e-d#edefd$Z.e-d#edefd%Z/e-d#eddfd&Z0e-d#eddddfd'Z1d#eddfd(Z2d#ede fd)Z3e-e e4e4d*e dd+d#ed,eeefd-e5ed.e5ed/ed0eeee6fd1eedefd2Z7e-e4e4d*dd3d#ed-e5ed.e5ed/ed1ee&ded4eff d5Z8gd6d*d7d8e"ed9edeeeeffd:Z9d#ede&fd;Z: de ddd<d=ed#eed>eeefd?ee d@ede&f dAZ;dBedCdd#ede%e&effdDZ< dddddde dddEd=ed#eedFeeeefdGeeeefdHeedIeedCedd>eeefd?ee d@ede&fdJZ= ddFeeeefdGeeeefdHeedIeedef dKZ>d#edefdLZ?e ddMd#ed=ed>eeefd@ede&f dNZ@dOedPeddfdQZAd#ede%ee&ffdRZBd#eddfdSZCd#eddfdTZDe4ddUdVeeefdWe5edXeeeeeeffdefdYZEdd[ZFddd\dWeeee5efd]eeee5efddZfd^ZGdVedefd_ZHd`eeeeIfdefdaZJd`eeeeIfdbeKdefdcZL dddddde4e4dedfe5eMdgeedhe6dieeNdjeeee6fdXeeeeeeffdke5edle5efdmZOddde4dndfe5eMdieeNdjeeee6fdXeeeeeeffdke5edeee6ff doZP dddpdqeege5eMfdieeNdeNfdrZQ dddpdqeege5eMfdieeNdeNfdsZRddpdieeNdeNfdtZSdueee&e"eeTgeUffdvZVddddd*dwdfe5eMdeedxeeWdXeeeeeeffdyeeeefdzedeeeffd{ZXd|ZYeZd}ee[fd~Z\e]d4d4d4d4d4dde5eeefde^d*deedWe5edXeeeeeeffdede_efdZ`e]d4d4d4d4d4dde5e%eeefeKfde^ddeedWe5edXeeeeeeffdede_e%eeKffdZ`d*de4ddddee5eeefe5e%eeefeKffdedeedWe5edXeeeeeeffdedee_ee_e%eeKfffdZ`dWe5efdZade5eeefde5ed4e_efdedede_ef dZbddZce-ifdededede dddd>eeeefe fd eeefdWeee5efd]eee5efdkeee5efd eeefded@eddfdZededede5eddfdZfeZddeegde_egfdZhe4ddeeeifdke5eddfdZjekdWeee5efd]eee5efde5ede%ed4ffdZle4e ddeeeifdke5edeeefddfdZme4ddke5edeIfdZne4ddeIdke5eddfdZoxZpS)rmaA text-processing pipeline. Usually you'll load this once per process, and pass the instance around your application. Defaults (class): Settings, data and factory methods for creating the `nlp` object and processing pipeline. lang (str): Two-letter ISO 639-1 or three-letter ISO 639-3 language codes, such as 'en' and 'eng'. DOCS: https://spacy.io/api/language Nrerror FactoryMeta _factory_metaTi@Bi) max_lengthmetar}create_vectors batch_sizer{rrr}rrJrrlc ddlm}|tjjj t j|j|_ t||_ d|_ d|_ i|_i|_t!|t"s;|dur7t%t&j(j+|t-t"|dur|j/dij/d} t1|j2|j4| }|s,d|jd di} tj6| d}|||_no|j2rc|j2rW|j2|j2k7r>t%t&j:j+|j2|j2 ||_|j2|j<j2|_g|_tA|_!||_"|s,d |jd d i} tj6| d }|||_#||_$tJ|_&y) aInitialise a Language object. vocab (Vocab): A `Vocab` object. If `True`, a vocab is created. meta (dict): Custom meta data for the Language class. Is written to by models to add model meta data. max_length (int): Maximum number of characters in a single text. The current models may run out memory on extremely long texts, due to large internal allocations. You should segment these texts into meaningful units, e.g. paragraphs, subsections etc, before passing them to spaCy. Default maximum length is 1,000,000 charas (1mb). As a rule of thumb, if all pipeline components are enabled, spaCy's default models currently requires roughly 1GB of temporary memory per 100,000 characters in one text. create_tokenizer (Callable): Function that takes the nlp object and returns a tokenizer. batch_size (int): Default batch size for pipe and evaluate. DOCS: https://spacy.io/api/language#init r!)register_factoriesNT)r{ vocab_typevectorsname) vectors_namero)ror{ tokenizer)'pipeline.factoriesrr$rG_entry_point_factoriesget_allDEFAULT_CONFIGmergedefault_config_configre_meta_path _optimizer _pipe_meta _pipe_configs isinstancerJ ValueErrorr&E918formattypegetrKrruresolverE150r{ _componentsrg _disabledrrrrFdefault_error_handler) selfr{rrr}rrkwargsrr vectors_cfg tokenizer_cfgs rk__init__zLanguage.__init__s< ; ,,446%++D,?,?@ $Z  /34602%'E,=V[[//eU /TU U D=88Ir266v>L DMM UE!($,,u*=i*HI !)!1!1+!>y!I*51EM ejjtyyEJJ/F !3!3 !3!TUU! 99  DI;=#&5$($,,u*=k*JKM'// >{K )$/$%0"rjc t|di|tj|jj |_|j|j dd<y)Nrorri)super__init_subclass__rrrurQrr)clsr __class__s rkrzLanguage.__init_subclass__sI !+F++11#,,2E2EF,/HH5!&)rjc|jSN)rrs rkpathz Language.paths zzrjctjtj}|jj r1|j jd|jj n&|j jd|j |j jdd|j jdd|j jd||j jdd|j jd d|j jd d|j jd d|j jd d|j jd t|jjt|jj|jjj|jjj|jjjd|j d<t|j |j d<t#|j$|j d<t#|j&|j d<t#|j(|j d<|j S)zCustom meta data of the language class. If a model is loaded, this includes details from the model's meta.json. RETURNS (Dict[str, Any]): The meta. DOCS: https://spacy.io/api/language#meta rrpipelineversionz0.0.0 spacy_version descriptionauthoremailurllicensespacy_git_version)widthrkeysrmoderlabels componentsdisabled)r$get_minor_version_ranger" __version__r{rr setdefaultr(vectors_lengthlenrn_keysrrre pipe_labelslist pipe_namescomponent_namesr)rrs rkrz Language.metas44U5F5FG ::?? JJ ! !&$**// : JJ ! !&$)) 4 fj1 i1 o}= mR0 h+ gr* eR( i, 1;?ZZ..4::--.JJ&&--JJ&&++JJ&&++ !  9 $D$4$45 8"&doo!6 :#'(<(<#= < !%dmm!4 :zzrjvaluec||_yr)rrrs rkrz Language.metas  rjcv|jjdi|jjdi|j|jdd<i}g}|jD]_}|j |}|j |}d|j i|||<|jsE|j|jat|j|jdd<t|j|jdd<||jd<|jdjdi}t||}||jdd<tj|js3tt j"j%|j |jS) zTrainable config for the current language instance. Includes the current pipeline components, as well as default training config. RETURNS (thinc.api.Config): The config. DOCS: https://spacy.io/api/language#config rotrainingrfactoryrrr score_weightsrQ)rrrr get_pipe_metaget_pipe_configrdefault_score_weightsappendrrrrEsrslyis_json_serializablerr&E961r)rrr pipe_name pipe_meta pipe_config prev_weightscombined_score_weightss rkrQzLanguage.configs r*  B/&*ii UF# -- FI**95I..y9K#,i.?.?"O;"OHY ..$$Y%D%DE  F +/t/C/C*D UJ'*.t}}*= UJ'%- \"||J/33ORH !6}l!S4J Z 1))$,,7V[[//t||/DE E||rjc||_yr)rrs rkrQzLanguage.config:s  rjc|jDcgc]\}}||jvs|}}}t|tjj dScc}}w)ziGet the names of all disabled components. RETURNS (List[str]): The disabled components. rattrrrrrCr&E926r)rr_namess rkrzLanguage.disabled>sQ&*%5%5P'$9OPPV[[-?-?Z-?-PQQQs AAc^t|jj}t|S)z_Get names of all available factories. RETURNS (List[str]): The factory names. )r factoriesrrCrrs rk factory_nameszLanguage.factory_namesIs& T^^((*+&&rjclt|jtjj dS)zoGet all (name, component) tuples in the pipeline, including the currently disabled components. rrr)rCrr&rrrs rkrzLanguage.componentsRs.    FKK$6$6L$6$I  rjc|jDcgc]\}}| }}}t|tjj dScc}}w)zGet the names of the available pipeline components. Includes all active and inactive pipeline components. RETURNS (List[str]): List of component name strings, in order. rrr)rrCr&rrrrrrs rkrzLanguage.component_names[sG04/?/?@|y!@@V[[-?-?EV-?-WXXA A c|jDcgc]\}}||jvs||f}}}t|tjj dScc}}w)zThe processing pipeline consisting of (name, component) tuples. The components are called on the Doc in order as it passes through the pipeline. RETURNS (List[Tuple[str, Callable[[Doc], Doc]]]): The pipeline. rrrr)rnppipess rkrzLanguage.pipelineesU%)$4$4PDAq8O!QPPV[[-?-?Z-?-PQQQs AAc|jDcgc]\}}| }}}t|tjj dScc}}w)zGet names of available active pipeline components. RETURNS (List[str]): List of component name strings, in order. rrr)rrCr&rrrs rkrzLanguage.pipe_namespsC 04}}=|y!==V[[-?-?\-?-RSS>rci}|jD]#\}}|j|j||<%t|S)zGet the component factories for the available pipeline components. RETURNS (Dict[str, str]): Factory names, keyed by component names. )rrrrB)rrrpipes rkpipe_factorieszLanguage.pipe_factoriesysJ  #// IOIt#'#5#5i#@#H#HIi  I **rjci}|jD]E\}}t|dr|jdur!t|ds.t|j||<Gt |S)zGet the labels set by the pipeline components, if available (if the component exposes a labels property and the labels are not hidden). RETURNS (Dict[str, List[str]]): Labels keyed by component name. hide_labelsTr)rhasattrrrrrB)rrrrs rkrzLanguage.pipe_labelsse** 1JD$t]+0@0@D0HtX&#DKK0t  1  ''rjrcp|j|}|tjvxs|tjvS)z=RETURNS (bool): Whether a factory of that name is registered.)get_factory_namerGrrr internal_names rk has_factoryzLanguage.has_factorys5,,T2 x)))P]h>P>P-PPrjc@|j|S|jd|S)zGet the internal factory name based on the language subclass. name (str): The factory name. RETURNS (str): The internal factory name. .)r)rrs rkrzLanguage.get_factory_names( 88 K((1TF##rjc|j|}||jvr|j|S||jvr|j|Sttjj d|)zGet the meta information for a given factory name. name (str): The component factory name. RETURNS (FactoryMeta): The meta for the given factory name. rrr)rrrr&E967rrs rkget_factory_metazLanguage.get_factory_metasq,,T2 C-- -$$]3 3 3$$ $$$T* *+++FGGrjc@||j|j|<y)zSet the meta information for a given factory name. name (str): The component factory name. value (FactoryMeta): The meta to set. N)rr)rrrs rkset_factory_metazLanguage.set_factory_metas9>#..t45rjc||jvr*ttjj d||j|S)zGet the meta information for a given component name. name (str): The component name. RETURNS (FactoryMeta): The meta for the given component name. componentr )rrr&r rrrs rkrzLanguage.get_pipe_metas> t &V[[//[t/LM Mt$$rjc||jvr)ttjj ||j|}|S)zGet the config used to create a pipeline component. name (str): The component name. RETURNS (Config): The config used to create the pipeline component. r)rrr&E960r)rrrs rkrzLanguage.get_pipe_configsE t)) )V[[//T/:; ;((. rjF)rassignsrequires retokenizesrfuncrrrrrrctts)ttjj ddvr)ttj j tts6tjj dt}t|dtdtffd } || |S| S) a~Register a new pipeline component factory. Can be used as a decorator on a function or classmethod, or called as a function with the factory provided as the func keyword argument. To create a component and add it to the pipeline, you can use nlp.add_pipe(name). name (str): The name of the component factory. default_config (Dict[str, Any]): Default configuration, describing the default values of the factory arguments. assigns (Iterable[str]): Doc/Token attributes assigned by this component, e.g. "token.ent_id". Used for pipeline analysis. requires (Iterable[str]): Doc/Token attributes required by this component, e.g. "token.ent_id". Used for pipeline analysis. retokenizes (bool): Whether the component changes the tokenization. Used for pipeline analysis. default_score_weights (Dict[str, Optional[float]]): The scores to report during training, and their default weight towards the final score used to select the best model. Weights should sum to 1.0 per component and will be combined and normalized for the whole pipeline. If None, the score won't be shown in the logs or be weighted. func (Optional[Callable]): Factory function if not used as a decorator. DOCS: https://spacy.io/api/language#factory r decoratorr rzdefault configstylercfg_type factory_funcrlc j }|tjvrbtjj|}t j ||s-t jj ||}t|t j|}d|vsd|vr)tt jj tjj||t tt t j! }j# |t%tjj't j(_|S)N)rrnew_funcrorr)r)rrrrscoresrrr)rrGrrr$ is_same_funcr&E004rr get_arg_namesE964registerrr1rrrrBrE957) r!r existing_funcerr arg_names factory_metarrrrrrrs rk add_factoryz%Language.factory..add_factorysD006M 2 22 !) 2 2 6 6} E ((}E ++,,! -C%S/)**<8II%y)@ !3!3!3!>??    ' ' L ' I&-&w/'116689&;'L  | 4 -""**,FKKCM rj) rrdrr&E963rE853reE962rr ) rrrrrrrrr,r/s ``````` rkrzLanguage.factorysF$$V[[//)/DE E $;V[[//T/:; ;.$/++$$&TD.add_component..factory_funcQs %%rjrr3)rrrr&E965rrd PipeCallablerrGrr __closure__ cell_contentsr$r%r)r5r!rr+closurecwrappedrrcomponent_namerrrrs` rk add_componentz)Language.component..add_componentMs$% !3!3!3!HII & &  & 006M 2 22!) 2 2 6 6} E '33CJG??!%!1t7K7KD7Q ", "8 " ">   & &rj)rrr$r)rprettyrrBc>t||}|r t|||S)aAnalyze the current pipeline components, print a summary of what they assign or require and check that all requirements are met. keys (List[str]): The meta values to display in the table. Corresponds to values in FactoryMeta, defined by @Language.factory decorator. pretty (bool): Pretty-print the results. RETURNS (dict): The data. )r)r/r0)rrrBanalysiss rkr/zLanguage.analyze_pipesps"!D1  t 4rjc|jD]\}}||k(s |cSttjj ||j )zGet a pipeline component for a given component name. name (str): Name of pipeline component to get. RETURNS (callable): The pipeline component. DOCS: https://spacy.io/api/language#get_pipe ropts)rKeyErrorr&E001rr)rrrrs rkget_pipezLanguage.get_pipesS%)$4$4 ! IyD    !v{{))t$:N:N)OPPrj)rQ raw_configvalidate factory_namerQrKrLc2||n|}t|ts6tjj d|t |}t |tj|s)t tjj ||j|setjj |dj|jdtj||j }t ||j#|}|j$r$t'|j$j)|}|j+|}|t,j.vr|}||d|d |i}||i} t-j0| | } t-j2d | |i| d } t'| } || d <| j5d d| j5d d| j5dd|r| j)|} | |j6|<| |S)aCreate a pipeline component. Mostly used internally. To create and add a component to the pipeline, you can use nlp.add_pipe. factory_name (str): Name of component factory. name (Optional[str]): Optional name to assign to component instance. Defaults to factory name if not set. config (Dict[str, Any]): Config parameters to use for this component. Will be merged with default config, if available. raw_config (Optional[Config]): Internals: the non-interpolated config. validate (bool): Whether to validate the component config against the arguments and types expected by the factory. RETURNS (Callable[[Doc], Doc]): The pipeline component. DOCS: https://spacy.io/api/language#create_pipe NrQrr, create_pipe)rrGmethodr lang_code)rorz @factories)rLcfgrror)rrer&r2rrrrrrrE002joinrr$rArrrrrrrGrrfillpopr) rrMrrQrKrLr,rrrSresolvedfilleds rkrPzLanguage.create_pipes0't\&$'++$$8$f$VCS/ !))&1V[[//v/>? ? -++$$!YYt112$))$/)) %CS/ !)),7   # #I445;;FCF--l;   2 2 2(MtSvS|]SV$##C(;s<'89HMeT(y <&  5$ 64  \\*-F#)4  %%rj source_namesourcec t|ts3ttjj |t ||jj|jjk7r3tjtjj |||jvrcttjj ||j dd|j ddj#|j|j%|}t'|dr||_|j*j-}t/j0|d|}||j2|<|jj4|jj4k7r@|jj4D]'}|jj4j7|)||d fS) auCreate a pipeline component by copying it from an existing model. source_name (str): Name of the component in the source pipeline. source (Language): The source nlp object to copy from. name (str): Optional alternative name to use in current pipeline. RETURNS (Tuple[Callable[[Doc], Doc], str]): The component and its factory name. )rr[rrrrrO)rmodelrGrr)rrmrr&E945rrr{rwarningswarnr'W113rrHE944rrUrJrrrQ interpolater$ copy_configrstringsadd)rrZr[rr source_configrss rkcreate_pipe_from_sourcez Language.create_pipe_from_sources&(+V[[//[f/VW W ::  !5!5 5 MM(--..K.@ A f44 4 ""$#[[016;;v3F2GH6#9#9:# {+ 4 DI 113 &&}\'B;'OP #.4 ::  !5!5 5\\)) * ""&&q) *[+++rj)beforeafterfirstlastr[rQrKrLrjrkrlrmct|ts7t|} tjj | |} t | ||n|}||jvr4t tjj ||jd|vrBtjtjj |jd||j|||\} }n|j|||| | } |j!||||}|j#||j$|<|j&j)||| f|j+| S)aAdd a component to the processing pipeline. Valid components are callables that take a `Doc` object, modify it and return it. Only one of before/after/first/last can be set. Default behaviour is "last". factory_name (str): Name of the component factory. name (str): Name of pipeline component. Overwrites existing component.name attribute if available. If no name is set and the component exposes no name attribute, component.__name__ is used. An error is raised if a name already exists in the pipeline. before (Union[str, int]): Name or index of the component to insert new component directly before. after (Union[str, int]): Name or index of the component to insert new component directly after. first (bool): If True, insert component first in the pipeline. last (bool): If True, insert component last in the pipeline. source (Language): Optional loaded nlp object to copy the pipeline component from. config (Dict[str, Any]): Config parameters to use for this component. Will be merged with default config, if available. raw_config (Optional[Config]): Internals: the non-interpolated config. validate (bool): Whether to validate the component config against the arguments and types expected by the factory. RETURNS (Callable[[Doc], Doc]): The pipeline component. DOCS: https://spacy.io/api/language#add_pipe rrrFr)name_in_configr)rrQrKrL)rrdreprr&E966rrrE007r_r`r'W119rWrirP_get_pipe_indexrrrinsert_link_components)rrMrrjrkrlrmr[rQrKrLbad_valr,pipe_component pipe_indexs rkadd_pipezLanguage.add_pipesXP,,<(G++$$wT$BCS/ !'t\ 4'' 'V[[//T@T@T/UV V V  MM(--..fjj>P.Q R  ,0+G+Gf4,H, (NL"--%! .N))&%E $ 5 5l C  T>,BC rjc||||d}td||||fDdk\r4ttjj ||j |st d|||fDst|jS|ryt|tr]||j vr4ttjj ||j |j j|St|tr`||j vr4ttjj ||j |j j|dzSt|tk(rV|t|jk\s|dkr7tjj d ||j }t||St|tk(rY|t|jk\s|dkr7tjj d ||j }t||dzSttjj ||j ) aDetermine where to insert a pipeline component based on the before/ after/first/last values. before (str): Name or index of the component to insert directly before. after (str): Name or index of component to insert directly after. first (bool): If True, insert component first in the pipeline. last (bool): If True, insert component last in the pipeline. RETURNS (int): The index of the new pipeline component. )rjrkrlrmc3$K|]}|du ywrri).0args rk z+Language._get_pipe_index..WsG3s$G)argsrGc3$K|]}|du ywrri)r~rs rkrz+Language._get_pipe_index..[sQ5,QrrrFr!rj)diridxrGrk)sumrr&E006rranyrrrrdrIindexrrfE959)rrjrkrlrmall_argsr,s rkruzLanguage._get_pipe_indexFs' %uudS G65%*FG G1 L ""t7K7K"L  sQ5&%:PQQt''( (   $T111 KK&&F9M9M&N''--f5 5 s #D000 KK&&E8L8L&M''--e4q8 8&\S T--..&1*kk(( f43G3G)!o%M %[C D,,--kk((U1E1E)!o%19 ++@T@T+UVVrjc||jvS)a Check if a component name is present in the pipeline. Equivalent to `name in nlp.pipe_names`. name (str): Name of the component. RETURNS (bool): Whether a component of the name exists in the pipeline. DOCS: https://spacy.io/api/language#has_pipe )rrs rkhas_pipezLanguage.has_pipe}st&&rj)rQrLc||jvr4ttjj ||j t |dr5tjj t||}t||jj|}|j|t|jr|t|jk(r|j||||S|j|||||S)aBReplace a component in the pipeline. name (str): Name of the component to replace. factory_name (str): Factory name of replacement component. config (Optional[Dict[str, Any]]): Config parameters to use for this component. Will be merged with default config, if available. validate (bool): Whether to validate the component config against the arguments and types expected by the factory. RETURNS (Callable[[Doc], Doc]): The new pipeline component. DOCS: https://spacy.io/api/language#replace_pipe rF__call__ro)rrQrL)rrjrQrL)rrr&rIrrrE968rqr remove_piperrr{)rrrMrQrLr,rzs rk replace_pipezLanguage.replace_pipes( t++ +V[[//T/PQ Q < ,++$$tL/A$MCS/ !))//5  4##$ c$:J:J6K(K==4! ==!! ! rjold_namenew_namec||jvr4ttjj ||j||jvr4ttj j ||j|jj |}||j|df|j|<|jj||j|<|jj||j|<||jddvr6|jddj|}||jdd|<|jy)zRename a pipeline component. old_name (str): Name of the component to rename. new_name (str): New name of the component. DOCS: https://spacy.io/api/language#rename_pipe rFr! initializerN) rrr&rIrrsrrrrWrrrw)rrriinit_cfgs rk rename_pipezLanguage.rename_pipesI 4// / ""t7K7K"L  t++ + ""t7K7K"L   & &x 0')9)9!) rjc||jvr4ttjj ||j|j j |jj|}|jj ||jj ||jjdij |d||jddvr!|jddj |||jvr|jj||j!|S)aRemove a component from the pipeline. name (str): Name of the component to remove. RETURNS (Tuple[str, Callable[[Doc], Doc]]): A `(name, component)` tuple of the removed component. DOCS: https://spacy.io/api/language#remove_pipe rF_sourced_vectors_hashesNrr)rrr&rIrrrWrrrrrrrrremoverw)rrremoveds rkrzLanguage.remove_pipes t++ +V[[//T@T@T/UV V""&&t';';'A'A$'GH D! t$ /488tD 4<< -l; ; LL &| 4 8 8 > 4== NN ! !$ ' rjc||jvr4ttjj ||j|j j |y)aDisable a pipeline component. The component will still exist on the nlp object, but it won't be run as part of the pipeline. Does nothing if the component is already disabled. name (str): The name of the component to disable. rFN)rrr&rIrrrfrs rk disable_pipezLanguage.disable_pipesI t++ +V[[//T@T@T/UV V 4 rjc||jvr4ttjj ||j||j vr|j j|yy)zEnable a previously disabled pipeline component so it's run as part of the pipeline. Does nothing if the component is already enabled. name (str): The name of the component to enable. rFN)rrr&rIrrrrrs rk enable_pipezLanguage.enable_pipesZ t++ +V[[//T@T@T/UV V 4== NN ! !$ ' !rj)disable component_cfgtextrrc |j|}|i}|jD]\}}||vr t|ds3ttj j t|||j}t|dr|j} ||fi|j|i}t|trttj j |t||S#t$r/}ttjj ||d}~wt$r}||||g|Yd}~d}~wwxYw)aApply the pipeline to some text. The text can span multiple sentences, and can contain arbitrary whitespace. Alignment into the original string is preserved. text (Union[str, Doc]): If `str`, the text to be processed. If `Doc`, the doc will be passed directly to the pipeline, skipping `Language.make_doc`. disable (List[str]): Names of the pipeline components to disable. component_cfg (Dict[str, dict]): An optional dictionary with extra keyword arguments for specific components. RETURNS (Doc): A container for accessing the annotations. DOCS: https://spacy.io/api/language#call Nrroget_error_handlerr)r returned_type) _ensure_docrrrr&E003rrrrrrHE109 Exceptionrr:E005) rrrrdocrproc error_handleres rkrzLanguage.__call__s9*t$  M-- YJD$w4, !3!3d4jt!3!TUU 66Mt01 $ 6 6 8  43>-"3"3D""=> c3' !3!3TRUY!3!WXX! Y"  G !3!3!3!>?QF 4dD3%33 4s$C55 E >*D(( E 4 EE  DisabledPipesctjtjtt |dk(rt |dttfr|d}|j|S)aVDisable one or more pipeline components. If used as a context manager, the pipeline will be restored to the initial state at the end of the block. Otherwise, a DisabledPipes object is returned, that has a `.restore()` method you can use to undo your changes. This method has been deprecated since 3.0 r!r)r) r_r`r'W096DeprecationWarningrrrtuple select_pipesrs rk disable_pipeszLanguage.disable_pipes'sP  hmm%78 u:?z%(T5MB!HE   //rj)renablerc||ttjt|tr|g}|ot|tr|g}|j Dcgc] }||vs| }}|:||k7r5ttj j|||j |}|J|Dcgc]}||jvs|}}t||Scc}wcc}w)aDisable one or more pipeline components. If used as a context manager, the pipeline will be restored to the initial state at the end of the block. Otherwise, a DisabledPipes object is returned, that has a `.restore()` method you can use to undo your changes. disable (str or iterable): The name(s) of the pipes to disable enable (str or iterable): The name(s) of the pipes to enable - all others will be disabled DOCS: https://spacy.io/api/language#select_pipes )rrr) rr&E991rrdrE992rrr)rrrr to_disableds rkrzLanguage.select_pipes4s >goV[[) ) gs #iG  &#& +/??Q4d&>P$QJQ"w*'< KK&&%wdoo' !G"""&A$..)@1AAT7++RBs CC*C>Cct||jkDr=ttjj t||j|j |S)z{Turn a text into a Doc object. text (str): The text to process. RETURNS (Doc): The processed doc. )lengthr)rrrr&E088rr)rrs rkmake_doczLanguage.make_docZsQ t9t & ""#d)"P ~~d##rjdoc_likec4t|tr|St|tr|j|St|tr$t|j j |Sttjjt|)zCreate a Doc if need be, or raise an error if the input is not a Doc, string, or a byte array (generated by Doc.to_bytes()).)r) rr:rdrbytesr{ from_bytesrr&E1041rr)rrs rkrzLanguage._ensure_docfsp h $O h $==* * h &tzz?--h7 7,,$x.,ABBrjcontextc6|j|}||_|S)z>Call _ensure_doc to generate a Doc and set its context object.)r_context)rrrrs rk_ensure_doc_with_contextz!Language._ensure_doc_with_contextqs x(  rjg)dropsgdlossesrexclude annotatesexamplesrrrrrrc |ttj|i}t|trt |dk(r|St |dt|}|-|j|j|_|j}|i}i} t|jD]_\} \} } |j| it|| | | <|| jd|| | jd|ja|jD]\} } | |vr%t| dr| j |fd|d|| |dvrI| |vrEt| t"j$r+| j&r| j(d vr| j+|| |vst-t/d |D| | |j0| |  |D] \} }| |_t5|S) a<Update the models in the pipeline. examples (Iterable[Example]): A batch of examples _: Should not be set - serves to catch backwards-incompatible scripts. drop (float): The dropout rate. sgd (Optimizer): An optimizer. losses (Dict[str, float]): Dictionary to update with the loss, keyed by component. component_cfg (Dict[str, Dict]): Config parameters for specific pipeline components, keyed by component name. exclude (Iterable[str]): Names of components that shouldn't be updated. annotates (Iterable[str]): Names of components that should set annotations on the predicted examples after updating. RETURNS (Dict[str, float]): The updated losses dictionary DOCS: https://spacy.io/api/language#update NrzLanguage.updaterrupdaterr)NF)TFNc34K|]}|jywr predictedr~egs rkrz"Language.update..s9"9)rrrr)rr&E989rrrr=_copy_examplesrcreate_optimizer enumeraterrrrrrr#TrainableComponent is_trainabler] finish_updateziprDrr_replace_numpy_floats)rrrrrrrrr pipe_kwargsrrrrrs rkrzLanguage.updateys: =V[[) ) >F h %#h-1*<M($56!(+ ;&"&"7"7"9//C  M (7 HOA|d  $ $T2 . (t)< =K  $  * *64 8   ( (t G  H -- 'JD$7"wtX'> HU$vUtATU-''"4)>)>?)) *==&&s+y "99!!.2.H.H*40   'GC$'BL ' '0%V,,rj)rrrrc f |i}t|trt|dk(r|St|d|-|j|j |_|j}t|j }tj||i}i fd}|j|_ |j|_ |j|_ |D]>\}} ||vs t| dsi | j|f||d|j|i@ jD]\} \} } || | | |S)a|Make a "rehearsal" update to the models in the pipeline, to prevent forgetting. Rehearsal updates run an initial copy of the model over some data, and update the model so its current predictions are more like the initial ones. This is useful for keeping a pretrained model on-track, even if you're updating it with a smaller set of examples. examples (Iterable[Example]): A batch of `Example` objects. sgd (Optional[Optimizer]): An optimizer. component_cfg (Dict[str, Dict]): Config parameters for specific pipeline components, keyed by component name. exclude (Iterable[str]): Names of components that shouldn't be updated. RETURNS (dict): Results from the update. EXAMPLE: >>> raw_text_batches = minibatch(raw_texts) >>> for labelled_batch in minibatch(examples): >>> nlp.update(labelled_batch) >>> raw_batch = [Example.from_dict(nlp.make_doc(text), {}) for text in next(raw_text_batches)] >>> nlp.rehearse(raw_batch) DOCS: https://spacy.io/api/language#rehearse rzLanguage.rehearsec||f|<||fSrri)keyWdWgradss rk get_gradsz$Language.rehearse..get_gradssRE#Jb5Lrjrehearser)rrrr=rrrrandomshuffle learn_rateb1b2rrritems)rrrrrrrrrrrrrrs @rkrzLanguage.rehearses:> >F h %#h-1*<M($78 ;&"&"7"7"9//CT]]#u  M  #~~ vv vv  JD$wgdJ&?E DMM ' :G:K:KDRT:U   "KKM LC!R QO  rjr get_examplescxtjtjt|j ||S)Nr)r_r`r'W089rr)rrrs rkbegin_trainingzLanguage.begin_trainings*  hmm%78|55rjc|=tjjdt|jgdfd}t |ds5t jjdt|}t||jj}tj|dt }|d }||| t!||d |d |d |jj&j(ddk\r/t+}|jj&j-|t |j.drEt1|j.j2|ddd}|j.j2|fd|i||j4D]c\} } t7| t8j:s!|dj=| i} t1| j2| d| } | j2|fd|i| e|j=d} | r(tj| t> } tA|| ||jC||_"|||_"n!|jD|jG|_"|d}||||jDS#t"$r-t#t j$j|d wxYw)aInitialize the pipe for training, using data examples if available. get_examples (Callable[[], Iterable[Example]]): Optional function that returns gold-standard Example objects. sgd (Optional[Optimizer]): An optimizer to use for updates. If not provided, will be created using the .create_optimizer() method. RETURNS (thinc.api.Optimizer): The optimizer. DOCS: https://spacy.io/api/language#initialize zUNo 'get_examples' callback provided to 'Language.initialize', creating dummy examples)xyz)wordsc2tjigSr)r< from_dict)rsrkrz)Language.initialize..get_examples#s))#r233rjrzLanguage.initialize)rQobjr)schema before_init vocab_datarr)datarr)rr!r)sectionrror pretraining after_init)$r$rrr:r{rr&E930rr TypeErrorrQrcrGrr3r?IOErrorE884rshaperto_opsrr6rrrr#InitializableComponentrr5r>rwrr)rrrr,rQIrops tok_settingsrr p_settings pretrain_cfgPrrs @rkrzLanguage.initialize s   KK  g djj8C 4|Z0++$$,$|2D%CC. ((*   VL1:J K &  "   D 1\?AiL!I,  ::   # #A &! +!#C JJ   % %c * 4>>< 01))+# L &DNN % %l M M M-- FJD$$ 9 9:|_00r: 3OOZD   E$E*E  Fzz-0   6JKA q! $  ?!DO __ $"335DO|_  ! t C D&++,,Qy\,BC C Ds J6Kct}|jjjddk\r%|jjj ||j D],\}}t |dst|j|_ .|||_ |jS|j|j|_ |jS)a Continue training a pretrained model. Create and return an optimizer, and initialize "rehearsal" for any pipeline component that has a .rehearse() method. Rehearsal is used to prevent models from "forgetting" their initialized "knowledge". To perform rehearsal, collect samples of text you want the models to retain performance on, and call nlp.rehearse() with a batch of Example objects. RETURNS (Optimizer): The optimizer. DOCS: https://spacy.io/api/language#resume_training r!_rehearsal_model) rr{rr r rrrr]rrr)rrrrrs rkresume_trainingzLanguage.resume_trainingYs ::   # #A &! + JJ   % %c *-- =JD$t/0(0(<% = ?!DO__ $"335DOrjrcv||_|jD]#\}}t|ds|j|%y)aSet an error handler object for all the components in the pipeline that implement a set_error_handler function. error_handler (Callable[[str, Callable[[Doc], Doc], List[Doc], Exception], NoReturn]): Function that deals with a failing batch of documents. This callable function should take in the component's name, the component itself, the offending batch of documents, and the exception that was thrown. DOCS: https://spacy.io/api/language#set_error_handler set_error_handlerN)rrrr)rrrrs rkrzLanguage.set_error_handlerrs<&3"-- 6JD$t01&&}5 6rj)rscorerr scorer_cfg per_componentrrrc(t|}t|dt|}| |j}|i}|i}|(t |}|j d|t di|}t}|D]'} |j| jj)|jd|D||} t|| D] \} } | | _ t} |j||} td|D}|| |z z | d<t!| S) aEvaluate a model's pipeline components. examples (Iterable[Example]): `Example` objects. batch_size (Optional[int]): Batch size to use. scorer (Optional[Scorer]): Scorer to use. If not passed in, a new one will be created. component_cfg (dict): An optional dictionary with extra keyword arguments for specific components. scorer_cfg (dict): An optional dictionary with extra keyword arguments for the scorer. per_component (bool): Whether to return the scores keyed by component name. Defaults to False. RETURNS (Scorer): The scorer containing the evaluation results. DOCS: https://spacy.io/api/language#evaluate zLanguage.evaluateroc34K|]}|jywrrrs rkrz$Language.evaluate..s -bR\\ -r)rr)rc3FK|]}t|jywr)rrrs rkrz$Language.evaluate..s;Bc",,';s!speedri)rr=rrrerr7timerr referencerrrrscorerr)rrrrrrrr start_timerdocsrend_timeresultsn_wordss rkevaluatezLanguage.evaluates-6>($78!(+  J  M  J >*%F   eT *%f%FW  -B MM",,++ , -yy -H -!'  8T* GBBL 7,,x},E;(;;"h&;<$W--rjcZd|jddi}tj|dS)zCCreate an optimizer, usually using the [training.optimizer] config. optimizerr)rQrGr)r subconfigs rkrzLanguage.create_optimizers0 $++j"9+"FG  *;77rjparamsc#@K|sdy|jDcgc].\}}t|drt|dr|j|0}}}|D]} t|d|D]} t|ycc}}w#t$rY:wxYw#t$rY2wxYww)aReplace weights of models in the pipeline with those provided in the params dictionary. Can be used as a contextmanager, in which case, models go back to their original weights after the block. params (dict): A dictionary of parameters keyed by model ID. EXAMPLE: >>> with nlp.use_params(optimizer.averages): >>> nlp.to_disk("/tmp/checkpoint") DOCS: https://spacy.io/api/language#use_params N use_paramsr])rrr.next StopIteration)rr,rrcontextsrs rkr.zLanguage.use_paramss #'--D$4.743I'H$ M  # M % %sXB3A: B B B, B7 B B  B B  B BBBB) as_tuplesrrr n_processtextsr2r3cyrrirr4r2rrrr3s rkrz Language.pipesrjcyrrir6s rkrz Language.pipes-0rjr!c#lK|rstttttt ft f|}fd|D}j|||||}|D]} | j} d| _| | fyttttt f|}|dk(rtj}|i}| j}g} jD]g\} } | |vr |j| i}|jd|tj t"| | |j$}| j'|i|dk7rIj)|r#t+j,t.j0j3|| ||}nfd|D}| D] }||} |D]} | yw) aXProcess texts as a stream, and yield `Doc` objects in order. texts (Iterable[Union[str, Doc]]): A sequence of texts or docs to process. as_tuples (bool): If set to True, inputs should be a sequence of (text, context) tuples. Output will then be a sequence of (doc, context) tuples. Defaults to False. batch_size (Optional[int]): The number of texts to buffer. disable (List[str]): Names of the pipeline components to disable. component_cfg (Dict[str, Dict]): An optional dictionary with extra keyword arguments for specific components. n_process (int): Number of processors to process texts. If -1, set `multiprocessing.cpu_count()`. YIELDS (Doc): Documents in the order of the original text. DOCS: https://spacy.io/api/language#pipe c3HK|]\}}j||ywr)r)r~rrrs rkrz Language.pipe..s'"ANw--dG<"s")rrr3rNr)rrrrr!c3@K|]}j|ywr)r)r~rrs rkrz Language.pipe..Os=tD$$T*=s)rrrrrdr:rLrrmp cpu_countrrrr functoolspartialrDrr_has_gpu_modelr_r`r'W114_multiprocessing_pipe)rr4r2rrrr3docs_with_contextsr$rrrrrrfrs` rkrz Language.pipes8 %c3h(D"EFNE"RW" 99"%#+ D %,,# Gn$ % XeCHo.6 ? I  M  J-- JD$w"&&tR0F   lJ 7!!&*&@&@ A LLO  >""7+ hmm,--eUIzRD>u=D "Dz " CI sF1F4c|jD]p\}}t|dxr |j}||vs|s't|ds4t|jdsKt |jj t spyy)Nrr]rTF)rrrr]rrr)rrrrrs rkr@zLanguage._has_gpu_modelUsp-- JD$"48NT=N=NLwlg&DJJ.tzz~~w7 rjrc #dKdttttfdttttt ft ffd}||}tj|\}}t|Dcgc]}tj} }tt|Dcgc]}tjdc}\} } tj||} t!| | |} | j#| j#t| | Dcgc]A\}}tj$t&|j(|||t+j,fC}}}|D]}|j/| D]}|j1t3j4dt7| D} t9t||dD]\}\}\}}}|0t|j:j=|}||_|nR|PtAjB|}|jEdddtGtHjJjM| ||zd k(s| jO | D]'}|jQtR|j1)| D]}|j1|D]}|jUtWd |Ds$tYjZt\j^yycc}wcc}wcc}}w#| D]'}|jQtR|j1)| D]}|j1|D]}|jUtWd |Ds$tYjZt\j^wwxYww) Nr4rlc3K|D]S}t|tr-|jtt|j f@|ttdfUywr)rr:to_bytesrrLr)r4rs rk prepare_inputz5Language._multiprocessing_pipe..prepare_inputksT " >h,#,,.[(BSBS0TUU#T+t%<==  >sAAF) chunk_size)targetrc3<K|]}|jywr)recv)r~rMs rkrz1Language._multiprocessing_pipe..s*  DIIK* sr!rrc3:K|]}|jdk(yw)rN)exitcode)r~rs rkrz1Language._multiprocessing_pipe..sE#s(O, > eE#u*-{:; < >%2%$8!$==)BCy7C#*CLI+!// ;E..dD*V[[5G5Ge5G5T*Uz>Q&KKM "& )*  &      j||}t |d|vr(|jd}|jA|||||d|vsJ||jBjEddg}|d} | |vr&tjF| |jBdg|| <|jd |}!d!}"d"|vrU|| jI|| jJD]0\}#}$|!tM|$d#gvs|| jO|#|!|d"d$}"2tQjR5tQjTd%d&'|jA|!|| |(ddd| |vr6tW|| jBjXjEdg|| <d)|jZvri|jZd)<|| |jZd)|<|"s\|| =a||jBj]|t)|t^r|g}t)|t^r|g}t)t^rgta|tatbk7rn|djd*g}%te|%rNtg|%ji|s4tQjjtljnj||%+|jqtsh||djd,g||dd}&tgfd-|&D|_:|dd.|_;|r| n||_<|G||}t)||s3t t j2jd/t+||S#1swYxYw)0a5Create the nlp object from a loaded config. Will set up the tokenizer and language data, add pipeline components etc. If no config is provided, the default config of the given language is used. config (Dict[str, Any] / Config): The loaded config. vocab (Vocab): A Vocab object. If True, a vocab is created. disable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to disable. Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling nlp.enable_pipe. enable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to enable. All other pipes will be disabled (and can be enabled using `nlp.enable_pipe`). exclude (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to exclude. Excluded components won't be loaded. meta (Dict[str, Any]): Meta overrides for nlp.meta. auto_fill (bool): Automatically fill in missing values in config based on defaults and function argument annotations. validate (bool): Validate the component config and arguments against the types expected by the factory. RETURNS (Language): The initialized Language class. DOCS: https://spacy.io/api/language#from_config rOrorrz@vectorszspacy.Vectors.v1rN) bad_lang_coderRrrr)rLrrbefore_creationafter_creationafter_pipeline_creation)r)r{r}rrcreation)rrrrOrFrr[)rrQ)rrQrLrKrrer)r{rrFreplace_listenerslistening_componentsTignorez\[W113\])message)r[rrenabled)rrrc3,K|] }|vs| ywrri)r~rrs rkrz'Language.from_config..sJ!'9IAJs rpipeline_creation)=rrrArrr&E985rrrE958r$rArdrWrGrVr2rr4rr issubclassE943rHE942is_interpolatedrcrUrE956E984r{r{rH load_modelrwrgetattrrr_catch_warningsfilterwarningshashrrrrdidr@rrgissubsetr`r'W123_resolve_component_statusrrrrQ)'rrQr{rrrrr}rL config_lang orig_pipelineorig_pretrainingrY resolved_nlpr}rrrrlang_clsro interpolatedr source_nlpssource_nlp_vectors_hashesvocab_brrGpipe_cfgrKr,rr]rZlisteners_replacedrrrdisabled_pipess' ` rk from_configzLanguage.from_configsF ""2FeFm   V[[//v/>? ? F5M )(24F'GF5M) $Um''/  "{chh'> """(-"7!hh--c2# !$u f !!&) <4 !::mT:!| ]]6H\RFF,|,|  '$4F= !$4F= !'' 5M" ( 4%i0&'89%&67"./H"I  &&s+Hx.!(C03& !3!3$x.!3!IJJ  -)    % %Cc3' !3!34PS9!3!UVV4:3I3Iv))+v ##L"5 $&!z2H /I(yy1 !3!3!3!NOO''(;@!:;1%8HH67 B*'.QH /T   II  ) gs #iG fc "XF gs #iG f:01 1Um'' 26G7|CL$9$9&$A MM((% ')66 ?7?VE]..z2>? @  5M* %  J~JJ |4(Vf " .)#.Cc3' KK&&,?tCy&Q {s /ZZ tok2vec_namer listenersc||jvrGtjj|||dj |j}t |||jvrGtjj|||dj |j}t ||j |}|j|}t|tjs3t tjj|t||j}|jj|g}|j |} |j |} |rt"j$j'd|t)t+|t)|k7r7tj,j|||t)|}t ||D]d} t#j.| | |d} d|j4vr!|j4d} | |d| dd } t#j6| | | f|D]}|j9}|j4jd }|ot)t;j<|j>}|d k(r ||}n9|d k(r ||||}n)t tj@j|t#jB| j|||jE||y y #t0$r.tj2j||| }t |wxYw)aFind listener layers (connecting to a token-to-vector embedding component) of a given pipeline component model and replace them with a standalone copy of the token-to-vector layer. This can be useful when training a pipeline with components sourced from an existing pipeline: if multiple components (e.g. tagger, parser, NER) listen to the same tok2vec component, but some of them are frozen and not updated, their performance may degrade significantly as the tok2vec component is updated with new data. To prevent this, listeners can be replaced with a standalone tok2vec layer that is owned by the component and doesn't change if the component isn't updated. tok2vec_name (str): Name of the token-to-vector component, typically "tok2vec" or "transformer". pipe_name (str): Name of pipeline component to replace listeners for. listeners (Iterable[str]): The paths to the listeners, relative to the component config, e.g. ["model.tok2vec"]. Typically, implementations will only connect to one tok2vec component, [model.tok2vec], but in theory, custom models can use multiple listeners. The value here can either be an empty list to not replace any listeners, or a complete (!) list of the paths to all listener layers used by the model. DOCS: https://spacy.io/api/language#replace_listeners rO)tok2vecrunknownrG)rrz%Replacing listeners of component '%s')rrpaths n_listeners)rrrr]replace_listener_cfgrreplace_listenerNr!) num_params)#rr&E889rrUrrJrrr#rvE888rr]rwrrr$rrrrE887 dot_to_objectrHE886attrsset_dot_to_objectcopyinspect signature parametersE1055replace_model_noderemove_listener)rrrrr,r tok2vec_cfg tok2vec_modelpipe_listenersrr listener_path new_config replace_funclistener new_modelreplace_listener_funcrs rkrzLanguage.replace_listenerss: t .++$$$$YYt/ %C S/ ! DOO +++$$$!YYt/ %C S/ !-- -**<8 '2#9#9:V[[//\W /VW W  --11)R@}}Y'%%i0  KK  Ey Q4 ?#s>'::kk(("(# #N 3 ) !o%"+ L *&&x? )1 )]-@-@@#0#6#67M#NL!-#G,hw.? .J"J&&x K L"+ =)..0 (5(;(;(?(?@R(S%(4"%))*?@KK"J"Q$9)$D #q$9)Xw$W ()<)< )<)STT'' HiH'')<# =A & * ++,,& =-C%S/) *s "L7Mmemc #K| t}t5}|j|jj |g}t |j dr9|j|j|j j ||jD]A\}}t |ds|j|j|j |C|dddy#1swYyxYww)aBegin a block where all resources allocated during the block will be freed at the end of it. If a resources was created within the memory zone block, accessing it outside the block is invalid. Behaviour of this invalid access is undefined. Memory zones should not be nested. The memory zone is helpful for services that need to process large volumes of text with a defined memory budget. Example ------- >>> with nlp.memory_zone(): ... for doc in nlp.pipe(texts): ... process_my_doc(doc) >>> # use_doc(doc) <-- Invalid: doc was allocated in the memory zone N memory_zone) rr enter_contextr{rrrrr)rrstackr1rrs rkrzLanguage.memory_zone4s$ ;&C[ E++DJJ,B,B3,GHIHt~~}5 3 3DNN4N4Ns4S TU== P44/OOE$7$78H8H8M$NO PI   s#C=BC135C1( C=1C:6C=rrctj|}i}fd|d<fd|d<fd|d<jD]\}}|vr t|ds|fd||<!fd |d <tj||y ) aaSave the current state to a directory. If a model is loaded, this will include the model. path (str / Path): Path to a directory, which will be created if it doesn't exist. exclude (Iterable[str]): Names of components or serialization fields to exclude. DOCS: https://spacy.io/api/language#to_disk c@jj|dgSNr{r)rto_diskrrs rkz"Language.to_disk..bs#T^^-C-C y.D. rjrcVtj|tjSr)r write_jsonrrrs rkrz"Language.to_disk..es!U-=-= $TYY/. rj meta.jsonc:jj|Sr)rQrrs rkrz"Language.to_disk..hsdkk.A.A!.Drj config.cfgrc*|j|dgSr)rrrs rkrz"Language.to_disk..nsT\\!gY\5Wrjc>jj|SNr)r{r)rrrs rkrz"Language.to_disk..os););Aw);)Orjr{N)r$ ensure_pathrrr)rrr serializersrrs` ` rkrzLanguage.to_diskTs% $ K $ K %E L!** XJD$w4+/3 WK   X P G T;0rjrc*t|tr|g}|}|rmt|tr|g}h|Dcgc] }||vs| c}|}tt||zr*t t j j||t|Scc}w)afDerives whether (1) `disable` and `enable` values are consistent and (2) resolves those to a single set of disabled components. Raises an error in case of inconsistency. disable (Union[str, Iterable[str]]): Name(s) of component(s) or serialization fields to disable. enable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to enable. pipe_names (Iterable[str]): Names of all pipeline components. RETURNS (Tuple[str, ...]): Names of components to exclude from pipeline w.r.t. specified includes and excludes. )rr) rrdrrgrr&E1042rr)rrrrrs rkrz"Language._resolve_component_statusrs$ gs #iG &#& -7S 9F;R)SJ 3v;+, !4!4FG!4!TUUZ  Ts BB)r overridesrcdtddffd }dtddffd }tj|}i}t|dz jr fd|d<||d<||d <fd |d <jD]\}}|vr t |d s|fd ||<!|d z jsd vrt d gztj|||_jS)aLoads state from a directory. Modifies the object in place and returns it. If the saved `Language` object contains a model, the model will be loaded. path (str / Path): A path to a directory. exclude (Iterable[str]): Names of components or serialization fields to exclude. RETURNS (Language): The modified `Language` object. DOCS: https://spacy.io/api/language#from_disk rrlNc|jrktj|}jj ||j dij dj j_yyNrr) existsr read_jsonrrrr{rr)rrrs rkdeserialize_metaz,Language.from_disk..deserialize_metas[{{}t,   &+/((9b*A*E*Ef*M ""' rjcb|jrjj|yyr)rr{ from_disk)rrrs rkdeserialize_vocabz-Language.from_disk..deserialize_vocabs({{} $$T7$;rjrc@jj|dS)NF)rcr)rQr)rrrs rkrz$Language.from_disk..s#DKK4I4Iu 5J5rjrr{c@jj|dgSr)rrrs rkrz$Language.from_disk..s#t~~/G/G y0H0 rjrrc*|j|dgSr)rrs rkrz$Language.from_disk..st~~G98F8rj) r r$rrrrrrrrw) rrrrrr deserializersrrs ` `` rkrzLanguage.from_disks$ N4 ND N .stzz':':7':'Krjr{c>jjdgSr)rrHrsrkrz#Language.to_bytes..s4>>+B+BG9+B+UrjrcTtjtjSr)r json_dumpsrrrsrkrz#Language.to_bytes..s5+;+; !$)) ,, rjrc8jjSr)rQrHrsrkrz#Language.to_bytes..sDKK,@,@,BrjrrHc(|jdgSr)rH)rs rkrz#Language.to_bytes..s$-- -2Rrj)rrr$rH)rrrrrs`` rkrHzLanguage.to_bytess79 K G#U K $ K %C L!** SJD$w4,,0 RK   S }}['22rj bytes_datacfd}i}fd|d<||d<fd|d<fd|d<jD]\}}|vr t|d s|fd ||<!tj||j S) a!Load state from a binary string. bytes_data (bytes): The data to load from. exclude (Iterable[str]): Names of components or serialization fields to exclude. RETURNS (Language): The `Language` object. DOCS: https://spacy.io/api/language#from_bytes ctj|}jj||j dij dj j _yr)r json_loadsrrrr{rr)brrs rkrz-Language.from_bytes..deserialize_metasO##A&D II  T "'+hhy"&=&A&A&&IDJJ   #rjc>jj|dS)NF)rc)rQrrrs rkrz%Language.from_bytes..s! 0F0F 51G1 rjrrc>jj|Sr)r{r)rrrs rkrz%Language.from_bytes..s4::+@+@G+@+Trjr{c@jj|dgSr)rrrs rkrz%Language.from_bytes..s#t~~/H/H y0I0 rjrrc*|j|dgSr)r)rrs rkrz%Language.from_bytes.. stG98G8rj)rrr$rrw)rrrrrrrs` ` rkrzLanguage.from_bytess J<> ' l#&6 k"!T g& k"** JD$w4.15#M$     M7;  rj)Tr)NNNN)rlrrlN)qr_r`rarbrNrurrrdrcrrrBr&r*rrr rrJboolrfr r r:rIrrpropertyrrsetterrrQrrrrr9rrrrrr classmethodrrrrrrrCrfloatrrr/rJrPrir{rurrrrrrrrrrrrrLrr<rrrrrrrrrr7r(rrrer.rr%rrr@rBrwr@rrrrr r staticmethodrrrHr __classcell__)rs@rkrmrmsHD(3-#N v{{3I.0M4]*+0%)F1 !SWEIF1UD[!F1 F1 38n F1 #8ZL(C5#::N,N#OP F1!7)[*@!ABF1F1 F1P5 #d38n##J [[$sCx.T   D ]]FtR$s)RR'tCy'' DsL'8!9:  YcYYR$uS,%678RRTDITT+S#X++ (T#tCy.1 ( (QsQtQQ $C$C$$ HC HM H H>C> >$>>%#%-% C F  *:);!1!3"2"4!7#>i > c5j)* >  S$sCx.%8 9: >#> c5j >DCG6$( 6xHW,=(=>?6i 6  6CGJ$( JxHW,=(=>?Ji J  JX=Ahy&9Y26lDIy I8 ST6,%)#'=A/3#9.7#9.SM 9.  9.  S$sCx.%8 9: 9.T#s(^,9.9. c3h9.v8 !$!!F %($'!$=@ c3h( 5>  SM  #  S$sCx.%8 9:   #   $'$'!$=@ 0eCHo{:;< 04= 0 SM 0 # 0 S$sCx.%8 9: 0 0 %[() * 0 0" $(!1!3=AS U38_ %xeCHo{6R0S'T T S  SSMS#S S$sCx.%8 9:SS x}huS+-='>?? @Sj hsm f-c3h(f-hsm!345f- f-  f- # f-P0,13g%)-A,@-A/1gd38nf,-gUD[! g sHSM)* g c8C=() gsHSM)*g38nggg ggRf=f=f=C= f=  f=Px~$@CSBT1#t)$12:3-1 1< !sHSM)* !c8C=() !SM ! sCx ! !L"2!3$4$6 7CI7# 7 S> 7  7r4D3E38C=3%30>N=O&&-5c]& &rj meta_dictc2tddt|S)Nc6t|tjSr)rnumpyfloatingvs rkrz'_replace_numpy_floats.. s*Q/rjct|Sr)rrs rkrz'_replace_numpy_floats.. s 58rj)r re)r s rkrr s /1CT)_ rjceZdZUdZeed<dZeeee fed<e Z e eed<e Z e eed<dZeed<e Ze eed <dZeeeeefed <y) raDataclass containing information about a component and its defaults provided by the @Language.component or @Language.factory decorator. It's created whenever a component is defined and stored on the Language class for each component instance and factory instance. rNrrrFrr$r)r_r`rarbrdrcrrr r rrrrrrr$rrrirjrkrr s} L/3NHT#s(^,3"WGXc]$#gHhsm%K!GFHSM#BF8Dhuo)=$>?Frjrc>eZdZdZdedeeddfdZdZdZ d d Z y) rz)Manager for temporary pipeline disabling.rorrlNc||_||_|jD]}|jj|tj ||j |jyr)rorrrrextend)rrorrs rkrzDisabledPipes.__init__( sQ JJ (D HH ! !$ ' ( d DJJrjc|Srrirs rk __enter__zDisabledPipes.__enter__0 s rjc$|jyr)restore)rrs rk__exit__zDisabledPipes.__exit__3 s  rjc|jD]^}||jjvr)ttj j ||jj|`g|ddy)zARestore the pipeline to its state when DisabledPipes was created.rN)rrorrr&E008rrrs rkrzDisabledPipes.restore6 saJJ 'D488333 !3!3!3!>?? HH  & 'Qrjr) r_r`rarbrmrrdrrrrrirjrkrr% s23 H T#Y 4 rjrrc|Dcgc]0}t|jj|j2c}Scc}w)zMake a copy of a batch of examples, copying the predicted Doc as well. This is used in contexts where we need to take ownership of the examples so that they can be mutated, for instance during Language.evaluate and Language.update. )r<rrr)rrs rkrr? s- 2: :2GBDDIIK & :: :s5= ensure_docr.underscore_statectj| |j}t|tr!|j |j yfd|D}|D] }||} |Dcgc]}|j |jdf!} }dgt|t| z z} | | z} |j| cc}w#t$rEddtjtjfg} dgtdz z} | | z} YewxYw#t$r#|j |j YywxYw)a9Worker for Language.pipe ensure_doc (Callable[[Union[str, Doc]], Doc]): Function to create Doc from text or raise an error if the input is neither a Doc nor a string. pipes (Iterable[Pipe]): The components to apply. receiver (multiprocessing.Connection): Pipe to receive text. Usually created by `multiprocessing.Pipe()` sender (multiprocessing.Connection): Pipe to send doc. Usually created by `multiprocessing.Pipe()` underscore_state (Tuple[dict, dict, dict]): The data in the Underscore class of the parent. Nc36K|]\}}||ywrri)r~rrrs rkrz_apply_pipes..f s!2C(G 8W-s)NNNr!)r; load_staterr_WorkDoneSentinelr\rHrrrr msgpack_dumps traceback format_excrWBrokenPipeError) rrreceiverrkr texts_with_ctxr$rr byte_docspaddingr error_msgs ` rkrYrYH s\&*+  '%\\^N.*;<  GUD "Dz "JNN##,,.#,,=NIN)*c..AC N.RSGG#   KK 7  O  'e&9&9):N:N:P&QRSI)*c..AA.EFGw&D '  LLN NN    s=ACC8$C"C?D'CA D$#D$')EEcXeZdZdZdeedeejde ddfdZ d dZ d d Z y) rVzAUtil for sending data to multiprocessing workers in Language.piperqueuesrJrlNcrt||_tt||_||_d|_y)Nr)iterrrr/rJcount)rrr/rJs rkrz_Sender.__init__ s-J 5=) $ rjctjt|jt |j |j D]\}}|j|y)z1Send chunk_size items from self.data to channels.N)rPislicerrrr/rJra)ritemrss rkrWz _Sender.send sJ ''  5- .  GD! EE$K  rjc|xjdz c_|j|jk\rd|_|jyy)znTell sender that comsumed one item. Data is sent to the workers after every chunk_size calls. r!rN)r2rJrWrs rkr`z _Sender.step s6 a :: (DJ IIK )rjr) r_r`rarbrr rr<rSrfrrWr`rirjrkrVrV s@KSM+/>GJ rjrVc eZdZy)r$N)r_r`rarirjrkr$r$ srjr$)r>rrPmultiprocessingr<rr&r_ contextlibrrrr dataclassesrrrpathlibr timeitr r typingr r r rrrrrrrrrrrrrrr cymem.cymemr thinc.apirrrr thinc.utilr rr"r#r$compatr%errorsr&r'git_infor(lang.punctuationr)r*r+lang.tokenizer_exceptionsr,r-rr. pipe_analysisr/r0r1schemasr2r3r4r5r6rr7rr9tokensr:tokens.underscorer;rr<r=training.initializer>r?r@rArBrCrDrErFrGrHrrIr{rJrKr9__file__parentDEFAULT_CONFIG_PATH load_configrDEFAULT_CONFIG_PRETRAIN_PATHrLrNr}rrmrerrrrrrdrrYrVr$rbrirjrkrPs 0!")& AA($!WWA!MM )09   !&s # 8n++.BB!!!"56 $H~447WWm$ QQ&(J<#:;2 A"A"HDTd   G G  G D4;Xg.;4=;6%S%0+>CD6 HS(3-/0 16 D$,- 6  6r:  ()rj