ij"ddlmZddlZddlmZddlmZ ddlZddl Z ddl m Z m Z ddlmZmZddlmZddlmZdd lmZmZeej2d z e _dd ZdZej:ej<j>k(r eeZGd d Z Gdde Z!Gdde Z"Gdde Z#dZ$dZ%dZ&dZ'ddZ( d d dZ)d!dZ*d"dZ+ d d#dZ,y#e$rdZYwxYw)$) annotationsN)SequenceMatcher)List)SBDetectmodels)packagesettings) cache_spacy)Package)infowarningminisbdc^ ||jdrdSdS#ttf$rYywxYw)zFGet appropriate processors for a language, including MWT if available.mwtz tokenize,mwttokenize)getKeyError TypeError) lang_code resourcess \/home/spazio/kingdom_calls/work5/transenv/lib/python3.12/site-packages/argostranslate/sbd.pyget_stanza_processorsrs;!*9!5!9!9%!@~PjP i s,,c eZdZUded<ddZy)ISentenceBoundaryDetectionModelr pkgctN)NotImplementedErrorselftexts rsplit_sentencesz/ISentenceBoundaryDetectionModel.split_sentences(s!!Nr!strreturnz List[str])__name__ __module__ __qualname____annotations__r"r#rrr$s L"r#rc"eZdZddZddZdZy)SpacySentencizerSmallcd||_t td|j?dt |jvr(tj |jdg|_n2t tdtj tdg|_|j jdy)z Packaging specific spacy when "xx_sent_ud_sm" doesn't cover the language improves performances over stanza. Please use small models ".._core/web_sm" for consistency. NzBSpaCy is not installed. Install spacy or change ChunkType settingsspacyparser)excludezSpaCy cache not initialized sentencizer) rr/ RuntimeErrorpackaged_sbd_pathr%loadnlp_cached_spacy_pathadd_piper rs r__init__zSpacySentencizerSmall.__init__2s  =cd d  ,C@U@UeZdZdddddddddZd dZd Zdd Zd Zy )MiniSBDSentencizerzh-hantzzh-hanspttrhien)ztzhpbazbneomstlc||_|jdz }d}|jrGtj|Dcgc]}|j ds|}}|rt ||dz }||}nj|jj|j|j}|tjvr$t|jjdd}||_ d|_ycc}w)Nrz.onnxrz0 is not available in MiniSBD, falling back to enrM)r package_pathexistsoslistdirendswithr%LANGUAGE_CODE_MAPPINGrr=minisbd_models list_modelsr langdetector)r rsbd_path model_filef model_filesr_s rr:zMiniSBDSentencizer.__init__\s##i/ ?? &(jj&:Rajj>Q1RKR KN!:;  !D--11 s}}D >5577488--..^_`  'Ss C.C.c|j-t|jtjdk(|_|jS)Ncuda)use_gpu)r`rr_r devicerCs r lazy_detectorz MiniSBDSentencizer.lazy_detectorxs1 == $TYY68QRDM}}r#ctd|jjdt||j j |Sr<)r rr=r%ri sentencesrs rr"z"MiniSBDSentencizer.split_sentences}sB 5dhh6H6H5ICPTI; WX!!#--d33r#cy)NrHr+rCs rrDzMiniSBDSentencizer.__str__rEr#NrFr$)r'r(r)r\r:rir"rDr+r#rrHrHMs8 8 4$r#rHc2eZdZdddZd dZdZd dZdZy) StanzaSentencizerrIrJ)rNrPc||_|jj|j|j|_d|_yr)rr\rr=stanza_lang_codestanza_pipeliner9s rr:zStanzaSentencizer.__init__s9 $ : : > > MM3==!  $r#c|jZtj|jt |j j dz dtjdk(d|_|jS)NstanzarrfWARNING)r_dir processorsrg logging_level) rqrsPipelinerpr%rrWr rhrCs r lazy_pipelinezStanzaSentencizer.lazy_pipelines_    '#)??**--89% 61' $D ###r#ctd|jjdt||j |}|j Dcgc]}|j c}Scc}wr<)r rr=r%ryrkr!r?s rr"z!StanzaSentencizer.split_sentencessY 5dhh6H6H5ICPTI; WX"d  "4(&)mm4d 444sA)cy)Nrnr+rCs rrDzStanzaSentencizer.__str__s"r#NrFr$)r'r(r)r\r:ryr"rDr+r#rrnrns$ $ $5 #r#rna I walked down to the river. Then I went to the I walked down to the river. ---------- Argos Translate is machine translation software. It is also Argos Translate is machine translation software. ---------- Argos Translate is written in Python and uses OpenAI. It also supports Argos Translate is written in Python and uses OpenAI. ---------- zzz ----------c`tj}|D]}|jdk(s|cSy)Nsbd)rget_installed_packagestype)packagesrs rget_sbd_packagers5--/H 88u J r#c@|d|}tdz|z}td||S)Nz generate_fewshot_sbd_prompt)fewshot_promptr ) input_textsentence_guess_lengthsentence_guess to_returns rrrs3 6!67N!@@>QI & 2 r#c|jt}td|t|dkry|djd}t|dkry|dS)Nparse_fewshot_response )splitFEWSHOT_BOUNDARY_TOKENr len) response_textresponses rrrsZ""#9:H !8, 8}q|!!$'H 8}q B<r#c|jt}|dk7rp|d|}td|d}d}tt |D]B}|d|}t }|j |||j}|dk(s||kDs?|}|}D|Sy)Nrzsbd_translated_guess:rg)findSENTENCE_BOUNDARY_TOKENr rangerrset_seqsratio) rsbd_translated_guesssbd_translated_guess_index best_index best_ratioicandidate_sentencesmrs rprocess_seq2seq_sbdrs!5!:!:;R!S!R'34O5OP $&:;  s:' #A!+BQ  "B KK*,@ AHHJEAv+ "  #r#cl|d|}td||jt|z}t||S)aGiven input text, return the index after the end of the first sentence. Args: input_text: The text to detect the first sentence of. sbd_translation: An ITranslation for detecting sentences. sentence_guess_length: Estimated number of chars > than most sentences. Returns: The index of the character after the end of the sentence. -1 if not found. Nzsentence_guess:)r translate DETECT_SENTENCE_BOUNDARIES_TOKENr)rsbd_translationrrrs rdetect_sentencersF 6!67N N+*44(>9 z+? @@r#)rr%rdictr&r%)r&zPackage | None))rr%rintr&r%)rr%r&z str | None)rr%rr%r&r)rr%rrr&r)- __future__rrYdifflibrtypingrr/ ImportErrorrsrrrr]argostranslaterr argostranslate.networkingr argostranslate.packager argostranslate.utilsr r r%data_dir cache_dirrr7 chunk_type ChunkTypeSPACYrr-rHrnrrrrrrrrrr+r#rrs8" #6,1*.x009<= (,,222u7H$""$;$85$85$p#7#P $B /!36,/*DGAA=@AAI EsC&&C0/C0