ij:^dZ ddlZddlZddlZddlZddlZddlZddlZddlZGdde Z ddZ dZ ddZ ddZd Zd Zd Zed k(rej&j)ej&j+ej,ej.Zej&j3ed Zej&j7er4ej8dej:dj=eeej@dkrvejBdejDe_"ejBdejFe_#ejHdejJe_%nejLejJjNde_%ejLejDjNde_"ejLejFjNddde_#e Z(e(jSZ*ejVe*jXjZde*_,e*j\jZdk7r-ejVe*j\jZde*_.e*j^jZdk7r.ejVe*j^jZdde*_/e*j`r-ejVe*j`jZde*_0e*j`ree*j`e*jbZ0ndZ0ej@dkrZe*jdjgde*_2e*jhr.e*jhDcgc]}|jgdc}e*_4e e*jXe*jje*jde0e*jhZ6e*j\D],Z7e*j^jqe6jse7.yycc}w)a) Copyright (c) 2004-2020 Joerg Tiedemann Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. Nc,eZdZddZdZdZdZdZy)BPENc |jdd}|j}|jdrbtt j dd|j dj dDcgc] }t|c}|_|dz }nd|_|jdt|D cgc]8\} } | |ks|dk(r)t| jd j d :c} } |_ t|jD]\} } t| d k7stjjd j!| |zd j#| tjjd tj$dt't)t+t|jD cgc] \} } | | f c} } |_ t'|jj-D cgc]\} } | d| dz| fc} } |_||_||_|r|ng|_i|_ycc}wcc} } wcc} } wcc} } w)Nrz #version:z(\.0+)*$.rr  z/Error: invalid line {0} in BPE codes file: {1} zLThe line should exist of exactly two subword units, separated by whitespace )seekreadline startswithtupleresubsplitintversion enumeratestrip bpe_codeslensysstderrwriteformatjoinexitdictreversedlistitemsbpe_codes_reverse separatorvocab glossariescache)selfcodesmergesr&r'r(offset firstlinexnitemicodepairs b/home/spazio/kingdom_calls/work5/transenv/lib/python3.12/site-packages/argostranslate/apply_bpe.py__init__z BPE.__init__/s 1 NN$    ,  VVKY__5Fr5JKQQRUVFDL aKF!DL JJqM'u- DF fl $**W%++C0 1  !0 GAt4yA~   FMMF CHHTN    c  (0i6O1P(Q R9AtdAY R "&6:nn6J6J6L M74d1gQ & M" # (2* W ( S Ns3I==II I& cd}t|t|jdz }|r||d|z }||j|z }t|t|jdz }|r|t|k7r ||| dz }|S)z:segment line, dealing with leading and trailing whitespacerr N)rlstripsegmentrstrip)r*lineoutleading_whitespacetrailing_whitespaces r5 process_linezBPE.process_linees YT[[-A)BB  4++, ,C t||D!!!$i#dkk'.B*CC #6#d)#C 4,,-. .C c|j|jdjd}dj|S)zGsegment single sentence (whitespace-tokenized string) with BPE encodingr r )segment_tokensrrr)r*sentencesegmentss r5r9z BPE.segmentvs5&&x~~g'>'D'DS'IJxx!!r@cg}|D]}|s|j|Dcgc]a}t||j|j|j|j |j |j|jD]}|c}}}|ddD] }|j||j z"|j|d|Scc}}w)z.segment a sequence of tokens with BPE encodingNr) _isolate_glossariesencoderr%r'r&rr)r(append)r*tokensoutputwordr9r<new_wordr1s r5rBzBPE.segment_tokens{s (D $77= !NN**JJNNLLJJOO    H !"  5 dT^^34 5 MM(2, '+ (. ' sA&Cc ~|g}|jD]$}|Dcgc]}t||D]}|}}}&|Scc}}wN)r(isolate_glossary)r*rK word_segmentsglossr9 out_segmentss r5rFzBPE._isolate_glossariessd __ E -$4We$D!M   s9)r@@NN)__name__ __module__ __qualname__r6r?r9rBrFr@r5rr.s4l"" 8r@rc|r#|jdtjd}n%tjtjd}|j ddtj dt jdd |j d d tj ddd d |j ddtddd |j ddtj dt jdd |j ddtddd |j dtj dddd |j dtddd |j d!td"ddd#$|S)%Nz apply-bpez!learn BPE-based word segmentation)formatter_class descriptionz--inputz-irPATHz%Input file (default: standard input).)typedefaultmetavarhelpz--codesz-cTz.File with BPE codes (created by learn_bpe.py).)r]r_requiredr`z--mergesz-mrINTzjUse this many BPE operations (<= number of learned symbols)default: Apply all the learned merge operationsz--outputz-owz&Output file (default: standard output)z --separatorz-srSSTRzCSeparator between non-final subword units (default: '%(default)s'))z --vocabularyzuVocabulary file (built with get_vocab.py). If provided, this script reverts any merge operations that produce an OOV.z--vocabulary-thresholdzkVocabulary threshold. If vocabulary is provided, any word with frequency < threshold will be treated as OOVz --glossaries+a'Glossaries. Words matching any of the words/regex provided in glossaries will not be affected by the BPE (i.e. they will neither be broken into subwords, nor concatenated with other subwords. Can be provided as a list of words/regex after the --glossaries argument. Enclose each regex in quotes.)r]nargsr^r_r`) add_parserargparseRawDescriptionHelpFormatterArgumentParser add_argumentFileTyperstdinrstdoutstr) subparsersparsers r5 create_parserrrs&& $@@;'  (($@@;      s #  4     s # =   <     s #  5    R     s #E    z   t   Mr@cbt}|d}|ddD]}|j||f|}|S)zReturn set of symbol pairs in a word. word is represented as tuple of symbols (symbols being variable-length strings) rrN)setadd)rKpairs prev_charchars r5 get_pairsrysF EEQIQR 9d#$  Lr@c||vr||Stjdjdj||r |f||<|fS|dk(rt |dz}n$|dk(rt |dd|ddzfz}nt t |} | s|S t| fd  } | vrn| \} } g} d }|t|kr |j| |}| j||||}||| k(r6|t|d z kr%||d z| k(r| j| | z|d z }n| j|||d z }|t|krt | } | }t|d k(rn t |} |ddk(r|dd}n0|djdr|dd|djddfz}|rt||||}|||<|S#| j||dYxYw)zREncode word based on list of BPE merge operations, which are applied consecutivelyz^({})$|r ))rr Nrr|c:j|tdS)Ninf)getfloat)r4rs r5zencode.."sY]]4u-Nr@)keyrrr r)rmatchrrrNotImplementedErrorryminrindexextendrHendswithreplacecheck_vocab_and_split)origrr%r'r&rr)r(rKrvbigramfirstsecondrLr2js ` r5rGrGs' u}T{ xx 45t<gd w&T{Y& F T#2Y48f#4"66!! dOE  U NO  "  v #d)m JJua(Qq * Aw%AD A $5$q1u+:O/QQ(Q#d)m? t9> dOE7 < Bx6CRy b  6 "CRyDH,,VR8:: $T+zrcrN)F):__doc__rhcodecsinspectioosrrwarningsobjectrrrryrGrrrrOrTpathdirnameabspathgetfile currentframe currentdirrnewdirisdir simplefilterwarnrDeprecationWarning version_info getwriterrrn getreaderrm TextIOWrapperbufferrq parse_argsargsopenr+nameinputrJrvocabulary_thresholdr&decoder(r,bper;rr?)gs0r5rsf ,  q&qhPf *FR82 P& z (<(<(<(>?@JWW\\*m 4F ww}}Vi(  J Q Q     & .%V%%g.szz: .%V%%g.szz: -$F$$W-cii8 $B$$SYY%5%5H %R%%cjj&7&7'J %R%% JJ  tTX  _F    DTZZ__w?DJ zz)# V[[7C  {{:%!fkk$++"2"2C'J  %&++doo&:&:WM $T__d6O6OP   & ..w7 ??:>//JQqxx0JDO djj$++t~~z4?? SC 2 #**4012g^KsP*