Ë
    ›¾j¿   ã                   óV   — d dl mZ d dlZd dlmZmZ d dlmZ d dlm	Z	  G d„ de
«      Zy)é    )Úprint_functionN)ÚCounterÚdefaultdict)Úreduce)Úpairwisec                   ó>   — e Zd ZdZd„ Zd
d„Zd„ Zd„ Zd„ Zd„ Z	dd	„Z
y)ÚSubwordTokenizerzd
    This is a Python port of the Subword NMT from
    https://github.com/rsennrich/subword-nmt
    c                 ó²   — | j                  |«      | _        | j                  «       \  | _        | _        t        j                  | j                  «      | _        y ©N)Úget_vocabularyÚvocabÚget_pair_statisticsÚstatsÚindicesÚcopyÚdeepcopyÚ	big_stats)ÚselfÚfilenames     ú]/home/spazio/kingdom_calls/work5/transenv/lib/python3.12/site-packages/sacremoses/subwords.pyÚ__init__zSubwordTokenizer.__init__   s?   € à×(Ñ(¨Ó2ˆŒ
Ø#'×#;Ñ#;Ó#=Ñ ˆŒ
�D”LÜŸ™ t§z¡zÓ2ˆ�ó    c                 óæ  — t        «       }t        |«      5 }|r@|D ]:  }|j                  «       j                  d«      \  }}||xx   t	        |«      z  cc<   Œ< n-|j                  |j                  «       j                  «       «       d d d «       t        |j                  «       D ��	ci c]  \  }}	t        |d d «      |d   dz   fz   |	“Œ  c}	}«      }|j                  «       S # 1 sw Y   Œ\xY wc c}	}w )Nú éÿÿÿÿu   î€€)
r   ÚopenÚstripÚsplitÚintÚupdateÚreadÚitemsÚtupleÚmost_common)
r   r   Úis_dictr   ÚfinÚlineÚwordÚcountÚkÚvs
             r   r   zSubwordTokenizer.get_vocabulary   sÜ   € Ü“	ˆÜ�(‹^ð 	1˜sÙØò .�DØ"&§*¡*£,×"4Ñ"4°SÓ"9‘K�D˜%Ø˜$“K¤3 u£:Ñ-”Kñ.ð —‘˜SŸX™X›Z×-Ñ-Ó/Ô0÷	1ô ØBGÇ+Á+Ã-×P¹¸¸AŒU�1�S�b�6‹]˜a ™e hÑ.Ð0Ñ0°!Ñ3ÓPó
ˆð × Ñ Ó"Ð"÷	1ð 	1üó Qs   –A0C!Â&#C-
Ã!C*c                 óÞ   — t        «       }t        d„ «      }t        | j                  «      D ]<  \  }\  }}t	        |«      D ]&  \  }}|||fxx   |z  cc<   |||f   |xx   dz  cc<   Œ( Œ> ||fS )z5Count frequency of all symbol pairs, and create indexc                  ó   — t        «       S r   )r   © r   r   ú<lambda>z6SubwordTokenizer.get_pair_statistics.<locals>.<lambda>-   s   € ¤g£i€ r   é   )r   r   Ú	enumerater   r   )r   r   r   Úir(   ÚfreqÚprevÚcurrs           r   r   z$SubwordTokenizer.get_pair_statistics(   s†   € ô “	ˆäÑ/Ó0ˆä(¨¯©Ó4ò 	,‰OˆA‰|��dÜ& t›nò ,‘
��dØ�d˜D�jÓ! TÑ)Ó!Ø˜˜d˜
Ñ# AÓ&¨!Ñ+Ô&ñ,ð	,ð
 �gˆ~Ðr   c                 óŠ   ‡‡‡— |\  ŠŠdj                  |«      j                  dd«      Šˆˆˆfd„}t        ||dd |d   f«      S )zÆ
        From https://stackoverflow.com/a/40367074/610569
            >>> modify_token(('s', 'h', 'e', 'r', 'l', 'o', 'c', 'k'), ('h', 'e'))
            ('S', 'he', 'r', 'l', 'o', 'c', 'k')
        Ú ú\z\\c                 ó<   •— | d   ‰k(  r|‰k(  r	| d d ‰fz   S | |fz   S )Nr   r.   )ÚaccÚeÚfirstÚpair_strÚseconds     €€€r   r/   z/SubwordTokenizer.modify_token.<locals>.<lambda>?   s7   ø€ Ø�2‰w˜%Ò A¨¢Kð ˜s ˜8 x kÑ1€ à˜�t‘ð r   r0   Nr   )ÚjoinÚreplacer   )r   ÚtokenÚpairÚfr<   r=   r>   s       @@@r   Úmodify_tokenzSubwordTokenizer.modify_token6   sO   ú€ ð ‰ˆˆvØ—7‘7˜4“=×(Ñ(¨¨vÓ6ˆõð 	
ô
 �a˜˜q˜r˜ U¨1¡X KÓ0Ð0r   c                 óô   — g }| j                   |   j                  «       D ]U  \  }}|dk  rŒ| j                  |   \  }}| j                  ||«      }||f| j                  |<   |j	                  ||||f«       ŒW |S )zJReplace all occurrences of a symbol pair ('A', 'B') with a new symbol 'AB'r0   )r   r"   r   rD   Úappend)r   rB   ÚchangesÚjr3   r(   Únew_words          r   Úreplace_pairzSubwordTokenizer.replace_pairE   s‡   € àˆØ—|‘| DÑ)×/Ñ/Ó1ò 	6‰GˆAˆtØ�aŠxØØŸ™ A™‰JˆD�$Ø×(Ñ(¨¨tÓ4ˆHØ% tÐ,ˆD�J‰J�q‰MØ�N‰N˜A˜x¨¨tÐ4Õ5ð	6ð ˆr   c                 óø  — d| j                   |<   t        «       | j                  |<   |\  }}||z   }|D ]ö  \  }}}}	d}
	 	 |j                  ||
«      }
|
t        |«      dz
  k  rÃ||
dz      |k(  r¸|
r<||
dz
  |
dz    }| j                   |xx   |	z  cc<   | j                  |   |xx   dz  cc<   |
t        |«      dz
  k  rc||
dz      |k7  s|
t        |«      dz
  k\  s||
dz      |k7  r<||
dz   |
dz    }| j                   |xx   |	z  cc<   | j                  |   |xx   dz  cc<   |
dz  }
n|
dz  }
Œí y# t        $ r Y nw xY wd}
	 	 |j                  ||
«      }
n# t        $ r Y �Œ,w xY w|
r<||
dz
  |
dz    }| j                   |xx   |	z  cc<   | j                  |   |xx   dz  cc<   |
t        |«      dz
  k  rD||
dz      |k7  r9||
|
dz    }| j                   |xx   |	z  cc<   | j                  |   |xx   dz  cc<   |
dz  }
Œ½)z×
        Minimally update the indices and frequency of symbol pairs
        if we merge a pair of symbols, only pairs that overlap with occurrences
        of this pair are affected, and need to be updated.
        r   r0   é   é   N)r   r   r   ÚindexÚ
ValueErrorÚlen)r   rB   Úchangedr<   r>   Únew_pairrH   r(   Úold_wordr3   r2   r4   Únexs                r   Úupdate_pair_statisticsz'SubwordTokenizer.update_pair_statisticsQ   sm  € ð ˆ�
‰
�4ÑÜ$›Yˆ�‰�TÑØ‰ˆˆvØ˜6‘>ˆØ'.ò O	Ñ#ˆAˆt�X˜tð ˆAð ðà Ÿ™ u¨aÓ0�Að ”s˜8“} qÑ(Ò(¨X°a¸!±e©_ÀÒ-Fñ à'¨¨A©°°A±Ð6˜ØŸ
™
 4Ó(¨DÑ0Ó(ØŸ™ TÑ*¨1Ó-°Ñ2Ó-àœ3˜x›=¨1Ñ,Ò,ð % Q¨¡U™O¨uÒ4Ø ¤C¨£M°AÑ$5Ò5Ø'¨¨A©™°&Ò8ð #+¨1¨q©5°1°q±5Ð"9˜CØ ŸJ™J s›O¨tÑ3›OØ ŸL™L¨Ñ-¨aÓ0°AÑ5Ó0ð ˜‘F‘Aà˜‘F�AðS ñO	øô "ò ÙðúðR ˆAð ðØŸ
™
 8¨QÓ/‘AøÜ!ò Úðúñ
 à  A¡¨¨A©Ð.�Dà—J‘J˜tÓ$¨Ñ,Ó$Ø—L‘L Ñ& qÓ)¨QÑ.Ó)ð
 ”s˜4“y 1‘}Ò$¨¨a°!©e©¸Ò)@à˜q 1 q¡5˜/�Cà—J‘J˜s“O tÑ+“OØ—L‘L Ñ% aÓ(¨AÑ-Ó(à�Q‘�ð3 s#   ¿D-Ä-	D9Ä8D9Å EÅ	E ÅE Nc                 óH  — t        | j                  j                  «       «      dz  }t        |«      D ]n  }| j                  j	                  |«      }|D ]L  \  }}	| j                  |«      }
| j                  ||
«       d| j                  |<   | j                  |   |k  sŒK  y  Œp y )Né
   r   )Úmaxr   ÚvaluesÚranger$   rJ   rU   )r   Únum_symbolsÚmin_freqÚjumpr%   Ú	thresholdr2   Úmost_freq_tokensrA   r)   rG   s              r   ÚlearnzSubwordTokenizer.learn¬   sœ   € ä˜Ÿ
™
×)Ñ)Ó+Ó,¨rÑ1ˆ	Ü�{Ó#ò 	ˆAØ#Ÿz™z×5Ñ5°dÓ;ÐØ 0ò ‘��uØ×+Ñ+¨EÓ2�Ø×+Ñ+¨E°7Ô;Ø$%�—
‘
˜5Ñ!Ø—:‘:˜eÑ$ xÓ/Úññ	r   )F)rL   r0   N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r   rD   rJ   rU   r`   r.   r   r   r	   r	      s,   „ ñò
3ó#ò ò1ò
òYôv
r   r	   )Ú
__future__r   r   Úcollectionsr   r   Ú	functoolsr   Úsacremoses.utilr   Úobjectr	   r.   r   r   ú<module>rj      s$   ðõ &ã ß ,Ý å $ôj�võ jr   