Ë
    ›¾ju  ã                   ó4   — d dl Z d dlZd dlmZ  G d„ d«      Zy)é    N)Úchainc                   ó‚   — e Zd ZdZg d¢ZddgZg d¢Zg d¢Zg d¢ZdgZ	d	d
gZ
dgZdgZg d¢Z	 	 	 	 	 	 	 dd„Zd„ Zd„ Zd„ Zy)ÚMosesPunctNormalizerz¯
    This is a Python port of the Moses punctuation normalizer from
    https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/normalize-punctuation.perl
    )
)z\rÚ )z\(z ()z\)z) ©z +ú )z\) ([.!:?;,])z)\g<1>)z\( ú()z \)ú))z(\d) %z\g<1>%)z :ú:)z ;ú;)ú`ú')ú''z " ))u   â€žú"©u   â€œr   ©u   â€�r   )u   â€“ú-)u   â€”z - r   )õ   Â´r   )u   ([a-zA-Z])â€˜([a-zA-Z])ú\g<1>'\g<2>)u   ([a-zA-Z])â€™([a-zA-Z])r   )u   â€˜r   )u   â€šr   ©õ   â€™r   )r   r   )u   Â´Â´r   ©u   â€¦z...))õ   Â Â«Â r   )u   Â«Â r   )õ   Â«r   )õ   Â Â»Â r   )u   Â Â»r   )õ   Â»r   )
)u   Â %ú%)u   nÂºÂ u   nÂº )u   Â :r   )u   Â ÂºCu    ÂºC)u   Â cmz cm)u   Â \?ú?)u   Â \!ú!)u   Â ;r   )u   ,Â z, r   )z"([,.]+)z\g<1>")z,"z",)z(\.+)"(\s*[^<])z"\g<1>\g<2>)õ
   (\d)Â (\d)z\g<1>,\g<2>)r    z\g<1>.\g<2>)$)u   ï¼Œú,)u   ã€‚\s*ú. )u   ã€�r!   r   r   )u   âˆ¶r   )u   ï¼šr   )u   ï¼Ÿr   )u   ã€Šr   )u   ã€‹r   )u   ï¼‰r
   )u   ï¼�r   )u   ï¼ˆr	   )u   ï¼›r   )u   ã€�r   )u   ã€Œr   )u   ï¼�Ú0)u   ï¼‘Ú1)u   ï¼’Ú2)u   ï¼“Ú3)u   ï¼”Ú4)u   ï¼•Ú5)u   ï¼–Ú6)u   ï¼—Ú7)u   ï¼˜Ú8)u   ï¼™Ú9)u   ï¼Ž\s*r"   )u   ï½žú~r   r   )u   â”�r   )u   ã€ˆú<)u   ã€‰ú>)u   ã€�ú[)u   ã€‘ú])u   ï¼…r   c                 óÂ  — |r-d| j                   d<   d| j                  d<   d| j                  d<   | j                  | j                   | j                  | j                  g| _        |r&| j                  j                  d| j                  «       |rT|dk(  r&| j                  j                  | j                  «       n)|d	v r%| j                  j                  | j                  «       |rO|d
v r&| j                  j                  | j                  «       n%| j                  j                  | j                  «       t        t        | j                  Ž «      | _        || _        || _        y)aÂ  
        :param language: The two-letter language code.
        :type lang: str
        :param penn: Normalize Penn Treebank style quotations.
        :type penn: bool
        :param norm_quote_commas: Normalize quotations and commas
        :type norm_quote_commas: bool
        :param norm_numbers: Normalize numbers
        :type norm_numbers: bool
        :param perl_parity: exact parity with perl script
        :type perl_parity: bool
        )r   r   é   )r   z "r   )r   z" é   é   Úen)ÚdeÚesÚfr)r7   r8   ÚczÚcsr9   N)ÚNORMALIZE_UNICODEÚFRENCH_QUOTESÚEXTRA_WHITESPACEÚHANDLE_PSEUDO_SPACESÚsubstitutionsÚinsertÚNORMALIZE_UNICODE_IF_NOT_PENNÚappendÚEN_QUOTATION_FOLLOWED_BY_COMMAÚ$DE_ES_FR_QUOTATION_FOLLOWED_BY_COMMAÚDE_ES_CZ_CS_FRÚOTHERÚlistr   Úpre_replace_unicode_punctÚpost_remove_control_chars)ÚselfÚlangÚpennÚnorm_quote_commasÚnorm_numbersrI   rJ   Úperl_paritys           ú^/home/spazio/kingdom_calls/work5/transenv/lib/python3.12/site-packages/sacremoses/normalize.pyÚ__init__zMosesPunctNormalizer.__init__   s5  € ñ. Ø)6ˆD×"Ñ" 2Ñ&Ø$=ˆD×Ñ˜qÑ!Ø$=ˆD×Ñ˜qÑ!ð ×!Ñ!Ø×"Ñ"Ø×ÑØ×%Ñ%ð	
ˆÔñ Ø×Ñ×%Ñ% a¨×)KÑ)KÔLáØ�tŠ|Ø×"Ñ"×)Ñ)¨$×*MÑ*MÕNØÐ+Ñ+Ø×"Ñ"×)Ñ)¨$×*SÑ*SÔTáØÐ5Ñ5Ø×"Ñ"×)Ñ)¨$×*=Ñ*=Õ>à×"Ñ"×)Ñ)¨$¯*©*Ô5ä!¤%¨×);Ñ);Ð"<Ó=ˆÔà)BˆÔ&Ø)BˆÕ&ó    c                 óþ   — | j                   r| j                  |«      }| j                  D ]%  \  }}t        j                  ||t        |«      «      }Œ' | j                  r| j                  |«      }|j                  «       S )z?
        Returns a string with normalized punctuation.
        )	rI   Úreplace_unicode_punctr@   ÚreÚsubÚstrrJ   Úremove_control_charsÚstrip©rK   ÚtextÚregexpÚsubstitutions       rQ   Ú	normalizezMosesPunctNormalizer.normalize¶   sv   € ð
 ×)Ò)Ø×-Ñ-¨dÓ3ˆDð %)×$6Ñ$6ò 	;Ñ ˆF�Lä—6‘6˜& ,´°D³	Ó:‰Dð	;ð ×)Ò)Ø×,Ñ,¨TÓ2ˆDà�z‰z‹|ÐrS   c                 ón   — | j                   D ]%  \  }}t        j                  ||t        |«      «      }Œ' |S )N)ÚREPLACE_UNICODE_PUNCTUATIONrV   rW   rX   r[   s       rQ   rU   z*MosesPunctNormalizer.replace_unicode_punctÊ   s7   € Ø$(×$DÑ$Dò 	;Ñ ˆF�LÜ—6‘6˜& ,´°D³	Ó:‰Dð	;àˆrS   c                 ó0   — t        j                  dd|«      S )Nz\p{C}r   )ÚregexrW   )rK   r\   s     rQ   rY   z)MosesPunctNormalizer.remove_control_charsÏ   s   € Ü�y‰y˜ 2 tÓ,Ð,rS   N)r6   TTTFFF)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r>   rB   r<   r=   r?   rD   rE   rF   rG   ra   rR   r_   rU   rY   © rS   rQ   r   r   
   sœ   „ ñò
Ðð &2°?Ð$CÐ!òÐò$€MòÐð '?Ð%?Ð"ð 	Ø,ð,Ð(ð 	-ð€Nð
 	-ð€Eò%#ÐðR ØØØØ"'Ø"'Øó5Cònò(ó
-rS   r   )rV   rc   Ú	itertoolsr   r   rh   rS   rQ   ú<module>rj      s   ðó 
Û å ÷F-ò F-rS   