Ë
    \›¾j£7  ã                   óR  — d Z ddlZddlZddlZddlZddlZddlmZ ddlZddlZ	ddl
Z
ddlZddlmZmZ ddlmZ ddlmZ ddlmZ ddlmZmZ dd	lmZ ddlmc mc mZ dd
l­ ddlm Z  ddl!m"Z"  ejF                  d«      Z$d„ Z%dd„Z&dd„Z'd„ Z(d„ Z)d„ Z*d„ Z+e,dk(  r e'«        yy)a,  
Entry point for training and evaluating a lemmatizer.

This lemmatizer combines a neural sequence-to-sequence architecture with an `edit` classifier 
and two dictionaries to produce robust lemmas from word forms.
For details please refer to paper: https://nlp.stanford.edu/pubs/qi2018universal.pdf.
é    N)Údatetime)ÚnnÚoptim)Ú
DataLoader)ÚVocab)ÚTrainer)ÚscorerÚedit)Úutils)Ú*)ÚCoNLL)Ú_training_loggingÚstanzac                  ór  — t        j                  «       } | j                  dt        dd¬«       | j                  dt        d d¬«       | j                  dt        d d¬«       | j                  dt        d d	¬«       | j                  d
t        d d	¬«       | j                  ddddg¬«       | j                  dt        d¬«       | j                  dddd¬«       | j                  ddd¬«       | j                  dt        d¬«       | j                  dt        d¬«       | j                  d t        d!¬«       | j                  d"t
        d#¬«       | j                  d$t
        d#¬«       | j                  d%t        d¬«       | j                  d&t        d!¬«       | j                  d'd(g d)¢d*¬+«       | j                  d,t        d¬«       | j                  d-t
        d#¬«       | j                  d.d/dd0¬«       | j                  d1t        t        t        j                  «      ¬«       | j                  d2t
        d3¬«       | j                  d4d5dd6¬«       | j                  d7d8dd9¬«       | j                  d:dd;¬«       | j                  d<t        d d=¬«       | j                  d>t        d d?¬«       | j                  d@t        d dA¬«       | j                  dBt
        d3dC¬«       | j                  dDt        dEdF¬«       | j                  dGt
        dHdI¬«       | j                  dJt
        dK¬«       | j                  dLt        dMdN¬«       | j                  dOt        dP¬«       | j                  dQt        d¬«       | j                  dRt
        dSdT¬«       | j                  dUt        dVdW¬«       | j                  dXt        dYdZ¬«       | j                  d[t        d\d]¬«       | j                  d^d_dd`¬a«       | j                  dbt        dc¬«       t        j                  | «       | j                  dddde¬«       | j                  dfd dg¬h«       | S )iNz
--data_dirz
data/lemmazDirectory for all lemma data.)ÚtypeÚdefaultÚhelpz--train_filezInput file for data loader.z--eval_filez--output_filezOutput CoNLL-U file.z--gold_filez--modeÚtrainÚpredict)r   Úchoicesz--shorthandz/Shorthand for the dataset to use.  lang_dataset)r   r   z	--no_dictÚensemble_dictÚstore_falsezADo not ensemble dictionary with seq2seq. By default use ensemble.)ÚdestÚactionr   z--dict_onlyÚ
store_truez)Only train a dictionary-based lemmatizer.)r   r   z--hidden_diméÈ   )r   r   z	--emb_dimé2   z--num_layersé   z--emb_dropoutg      à?z	--dropoutz--max_dec_lenz--beam_sizez--attn_typeÚsoft)r   ÚmlpÚlinearÚdeepzAttention type)r   r   r   z	--pos_dimz--pos_dropoutz	--no_editr
   zODo not use edit classifier in lemmatization. By default use an edit classifier.z
--num_editz--alphag      ð?z--no_posÚposz:Do not use UPOS in lemmatization. By default UPOS is used.z	--no_copyÚcopyzhDo not use copy mechanism in lemmatization. By default copy mechanism is used to improve generalization.z--charlmzVTurn on contextualized char embedding using pretrained character-level language model.z--charlm_shorthandz=Shorthand for character-level language model training corpus.z--charlm_forward_filez$Exact path to use for forward charlmz--charlm_backward_filez%Exact path to use for backward charlmz--sample_trainzSubsample training data.z--optimÚadamzsgd, adagrad, adam or adamax.z--lrgü©ñÒMbP?zLearning ratez
--lr_decaygÍÌÌÌÌÌì?z--decay_epoché   z&Decay the lr starting from this epoch.z--num_epoché<   z--batch_sizez--max_grad_normg      @zGradient clipping.z
--log_stepé   zPrint log every k steps.z
--save_dirzsaved_models/lemmazRoot dir for saving models.z--save_namez%{shorthand}_{embedding}_lemmatizer.ptzFile name to save the modelz
--caselessFznLowercase everything first before processing.  This will happen automatically if 100%% of the data is caseless)r   r   r   z--seediÒ  z--wandbzStart a wandb session and write the results of training.  Only applies to training.  Use --wandb_name instead to specify a namez--wandb_namezWName of a wandb session to start when training.  Will default to the dataset short name)r   r   )ÚargparseÚArgumentParserÚadd_argumentÚstrÚintÚfloatÚlenr
   Ú
EDIT_TO_IDr   Úadd_device_args)Úparsers    úb/home/spazio/kingdom_calls/work5/transenv/lib/python3.12/site-packages/stanza/models/lemmatizer.pyÚbuild_argparser4   !   sI  € Ü×$Ñ$Ó&€FØ
×Ñ˜¬3¸ÐKjÐÔkØ
×Ñ˜¬S¸$ÐEbÐÔcØ
×Ñ˜¬C¸ÐDaÐÔbØ
×Ñ˜¬c¸4ÐF\ÐÔ]Ø
×Ñ˜¬C¸ÐDZÐÔ[à
×Ñ˜¨'¸GÀYÐ;OÐÔPØ
×Ñ˜¬CÐ6gÐÔhà
×Ñ˜¨/À-ð  WZÐô  [Ø
×Ñ˜¨lÐAlÐÔmà
×Ñ˜¬S¸#ÐÔ>Ø
×Ñ˜¬#°rÐÔ:Ø
×Ñ˜¬S¸!ÐÔ<Ø
×Ñ˜¬e¸SÐÔAØ
×Ñ˜¬%¸ÐÔ=Ø
×Ñ˜¬c¸2ÐÔ>Ø
×Ñ˜¬C¸ÐÔ;à
×Ñ˜¨vÒ?`ÐgwÐÔxØ
×Ñ˜¬#°rÐÔ:Ø
×Ñ˜¬e¸SÐÔAØ
×Ñ˜¨&¸ð  N_Ðô  `Ø
×Ñ˜¬3¼¼D¿O¹OÓ8LÐÔMØ
×Ñ˜	¬°sÐÔ;Ø
×Ñ˜
¨°}ð  LHÐô  IØ
×Ñ˜¨&¸ð  NxÐô  yà
×Ñ˜
¨<ð  ?WÐô  XØ
×ÑÐ,´3Àð  LKÐô  LØ
×ÑÐ/´cÀ4ÐNtÐÔuØ
×ÑÐ0´sÀDÐOvÐÔwà
×ÑÐ(¬u¸cÐHbÐÔcØ
×Ñ˜	¬°VÐBaÐÔbØ
×Ñ˜¤U°D¸ÐÔOØ
×Ñ˜¬5¸#ÐÔ>Ø
×Ñ˜¬c¸2ÐDlÐÔmØ
×Ñ˜¬C¸ÐÔ<Ø
×Ñ˜¬S¸"ÐÔ=Ø
×ÑÐ)´¸sÐI]ÐÔ^Ø
×Ñ˜¬3¸ÐA[ÐÔ\Ø
×Ñ˜¬3Ð8LÐSpÐÔqØ
×Ñ˜¬CÐ9`ð  hEÐô  Fà
×Ñ˜¨e¸Lð  P@Ðô  Aà
×Ñ˜¤s°DÐÔ9Ü	×Ñ˜&Ô!à
×Ñ˜	¨,ð  >Ðô  @Ø
×Ñ˜°ð  <UÐô  VØ€Mó    c                 óÀ   — t        «       }|j                  | ¬«      } | j                  rd| _        t	        | «      } | d   r| d   j                  d«      d   nd}|| d<   | S )N©ÚargsTÚ	shorthandÚ_r   Ú Úlang)r4   Ú
parse_argsÚ
wandb_nameÚwandbÚvarsÚsplit)r8   r2   r<   s      r3   r=   r=   Z   se   € ÜÓ€FØ×Ñ $ÐÓ'€Dà‡‚ØˆŒ
ä�‹:€Dà.2°;Ò.?ˆ4�Ñ×"Ñ" 3Ó'¨Ò*ÀR€DØ€Dˆ�LØ€Kr5   c                 óØ   — t        | ¬«      } t        j                  | d   «       t        j	                  dj                  | d   «      «       | d   dk(  rt        | «       y t        | «       y )Nr7   ÚseedzRunning lemmatizer in {} modeÚmoder   )r=   r   Úset_random_seedÚloggerÚinfoÚformatr   Úevaluater7   s    r3   ÚmainrJ   g   sT   € Ü˜4Ô €Dä	×Ñ˜$˜v™,Ô'ä
‡K�KÐ/×6Ñ6°t¸F±|ÓDÔEàˆF�|�wÒÜˆd�ä��r5   c                 óœ   — | j                   D ]=  }|j                  D ],  }|j                  j                  «       |j                  k7  sŒ+  y Œ? y)NFT)Ú	sentencesÚwordsÚtextÚlower)ÚdocÚsentenceÚwords      r3   Úall_lowercaserS   s   sG   € Ø—M‘Mò ˆØ—N‘Nò 	ˆDØ�y‰y�‰Ó  D§I¡IÓ-Úñ	ðð r5   c                 ó  — d}| d   r| d   rd}| d   j                  | d   |¬«      }t        j                  j                  |«      d   }|j	                  | d   «      s#t        j                  j                  | d   |«      }|S )	NÚnocharlmÚcharlmÚcharlm_forward_fileÚ	save_namer9   )r9   Ú	embeddingr   Úsave_dir)rH   ÚosÚpathrA   Ú
startswithÚjoin)r8   rY   Ú
model_fileÚ	model_dirs       r3   Úbuild_model_filenamera   z   sˆ   € Ø€IØˆH‚~˜$Ð4Ò5Øˆ	Ø�kÑ"×)Ñ)°D¸Ñ4EØ4=ð *ó ?€Jä—‘—‘˜jÓ)¨!Ñ,€IØ×Ñ  ZÑ 0Ô1Ü—W‘W—\‘\ $ zÑ"2°JÓ?ˆ
ØÐr5   c                 óŒ  — t         j                  dj                  | d   «      «       t        j                  | d   ¬«      }t        || d   | d¬«      }|j                  }|d   j                  | d<   |d	   j                  | d
<   t        j                  | d   ¬«      }t        || d   | |d¬«      }t        j                  | d   «       t        | «      }t         j                  d|«       | d   }| d   }t        j                  | «       t        |«      dk(  st        |«      dk(  rt         j                  d«       y | d   s%t        |«      rt         j                  d«       d| d<   t         j                  d|«       t        | || d   ¬«      }	t         j                  d«       |	j!                  |j#                  «       «       t         j                  d«       |	j%                  |j&                  j)                  t*        t,        g«      «      }
|j&                  j/                  t0        g|
«       t        j2                  |j&                  |«       t5        j6                  ||«      \  }}}t         j                  dj                  |dz  «      «       | j)                  dd«      r|	j9                  |«       y | d   rcdd l}| d   r| d   nd | d!   z  }|j=                  || ¬"«       |j>                  jA                  d#d$¬%«       |j>                  jA                  d&d'¬%«       t         j                  d(«       d}t        |«      | d)   z  }g }g }| d*   }tC        jB                  «       }d+}tE        d,| d)   d,z   «      D �]Ï  }d}tG        |«      D ]¨  \  }}tC        jB                  «       }|d,z  }|	jI                  |d¬-«      }||z  }|| d.   z  dk(  sŒCtC        jB                  «       |z
  }t         j                  |j                  tK        jL                  «       jO                  d/«      |||| d)   |||«      «       Œª t         j                  d«       g }
g }tG        |«      D ]*  \  }}|	jQ                  || d0   «      \  }}|
|z  }
|€Œ&||z  }Œ, |	jS                  |j&                  j)                  t*        g«      |
|¬1«      }
| j)                  d2d«      rJt         j                  d3«       |	jU                  |j&                  j)                  t*        t,        g«      |
«      }
|j&                  j/                  t0        g|
«       t        j2                  |j&                  |«       t5        j6                  ||«      \  }}} ||jV                  z  | d   z  }t         j                  d4j                  ||| «      «       | d   rjY                  || d5œ«       |d,k(  s| t[        |«      kD  r(|	j9                  |«       t         j                  d6«       |
}|| d7   kD  r(| |d8   k  r | d9   d:v r|| d;   z  }|	j]                  |«       || gz  }t         j                  d<«       �ŒÒ t         j                  d=j                  «      «       | d   rj_                  «        t[        |«      dz  ta        jb                  |«      d,z   }"}!t         j                  d>j                  |!|"«      «       y )?Nz$[Loading data with batch size {}...]Ú
batch_sizeÚ
train_file©Ú
input_fileF)Ú
evaluationÚcharÚ
vocab_sizer#   Úpos_vocab_sizeÚ	eval_fileT©Úvocabrg   rZ   zUsing full savename: %sÚoutput_fileÚ	gold_filer   z5[Skip training because no training data available...]ÚcaselesszBBuilding a caseless model, as all of the training data is caselesszBuilding lemmatizer in %sÚdevice)r8   rm   rq   z)[Training dictionary-based lemmatizer...]zEvaluating on dev set...zDev F1 = {:.2f}éd   Ú	dict_onlyr?   r>   z%s_lemmatizerr9   )ÚnameÚconfigÚ
train_lossÚmin)ÚsummaryÚ	dev_scoreÚmaxz&[Training seq2seq-based lemmatizer...]Ú	num_epochÚlrzJ{}: step {}/{} (epoch {}/{}), loss = {:.6f} ({:.3f} sec/batch), lr: {:.6f}r   )ÚevalÚlog_stepz%Y-%m-%d %H:%M:%SÚ	beam_size©Úeditsr   z'[Ensembling dict with seq2seq model...]z1epoch {}: train_loss = {:.6f}, dev_score = {:.4f})rv   ry   znew best model saved.Údecay_epochéÿÿÿÿr   )ÚsgdÚadagradÚlr_decayr;   zTraining ended with {} epochs.z#Best dev F1 = {:.2f}, at epoch = {})2rF   rG   rH   r   Ú	conll2docr   rm   Úsizer   Ú
ensure_dirra   Úprint_configr/   ÚwarningrS   r   Ú
train_dictÚraw_dataÚpredict_dictrP   ÚgetÚTEXTÚUPOSÚsetÚLEMMAÚwrite_doc2conllr	   ÚscoreÚsaver?   ÚinitÚrunÚdefine_metricÚtimeÚrangeÚ	enumerateÚupdater   ÚnowÚstrftimer   ÚpostprocessÚensembleÚnum_examplesÚlogrz   Ú	update_lrÚfinishÚnpÚargmax)#r8   Ú	train_docÚtrain_batchrm   Údev_docÚ	dev_batchr_   Úsystem_pred_filero   ÚtrainerÚ	dev_predsr:   Údev_fr?   r>   Úglobal_stepÚ	max_stepsÚdev_score_historyÚbest_dev_predsÚ
current_lrÚglobal_start_timeÚ
format_strÚepochrv   ÚiÚbatchÚ
start_timeÚlossÚdurationÚ	dev_editsÚpredsr�   ry   Úbest_fÚ
best_epochs#                                      r3   r   r   …   s%  € ä
‡K�KÐ6×=Ñ=¸dÀ<Ñ>PÓQÔRÜ—‘¨4°Ñ+=Ô>€IÜ˜Y¨¨\Ñ(:¸DÈUÔS€KØ×Ñ€EØ˜v™×+Ñ+€DˆÑØ" 5™\×.Ñ.€DÐ	ÑÜ�o‰o¨¨kÑ):Ô;€GÜ˜7 D¨Ñ$6¸ÀEÐVZÔ[€Iä	×Ñ�T˜*Ñ%Ô&Ü% dÓ+€JÜ
‡K�KÐ)¨:Ô6ð ˜MÑ*ÐØ�[Ñ!€Iä	×Ñ�tÔô ˆ;Ó˜1Ò¤ I£°!Ò 3Ü�‰ÐNÔOØà�
Ò¤¨iÔ 8Ü�‰ÐXÔYØˆˆZÑô ‡K�KÐ+¨ZÔ8Ü˜4 u°T¸(±^ÔD€GÜ
‡K�KÐ;Ô<Ø×Ñ�{×+Ñ+Ó-Ô.Ü
‡K�KÐ*Ô+Ø×$Ñ$ Y§]¡]×%6Ñ%6¼¼d°|Ó%DÓE€IØ‡M�M×Ñ”u�g˜yÔ)Ü	×Ñ˜)Ÿ-™-Ð)9Ô:Ü—,‘,Ð/°Ó;�K€A€qˆ%Ü
‡K�KÐ!×(Ñ(¨°©Ó5Ô6à‡x�x�˜UÔ#à�‰�ZÕ à�Š=ÛØ/3°LÒ/A˜˜lÒ+ÀÐY]Ð^iÑYjÑGjˆJØ�J‰J˜J¨tˆJÔ4Ø�I‰I×#Ñ# L¸%Ð#Ô@Ø�I‰I×#Ñ# K¸Ð#Ô?ô 	�‰Ð<Ô=ØˆÜ˜Ó$ t¨KÑ'8Ñ8ˆ	ØÐØˆØ˜$‘Zˆ
Ü ŸI™I›KÐØaˆ
ô ˜1˜d ;Ñ/°Ñ1Ó2ó 2	ˆEØˆJÜ% kÓ2ò t‘��5Ü!ŸY™Y›[�
Ø˜qÑ �Ø—~‘~ e°%�~Ó8�Ø˜dÑ"�
Ø  jÑ!1Ñ1°QÓ6Ü#Ÿy™y›{¨ZÑ7�HÜ—K‘K 
× 1Ñ 1´(·,±,³.×2IÑ2IÐJ]Ó2^Ð`kØ2;¸UÀDÈÑDUÐW[Ð]eÐgqó!sõ tðtô �K‰KÐ2Ô3ØˆIØˆIÜ% iÓ0ò '‘��5Ø&Ÿ™¨u°d¸;Ñ6GÓH‘��uØ˜UÑ"�	ØÑ$Ø Ñ&‘Ið	'ð
  ×+Ñ+¨I¯M©M×,=Ñ,=¼t¸fÓ,EÀyÐXaÐ+ÓbˆIð �x‰x˜¨Ô/Ü—‘ÐEÔFØ#×,Ñ,¨Y¯]©]×->Ñ->ÄÄd¸|Ó-LÈiÓX�	Ø�M‰M×Ñœu˜g yÔ1Ü×!Ñ! )§-¡-Ð1AÔBÜ$Ÿl™lÐ+;¸YÓG‰OˆAˆq�)à# k×&>Ñ&>Ñ>ÀÀlÑASÑSˆJÜ�K‰KÐK×RÑRÐSXÐZdÐfoÓpÔqà�GŠ}Ø—	‘	¨À)ÑLÔMð ˜Šz˜Y¬Ð->Ó)?Ò?Ø—‘˜ZÔ(Ü—‘Ð3Ô4Ø!*�ð �t˜MÑ*Ò*¨yÐ<MÈbÑ<QÒ/QØ˜‘MÐ%7Ñ7Ø˜d :Ñ.Ñ.�
Ø×!Ñ! *Ô-à ) Ñ,ÐÜ�K‰K˜ŽOðe2	ôh 	�‰Ð4×;Ñ;¸EÓBÔCà�Š=Ø�L‰LŒNä Ð!2Ó3°CÑ7¼¿¹ÐCTÓ9UÐVWÑ9W�
ˆÜ�‰Ð9×@Ñ@ÀÈÓTÕUr5   c                 óº  — | d   }| d   }t        | «      }t        || d   | ¬«      }|j                  |j                  }}| D ]1  }|j	                  d«      s|j	                  d«      s|dv sŒ*| |   ||<   Œ3 t
        j                  dj                  | d	   «      «       t        j                  | d
   ¬«      }t        || d	   ||d¬«      }	t        |	«      dk(  r(t
        j                  dj                  | d   «      «       y |j                  |	j                  j                  t         t"        g«      «      }
|j                  dd«      r|
}�nt
        j                  d«       g }g }t%        |	«      D ]*  \  }}|j'                  || d   «      \  }}||z  }|€Œ&||z  }Œ, |j)                  |	j                  j                  t         g«      ||¬«      }|j                  dd«      rJt
        j                  d«       |j+                  |	j                  j                  t         t"        g«      |«      }|j-                  «       r|j/                  |	j                  |«      }|	j                  j1                  t2        g|«       t        j4                  |	j                  |«       |�Ft7        j8                  ||«      \  }}}t
        j                  dj                  | d   |dz  «      «       y y )Nrn   ro   rq   )r_   rq   r8   Ú_dirÚ_file)r9   z"Loading data with batch size {}...rc   rk   re   Trl   r   zDSkip evaluation because no dev data is available...
Lemma score:
{} r9   rs   FzRunning the seq2seq model...r   r€   r   z,[Ensembling dict with seq2seq lemmatizer...]z*Finished evaluation
Lemma score:
{} {:.2f}rr   )ra   r   r8   rm   ÚendswithrF   rG   rH   r   r‡   r   r/   r‹   rŽ   rP   r�   r�   r‘   rœ   r   r    r¡   Úhas_contextual_lemmatizersÚupdate_contextual_predsr’   r“   r”   r	   r•   )r8   r¬   ro   r_   r­   Úloaded_argsrm   ÚkrP   r¹   Ú
dict_predsr¾   r�   r¸   ÚbÚpsÚesr:   r•   s                      r3   rI   rI     s‡  € à˜MÑ*ÐØ�[Ñ!€IÜ% dÓ+€Jô  °D¸±NÈÔN€GØ Ÿ™ w§}¡}�€Kàò %ˆØ�:‰:�fÔ §¡¨GÔ!4¸¸]Ò8JØ! !™WˆK˜ŠNð%ô
 ‡K�KÐ4×;Ñ;¸DÀÑ<NÓOÔPÜ
�/‰/ T¨+Ñ%6Ô
7€CÜ�s˜D Ñ.°À5ÐUYÔZ€Eô ˆ5ƒz�Q‚Ü�‰Ð_×fÑfÐgkÐlwÑgxÓyÔzØà×%Ñ% e§i¡i§m¡m´T¼4°LÓ&AÓB€Jà‡��{ EÔ*ØŠä�‰Ð2Ô3ØˆØˆÜ˜eÓ$ò 	‰DˆAˆqØ—_‘_ Q¨¨[Ñ(9Ó:‰FˆB�Ø�R‰KˆEØ‰~Ø˜‘‘ð		ð
 ×#Ñ# E§I¡I§M¡M´4°&Ó$9¸5ÈÐ#ÓNˆà�?‰?˜?¨EÔ2Ü�K‰KÐFÔGØ×$Ñ$ U§Y¡Y§]¡]´D¼$°<Ó%@À%ÓHˆEà×-Ñ-Ô/Ø×3Ñ3°E·I±I¸uÓEˆEð 
‡I�I‡M�M”5�'˜5Ô!Ü	×Ñ˜%Ÿ)™)Ð%5Ô6ØÐÜ—l‘lÐ#3°YÓ?‰ˆˆ1ˆeä�‰ÐB×IÑIÈ$È{ÑJ[Ð]bÐcfÑ]fÓgÕhð r5   Ú__main__)N)-Ú__doc__ÚloggingÚsysr[   Úshutilrš   r   r)   Únumpyr¦   ÚrandomÚtorchr   r   Ústanza.models.lemma.datar   Ústanza.models.lemma.vocabr   Ústanza.models.lemma.trainerr   Ústanza.models.lemmar	   r
   Ústanza.models.commonr   Ú%stanza.models.common.seq2seq_constantÚmodelsÚcommonÚseq2seq_constantÚconstantÚstanza.models.common.docÚstanza.utils.conllr   Ústanza.modelsr   Ú	getLoggerrF   r4   r=   rJ   rS   ra   r   rI   Ú__name__© r5   r3   ú<module>rå      sš   ðñó Û 
Û 	Û Û Ý Û Û Û Û ß å /Ý +Ý /ß ,Ý &ß 8Ó 8Ü &Ý $Ý +à	ˆ×	Ñ	˜8Ó	$€ò7óró
òò	ò{Vòz4iðl ˆzÒÙ…Fð r5   