opus1m+bt-2021-04-10.yml

release: eng-mkh/opus1m+bt-2021-04-10.zip
release-date: 2021-04-10
dataset-name: opus1m+bt
modeltype: transformer-align
vocabulary:
   source: opus1m+bt.spm32k-spm32k.vocab.yml
   target: opus1m+bt.spm32k-spm32k.vocab.yml
pre-processing: normalization + SentencePiece (spm32k,spm32k)
subwords:
   source: spm32k
   target: spm32k
subword-models:
   source: source.spm
   target: target.spm
source-languages:
   - eng
target-languages:
   - kha
   - khm
   - mnw
   - ngt
   - vie
use-target-labels:
   - ">>kha<<"
   - ">>khm<<"
   - ">>khm_Latn<<"
   - ">>mnw<<"
   - ">>ngt_Latn<<"
   - ">>vie<<"
   - ">>vie_Hani<<"
training-data:
   eng-khm: Tatoeba-train (289332) 
   eng-khm_Latn: Tatoeba-train (10780) 
   eng-mnw: wiki.aa.mnw-eng (994) 
   eng-vie: Tatoeba-train (1000000) 
validation-data:
   eng-khm: Tatoeba-dev, 965
   eng-khm_Latn: Tatoeba-dev, 35
   eng-vie: Tatoeba-dev, 2824
   total-size-shuffled: 1997
   devset-selected: top 1997  lines of Tatoeba-dev.src.shuffled!
test-data:
   Tatoeba-test.eng-kha: 1314/9269
   Tatoeba-test.eng-khm_Latn: 11/91
   Tatoeba-test.eng-khm: 752/1737
   Tatoeba-test.eng-mnw: 9/44
   Tatoeba-test.eng-ngt: 17/101
   Tatoeba-test.eng-vie_Hani: 1/1
   Tatoeba-test.eng-vie: 2500/24426
   Tatoeba-test.eng-multi: 4592/35578
   tico19-test.eng-khm: 2100/20941
BLEU-scores:
   Tatoeba-test.eng-kha: 0.6
   Tatoeba-test.eng-khm_Latn: 0.8
   Tatoeba-test.eng-khm: 0.0
   Tatoeba-test.eng-mnw: 0.6
   Tatoeba-test.eng-ngt: 0.2
   Tatoeba-test.eng-vie_Hani: 2.1
   Tatoeba-test.eng-vie: 34.0
   Tatoeba-test.eng-multi: 21.5
   tico19-test.eng-khm: 0.6
chr-F-scores:
   Tatoeba-test.eng-kha: 0.088
   Tatoeba-test.eng-khm_Latn: 0.065
   Tatoeba-test.eng-khm: 0.013
   Tatoeba-test.eng-mnw: 0.001
   Tatoeba-test.eng-ngt: 0.033
   Tatoeba-test.eng-vie_Hani: 0.000
   Tatoeba-test.eng-vie: 0.514
   Tatoeba-test.eng-multi: 0.339
   tico19-test.eng-khm: 0.029