-
Notifications
You must be signed in to change notification settings - Fork 3
/
create-phrase-table.sh
82 lines (63 loc) · 3.34 KB
/
create-phrase-table.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
corpus_name="$1"
source_lang=${2:-"de"}
target_lang=${3:-"en"}
out_dir="/datadrive/nlp/output"
mosesdecoder_path="/datadrive/mosesdecoder"
lm_path="/datadrive/nlp/lm/dummy.arpa"
clean_corpus_n_perl=$mosesdecoder_path/scripts/training/clean-corpus-n.perl
tokenize_perl=$mosesdecoder_path/scripts/tokenizer/tokenizer.perl
train_truecaser_perl=$mosesdecoder_path/scripts/recaser/train-truecaser.perl
truecase_perl=$mosesdecoder_path/scripts/recaser/truecase.perl
detruecase_perl=$mosesdecoder_path/scripts/recaser/detruecase.perl
train_model_perl=$mosesdecoder_path/scripts/training/train-model.perl
mkdir $out_dir
echo "Cleaning..."
$clean_corpus_n $corpus_name \
$source_lang $target_lang $out_dir/corpus.clean 1 50
echo "Tokenizing..."
$tokenizer -l $source_lang -threads 10 \
< $out_dir/corpus.clean.$source_lang \
> $out_dir/corpus.clean.tok.$source_lang
$tokenizer -l $target_lang -threads 10 \
< $out_dir/corpus.clean.$target_lang \
> $out_dir/corpus.clean.tok.$target_lang
# IF case sensitive model
echo "Train Truecaser..."
$train_truecaser --model $out_dir/truecase-model.$source_lang \
--corpus $out_dir/corpus.clean.tok.$source_lang
$train_truecaser --model $out_dir/truecase-model.$target_lang \
--corpus $out_dir/corpus.clean.tok.$target_lang
echo "Truecase..."
$truecase --model $out_dir/truecase-model.$source_lang \
< $out_dir/corpus.clean.tok.$source_lang \
> $out_dir/corpus.clean.tok.truecase.$source_lang
$truecase --model $out_dir/truecase-model.$target_lang \
< $out_dir/corpus.clean.tok.$target_lang \
> $out_dir/corpus.clean.tok.truecase.$target_lang
# ENDIF case sensitive model
# IF lowercase model
# echo "Train Recaser..."
# $mosesdecoder_path/scripts/recaser/train-recaser.perl --first-step 2 \
# --dir $out_dir/recaser-$source_lang/ \
# --corpus $out_dir/corpus.clean.tok.$source_lang
# $mosesdecoder_path/scripts/recaser/train-recaser.perl --first-step 2 \
# --dir $out_dir/recaser-$target_lang/ \
# --corpus $out_dir/corpus.clean.tok.$target_lang
# echo "Lowercase..."
# $mosesdecoder_path/scripts/tokenizer/lowercase.perl
# < $out_dir/corpus.clean.tok.$source_lang \
# > $out_dir/corpus.clean.tok.lower.$source_lang
# $mosesdecoder_path/scripts/tokenizer/lowercase.perl < $out_dir/corpus.clean.tok.$target_lang \
# > $out_dir/corpus.clean.tok.lower.$target_lang
# ENDIF lowercase model
echo "Train Model..."
$train_model -root-dir $out_dir/train/ \
-external-bin-dir $mosesdecoder_path/tools/bin/ --mgiza --parallel \
--corpus $out_dir/corpus.clean.tok \
--f $source_lang --e $target_lang \
-lm 0:3:$lm_path:8
echo "Binarize the Phrase Table..."
$mosesdecoderpath/bin/CreateOnDiskPt 1 1 4 100 2 $out_dir/train/model/phrase-table.gz phrase-table-bin-folder
echo "Create Compact Phrase Table..."
$mosesdecoderpath/bin/processPhraseTableMin -in phrase-table.gz -out phrase-table-compact -nscores 4 -threads 16
echo "Done."