-
Notifications
You must be signed in to change notification settings - Fork 5
/
Copy pathrun-evaluation.sh
113 lines (100 loc) · 3.14 KB
/
run-evaluation.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
set -e
set -u
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CONDA_PREFIX/lib/
BASEDIR=$(realpath `dirname $0`)
WS=$BASEDIR
DATA=$WS/data
FORMAT=$DATA/format
RAW=$DATA/raw
OUTPUT=$WS/output
SCRIPT=$WS/scripts
MODEL=$WS/model
BATCH_SIZE=400
all_lang_pairs=(en-zh zh-en en-de de-en en-ja ja-en de-fr fr-de)
model_names=(text-davinci-003 alpaca-7b)
# comet
for model_name in ${model_names[@]}
do
cd $OUTPUT/$model_name
for lp in ${all_lang_pairs[@]}
do
src=${lp%%-*}
tgt=${lp##*-}
CAND1=wmt22.$src-$tgt.$tgt.0-shot.trans
CAND2=wmt22.$src-$tgt.$tgt.kw.0-seed.trans
CAND3=wmt22.$src-$tgt.$tgt.topic.0-seed.trans
CAND4=wmt22.$src-$tgt.$tgt.demo.0-seed.trans
CAND5=wmt22.$src-$tgt.$tgt.rerank.0-seed.trans
CAND6=wmt22.$src-$tgt.$tgt.rerank_bound.0-seed.trans
CAND7=wmt22.$src-$tgt.$tgt.maps.0-seed.trans
CAND8=wmt22.$src-$tgt.$tgt.maps_bound.0-seed.trans
CANDS=($CAND1 $CAND2 $CAND3 $CAND4 $CAND5 $CAND6 $CAND7 $CAND8)
src_file=$RAW/wmt22.$src-$tgt.$src
ref_file=$RAW/wmt22.$src-$tgt.$tgt
for CAND in ${CANDS[@]}
do
if [[ ! -s $CAND ]]
then
echo "ERROR: file not found $CAND"
exit 1
fi
done
echo "COMET: $lp $model_name"
python3 $SCRIPT/score.py \
--sys ${CANDS[@]} \
--src $src_file \
--ref $ref_file \
--comet-model-name Unbabel/wmt22-comet-da \
--metric comet \
--batch-size $BATCH_SIZE
python3 $SCRIPT/compare.py \
-t ${CANDS[@]} \
-s $src_file \
-r $ref_file \
--model Unbabel/wmt22-comet-da \
--batch_size $BATCH_SIZE \
--metric comet
done
done
# bleurt
for model_name in ${model_names[@]}
do
cd $OUTPUT/$model_name
for lp in ${all_lang_pairs[@]}
do
src=${lp%%-*}
tgt=${lp##*-}
CAND1=wmt22.$src-$tgt.$tgt.0-shot.trans
CAND2=wmt22.$src-$tgt.$tgt.kw.0-seed.trans
CAND3=wmt22.$src-$tgt.$tgt.topic.0-seed.trans
CAND4=wmt22.$src-$tgt.$tgt.demo.0-seed.trans
CAND5=wmt22.$src-$tgt.$tgt.rerank.0-seed.trans
CAND6=wmt22.$src-$tgt.$tgt.rerank_bound.0-seed.trans
CAND7=wmt22.$src-$tgt.$tgt.maps.0-seed.trans
CAND8=wmt22.$src-$tgt.$tgt.maps_bound.0-seed.trans
CANDS=($CAND1 $CAND2 $CAND3 $CAND4 $CAND5 $CAND6 $CAND7 $CAND8)
src_file=$RAW/wmt22.$src-$tgt.$src
ref_file=$RAW/wmt22.$src-$tgt.$tgt
for CAND in ${CANDS[@]}
do
if [[ ! -s $CAND ]]
then
echo "ERROR: file not found $CAND"
exit 1
fi
done
echo "BLEURT: $lp $model_name"
python3 $SCRIPT/score.py \
--sys ${CANDS[@]} \
--src $src_file \
--ref $ref_file \
--metric bleurt \
--batch-size $BATCH_SIZE
python3 $SCRIPT/compare.py \
-t ${CANDS[@]} \
-s $src_file \
-r $ref_file \
--batch_size $BATCH_SIZE \
--metric bleurt
done
done