-
Notifications
You must be signed in to change notification settings - Fork 2
/
Copy pathevaluate.sh
executable file
·115 lines (94 loc) · 5.06 KB
/
evaluate.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
#!/bin/bash
# this is the script for final evaluation
# the architecture was chosen in architecture exploration
# the hyperparam are chosen in hyperparameter tuning
set -e
# TODO should take best configuration from tuning, but we know it's conf_4 (huric:conf_1, fn_ft:conf_7)
best_config=conf_4
best_config_h=conf_4
#best_config_h=`python select_best.py nlunetwork/results/tuning/aggregated_huric.json`
#best_config_fn=`python select_best.py nlunetwork/results/tuning/aggregated_fn.json`
# 1. cross validate on HuRIC with the tuned parameters from HuRIC
# select the parameters from the best configuration (don't trust the conf file name, maybe some custom parameter changed)
### set -a
### source nlunetwork/results/tuning/${best_config}/huric/modern_right/config.env
### set +a
export DATASET=huric/modern_right
export OUTPUT_FOLDER=tuning/optimized/h_only
export MODE=cross
CONFIG_FILE=configurations/conf_4.env python nlunetwork/main.py
python nlunetwork/results_aggregator.py nlunetwork/results/tuning/optimized/h_only ${DATASET}
# 2. train FrameNet and test on HuRIC (transfer learning?)
# 2a. FrameNet fullText (fewer samples)
# 2b. FrameNet lu (lots of samples)
# 2.1. with the hyperparams from FrameNet
# 2.2. with the hyperparams from HuRIC
## # train on full FrameNet with the hyperparam of FrameNet
## set -a
## source nlunetwork/results/tuning/${best_config_fn}/framenet/subset_both/config.env
## set +a
## export DATASET=framenet/subset_both
## export OUTPUT_FOLDER=tuning/optimized/fn_only_hyper_fn
## export MODE=train_all
## python nlunetwork/main.py
##
##
## # framenet from lu
## MAX_EPOCHS=10 DATASET=framenet/modern_lu_subset_right OUTPUT_FOLDER=tuning/experiment_lu MODE=train_all CONFIG_FILE=configurations/conf_4.env python nlunetwork/main.py
## DATASET=huric/modern_right OUTPUT_FOLDER=tuning/experiment_lu MODE=test_all MODEL_PATH=nlunetwork/results/tuning/experiment_lu/conf_4/framenet/modern_lu_subset_right/ python nlunetwork/main.py
##
## MAX_EPOCHS=10 DATASET=framenet/modern_lu_subset_right OUTPUT_FOLDER=tuning/experiment_lu_cross MODE=cross CONFIG_FILE=configurations/conf_4.env python nlunetwork/main.py
# train all FrameNet subset on the hyperparam of HuRIC
### set -a
### source nlunetwork/results/tuning/${best_config}/huric/modern_right/config.env
### set +a
export DATASET=framenet/subset_ft_both
export OUTPUT_FOLDER=tuning/optimized/fn_only_hyper_h
export MODE=train_all
CONFIG_FILE=configurations/conf_4.env python nlunetwork/main.py
MAX_EPOCHS=10 OUTPUT_FOLDER=tuning/optimized/fn_lu_only_hyper_h DATASET=framenet/modern_lu_subset_right CONFIG_FILE=configurations/conf_4.env python nlunetwork/main.py
## # now test all with fn hyperparam
## export MODE=test_all
## export OUTPUT_FOLDER=tuning/optimized/fn_only_hyper_fn
## export MODEL_PATH=nlunetwork/results/tuning/optimized/fn_only_hyper_fn/${best_config_fn}/framenet/subset_both/
## export DATASET=huric/modern_right
## python nlunetwork/main.py
# now test all with h hyperparams
export MODE=test_all
export OUTPUT_FOLDER=tuning/optimized/fn_only_hyper_h
export MODEL_PATH=nlunetwork/results/tuning/optimized/fn_only_hyper_h/${best_config_h}/framenet/subset_ft_both/
export DATASET=huric/modern_right
python nlunetwork/main.py
# this line is not necessary, just one epoch
#python nlunetwork/results_aggregator.py nlunetwork/results/tuning/optimized/fn_only_hyper_h huric/modern_right
export MODE=test_all
export OUTPUT_FOLDER=tuning/optimized/fn_lu_only_hyper_h
export MODEL_PATH=nlunetwork/results/tuning/optimized/fn_lu_only_hyper_h/${best_config_h}/framenet/modern_lu_subset_right/
export DATASET=huric/modern_right
python nlunetwork/main.py
# also this line is not necessary, just one epoch
#python nlunetwork/results_aggregator.py nlunetwork/results/tuning/optimized/fn_lu_only_hyper_h huric/modern_right
# 3. dataset combination
## # simpler: conf_4
## CONFIG_FILE=configurations/conf_4.env MAX_EPOCHS=10 DATASET=huric/with_framenet OUTPUT_FOLDER=tuning/both_experiment MODE=cross python nlunetwork/main.py
## CONFIG_FILE=configurations/conf_4.env MAX_EPOCHS=10 DATASET=huric/with_framenet_lu OUTPUT_FOLDER=tuning/both_experiment MODE=cross python nlunetwork/main.py
##
## # cross-validation with the hyperparam from FrameNet
## set -a
## source nlunetwork/results/tuning/${best_config_fn}/${DATASET}/config.env
## set +a
## export DATASET=huric/with_framenet
## export OUTPUT_FOLDER=tuning/optimized/
## export MODE=cross
## python nlunetwork/main.py
# cross-validation with the hyperparams of HuRIC
### set -a
### source nlunetwork/results/tuning/${best_config}/huric/modern_right/config.env
### set +a
export DATASET=huric/with_framenet_ft
export OUTPUT_FOLDER=tuning/optimized/h_and_fn_ft
export MODE=cross
CONFIG_FILE=configurations/conf_4.env python nlunetwork/main.py
python nlunetwork/results_aggregator.py nlunetwork/results/tuning/optimized/h_and_fn_ft huric/with_framenet_ft
MAX_EPOCHS=50 DATASET=huric/with_framenet_lu OUTPUT_FOLDER=tuning/optimized/h_and_fn_lu CONFIG_FILE=configurations/conf_4.env MODE=cross python nlunetwork/main.py
python nlunetwork/results_aggregator.py nlunetwork/results/tuning/optimized/h_and_fn_lu huric/with_framenet_lu