Skip to content

Commit 35faa4d

Browse files
vimalmanohardanpovey
vimalmanohar
authored and
danpovey
committed
[src,egs,scripts] Cosmetic and other minor fixes, some required for segmentation PR (kaldi-asr#1784)
1 parent ff2ad50 commit 35faa4d

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

68 files changed

+1785
-766
lines changed

‎egs/aspire/s5/local/multi_condition/get_ctm.sh‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,7 @@ decode_mbr=true
77
filter_ctm_command=cp
88
glm=
99
stm=
10-
window=10
11-
overlap=5
10+
resolve_overlaps=true
1211
[ -f ./path.sh ] && . ./path.sh
1312
. parse_options.sh || exit 1;
1413

@@ -61,11 +60,16 @@ lattice-align-words-lexicon --output-error-lats=true --output-if-empty=true --ma
6160
$lang/phones/align_lexicon.int $model ark:- ark:- | \
6261
lattice-to-ctm-conf $frame_shift_opt --decode-mbr=$decode_mbr ark:- $decode_dir/score_$LMWT/penalty_$wip/ctm.overlapping || exit 1;
6362

63+
ctm=$decode_dir/score_$LMWT/penalty_$wip/ctm.overlapping
6464
# combine the segment-wise ctm files, while resolving overlaps
65-
python local/multi_condition/resolve_ctm_overlaps.py --overlap $overlap --window-length $window $data_dir/utt2spk $decode_dir/score_$LMWT/penalty_$wip/ctm.overlapping $decode_dir/score_$LMWT/penalty_$wip/ctm.merged || exit 1;
66-
merged_ctm=$decode_dir/score_$LMWT/penalty_$wip/ctm.merged
65+
if $resolve_overlaps; then
66+
utils/ctm/resolve_ctm_overlaps.py $data_dir/segments \
67+
$decode_dir/score_$LMWT/penalty_$wip/ctm.overlapping \
68+
$decode_dir/score_$LMWT/penalty_$wip/ctm.merged || exit 1;
69+
ctm=$decode_dir/score_$LMWT/penalty_$wip/ctm.merged
70+
fi
6771

68-
cat $merged_ctm | utils/int2sym.pl -f 5 $lang/words.txt | \
72+
cat $ctm | utils/int2sym.pl -f 5 $lang/words.txt | \
6973
utils/convert_ctm.pl $data_dir/segments $data_dir/reco2file_and_channel | \
7074
sort -k1,1 -k2,2 -k3,3nb > $decode_dir/score_$LMWT/penalty_$wip/ctm || exit 1;
7175
# Remove some stuff we don't want to score, from the ctm.

‎egs/aspire/s5/local/multi_condition/resolve_ctm_overlaps.py‎

Lines changed: 0 additions & 100 deletions
This file was deleted.

‎egs/aspire/s5/local/score_aspire.sh‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,9 @@ word_ins_penalties=0.0,0.25,0.5,0.75,1.0
1414
default_wip=0.0
1515
ctm_beam=6
1616
decode_mbr=true
17-
window=30
18-
overlap=5
1917
cmd=run.pl
2018
stage=1
19+
resolve_overlaps=true
2120
tune_hyper=true # if true:
2221
# if the data set is "dev_aspire" we check for the
2322
# best lmwt and word_insertion_penalty,
@@ -89,7 +88,7 @@ if $tune_hyper ; then
8988
# or use the default values
9089

9190
if [ $stage -le 1 ]; then
92-
if [ "$act_data_set" == "dev_aspire" ]; then
91+
if [[ "$act_data_set" =~ "dev_aspire" ]]; then
9392
wip_string=$(echo $word_ins_penalties | sed 's/,/ /g')
9493
temp_wips=($wip_string)
9594
$cmd WIP=1:${#temp_wips[@]} $decode_dir/scoring/log/score.wip.WIP.log \
@@ -98,8 +97,8 @@ if $tune_hyper ; then
9897
echo \$wip \&\& \
9998
$cmd LMWT=$min_lmwt:$max_lmwt $decode_dir/scoring/log/score.LMWT.\$wip.log \
10099
local/multi_condition/get_ctm.sh --filter-ctm-command "$filter_ctm_command" \
101-
--window $window --overlap $overlap \
102100
--beam $ctm_beam --decode-mbr $decode_mbr \
101+
--resolve-overlaps $resolve_overlaps \
103102
--glm data/${act_data_set}/glm --stm data/${act_data_set}/stm \
104103
LMWT \$wip $lang data/${segmented_data_set}_hires $model $decode_dir || exit 1;
105104

@@ -124,7 +123,7 @@ wipfile.close()
124123
fi
125124

126125

127-
if [ "$act_data_set" == "test_aspire" ] || [ "$act_data_set" == "eval_aspire" ]; then
126+
if [[ "$act_data_set" =~ "test_aspire" ]] || [[ "$act_data_set" =~ "eval_aspire" ]]; then
128127
# check for the best values from dev_aspire decodes
129128
dev_decode_dir=$(echo $decode_dir|sed "s/test_aspire/dev_aspire_whole/g; s/eval_aspire/dev_aspire_whole/g")
130129
if [ -f $dev_decode_dir/scoring/bestLMWT ]; then

‎egs/aspire/s5/path.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,5 @@ export KALDI_ROOT=`pwd`/../../..
22
export PATH=$PWD/utils/:$KALDI_ROOT/tools/openfst/bin:$PWD:$PATH
33
[ ! -f $KALDI_ROOT/tools/config/common_path.sh ] && echo >&2 "The standard file $KALDI_ROOT/tools/config/common_path.sh is not present -> Exit!" && exit 1
44
. $KALDI_ROOT/tools/config/common_path.sh
5+
export PATH=$KALDI_ROOT/tools/sctk/bin:$PATH
56
export LC_ALL=C

‎egs/babel/s5d/cmd.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@
77

88
#a) JHU cluster options
99
export train_cmd="queue.pl -l arch=*64"
10-
export decode_cmd="queue.pl -l arch=*64,mem_free=2G,ram_free=2G"
11-
export mkgraph_cmd="queue.pl -l arch=*64,ram_free=4G,mem_free=4G"
10+
export decode_cmd="queue.pl --mem 4G"
11+
export mkgraph_cmd="queue.pl --mem 4G"
1212

1313
#export cuda_cmd="..."
1414

‎egs/babel/s5d/conf/common.limitedLP‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ use_pitch=true
3838
lmwt_plp_extra_opts=( --min-lmwt 8 --max-lmwt 12 )
3939
lmwt_bnf_extra_opts=( --min-lmwt 15 --max-lmwt 22 )
4040
lmwt_dnn_extra_opts=( --min-lmwt 10 --max-lmwt 15 )
41-
41+
lmwt_chain_extra_opts=( --min-lmwt 8 --max-lmwt 12 )
4242
dnn_beam=16.0
4343
dnn_lat_beam=8.5
4444

‎egs/babel/s5d/local/lattice_to_ctm.sh‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,13 @@ word_ins_penalty=0.5
1010
min_lmwt=7
1111
max_lmwt=17
1212
model=
13+
resolve_overlaps=false # If true, the words decoded in the regions where
14+
# two segments A and B are overlapping are resolved
15+
# such that only words before the mid-point of the
16+
# overlapping region are taken to be hypothesized for
17+
# segment A, and only words after the mid-point of
18+
# the overlapping region are taken to be
19+
# hypothesized for segment B.
1320

1421
#end configuration section.
1522

@@ -43,10 +50,22 @@ for f in $lang/words.txt $model $data/segments $data/reco2file_and_channel $dir/
4350
[ ! -f $f ] && echo "$0: expecting file $f to exist" && exit 1;
4451
done
4552

53+
if [ -f $dir/../frame_subsampling_factor ]; then
54+
factor=$(cat $dir/../frame_subsampling_factor) || exit 1
55+
frame_shift_opt="--frame-shift=0.0$factor"
56+
echo "$0: $dir/../frame_subsampling_factor exists, using $frame_shift_opt"
57+
fi
58+
4659
name=`basename $data`; # e.g. eval2000
4760

4861
mkdir -p $dir/scoring/log
4962

63+
resolve_overlaps_cmd="cat"
64+
65+
if $resolve_overlaps; then
66+
resolve_overlaps_cmd="utils/ctm/resolve_ctm_overlaps.py $data/segments - -"
67+
fi
68+
5069
if [ $stage -le 0 ]; then
5170
if [ ! -f $lang/phones/word_boundary.int ] ; then
5271
$cmd LMWT=$min_lmwt:$max_lmwt $dir/scoring/log/get_ctm.LMWT.log \
@@ -56,8 +75,9 @@ if [ $stage -le 0 ]; then
5675
lattice-add-penalty --word-ins-penalty=$word_ins_penalty ark:- ark:- \| \
5776
lattice-prune --beam=$beam ark:- ark:- \| \
5877
lattice-align-words-lexicon $lang/phones/align_lexicon.int $model ark:- ark:- \| \
59-
lattice-to-ctm-conf --decode-mbr=$decode_mbr ark:- - \| \
78+
lattice-to-ctm-conf $frame_shift_opt --decode-mbr=$decode_mbr ark:- - \| \
6079
utils/int2sym.pl -f 5 $lang/words.txt \| tee $dir/score_LMWT/$name.utt.ctm \| \
80+
$resolve_overlaps_cmd \| \
6181
utils/convert_ctm.pl $data/segments $data/reco2file_and_channel \
6282
'>' $dir/score_LMWT/$name.ctm || exit 1;
6383
else
@@ -68,8 +88,9 @@ if [ $stage -le 0 ]; then
6888
lattice-add-penalty --word-ins-penalty=$word_ins_penalty ark:- ark:- \| \
6989
lattice-prune --beam=$beam ark:- ark:- \| \
7090
lattice-align-words $lang/phones/word_boundary.int $model ark:- ark:- \| \
71-
lattice-to-ctm-conf --decode-mbr=$decode_mbr ark:- - \| \
91+
lattice-to-ctm-conf $frame_shift_opt --decode-mbr=$decode_mbr ark:- - \| \
7292
utils/int2sym.pl -f 5 $lang/words.txt \| tee $dir/score_LMWT/$name.utt.ctm \| \
93+
$resolve_overlaps_cmd \| \
7394
utils/convert_ctm.pl $data/segments $data/reco2file_and_channel \
7495
'>' $dir/score_LMWT/$name.ctm || exit 1;
7596
fi

‎egs/babel/s5d/local/run_kws_stt_task2.sh‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,10 @@ extra_kws=false
3232
cmd=run.pl
3333
max_states=150000
3434
wip=0.5 #Word insertion penalty
35+
resolve_overlaps=false # Set this to true, if there are overlapping segments
36+
# as input and the words in the CTM in the
37+
# overlapping regions must be resolved to one
38+
# of the segments.
3539
#End of options
3640

3741
if [ $(basename $0) == score.sh ]; then
@@ -56,7 +60,7 @@ decode_dir=$3;
5660
if ! $skip_stt ; then
5761
if [ ! -f $decode_dir/.score.done ] && [ ! -f $decode_dir/.done.score ]; then
5862
local/lattice_to_ctm.sh --cmd "$cmd" --word-ins-penalty $wip \
59-
--min-lmwt ${min_lmwt} --max-lmwt ${max_lmwt} \
63+
--min-lmwt ${min_lmwt} --max-lmwt ${max_lmwt} --resolve-overlaps $resolve_overlaps \
6064
$data_dir $lang_dir $decode_dir
6165

6266
if ! $skip_scoring ; then

‎egs/babel/s5d/path.sh‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
1-
export KALDI_ROOT=/export/a09/jtrmal/kaldi/
1+
export KALDI_ROOT=../../..
22
. $KALDI_ROOT/tools/env.sh
3-
. /export/a09/jtrmal/kaldi-current/tools/env.sh
4-
. /export/babel/data/software/env.sh
3+
. /export/babel/data/software/env.sh
54
export PATH=$PWD/utils/:$KALDI_ROOT/tools/sph2pipe_v2.5/:$KALDI_ROOT/src/bin:$KALDI_ROOT/tools/openfst/bin:$KALDI_ROOT/src/fstbin/:$KALDI_ROOT/src/gmmbin/:$KALDI_ROOT/src/featbin/:$KALDI_ROOT/src/lm/:$KALDI_ROOT/src/sgmmbin/:$KALDI_ROOT/src/sgmm2bin/:$KALDI_ROOT/src/fgmmbin/:$KALDI_ROOT/src/latbin/:$KALDI_ROOT/src/nnet3bin:$KALDI_ROOT/src/nnetbin:$KALDI_ROOT/src/nnet2bin/:$KALDI_ROOT/src/online2bin:$KALDI_ROOT/src/ivectorbin:$KALDI_ROOT/src/kwsbin:$PWD:$PATH
65
export LC_ALL=C
76

‎egs/babel/s5d/run-4-anydecode.sh‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -232,13 +232,17 @@ if [ ! -f $dataset_dir/.done ] ; then
232232
elif [ "$dataset_segments" == "train" ] ||\
233233
[ "$dataset_segments" == "pem" ]; then
234234
. ./local/datasets/supervised_pem.sh || exit 1
235+
elif [[ $dataset_segments =~ seg* ]]; then
236+
echo "Using ${dataset_dir} directly"
235237
else
236238
echo "Unknown type of the dataset: \"$dataset_segments\"!";
237239
echo "Valid dataset types are: seg, uem, pem";
238240
exit 1
239241
fi
240242
elif [ "$dataset_kind" == "unsupervised" ] ; then
241-
if [ "$dataset_segments" == "seg" ] ; then
243+
if [ "$dataset_segments" == "seg" ]; then
244+
. ./local/datasets/unsupervised_seg.sh
245+
elif [[ $dataset_segments =~ *seg* ]]; then
242246
. ./local/datasets/unsupervised_seg.sh
243247
elif [ "$dataset_segments" == "uem" ] ; then
244248
. ./local/datasets/unsupervised_uem.sh
@@ -251,6 +255,8 @@ if [ ! -f $dataset_dir/.done ] ; then
251255
echo "does not really make any sense!"
252256
exit 1
253257
#. ./local/datasets/unsupervised_pem.sh
258+
elif [[ $dataset_segments =~ seg* ]]; then
259+
echo "Using ${dataset_dir} directly"
254260
else
255261
echo "Unknown type of the dataset: \"$dataset_segments\"!";
256262
echo "Valid dataset types are: seg, uem, pem";

0 commit comments

Comments
 (0)