[src] Add --write-per-frame-acoustic-loglikes options to alignment programs (kaldi-asr#1601)

naxingyu · danpovey · commit ad6491ab596f · 2017-05-05T15:53:31.000-04:00
diff --git a/src/decoder/decoder-wrappers.cc b/src/decoder/decoder-wrappers.cc
@@ -19,6 +19,7 @@
 
 #include "decoder/decoder-wrappers.h"
 #include "decoder/faster-decoder.h"
+#include "lat/lattice-functions.h"
 
 namespace kaldi {
 
@@ -419,13 +420,13 @@ void ModifyGraphForCarefulAlignment(
   fst::Concat(fst, fst_rhs);
 }
 
-    
+
 void AlignUtteranceWrapper(
     const AlignConfig &config,
     const std::string &utt,
     BaseFloat acoustic_scale,  // affects scores written to scores_writer, if
                                // present
-    fst::VectorFst<fst::StdArc> *fst,  // non-const in case config.careful == 
+    fst::VectorFst<fst::StdArc> *fst,  // non-const in case config.careful ==
                                        // true.
     DecodableInterface *decodable,  // not const but is really an input.
     Int32VectorWriter *alignment_writer,
@@ -434,7 +435,8 @@ void AlignUtteranceWrapper(
     int32 *num_error,
     int32 *num_retried,
     double *tot_like,
-    int64 *frame_count) {
+    int64 *frame_count,
+    BaseFloatVectorWriter *per_frame_acwt_writer) {
 
   if ((config.retry_beam != 0 && config.retry_beam <= config.beam) ||
       config.beam <= 0.0) {
@@ -460,7 +462,7 @@ void AlignUtteranceWrapper(
   decoder.Decode(decodable);
 
   bool ans = decoder.ReachedFinal();  // consider only final states.
-  
+
   if (!ans && config.retry_beam != 0.0) {
     if (num_retried != NULL) (*num_retried)++;
     KALDI_WARN << "Retrying utterance " << utt << " with beam "
@@ -477,15 +479,15 @@ void AlignUtteranceWrapper(
     if (num_error != NULL) (*num_error)++;
     return;
   }
-  
+
   fst::VectorFst<LatticeArc> decoded;  // linear FST.
   decoder.GetBestPath(&decoded);
   if (decoded.NumStates() == 0) {
     KALDI_WARN << "Error getting best path from decoder (likely a bug)";
     if (num_error != NULL) (*num_error)++;
     return;
   }
-    
+
   std::vector<int32> alignment;
   std::vector<int32> words;
   LatticeWeight weight;
@@ -499,10 +501,16 @@ void AlignUtteranceWrapper(
 
   if (alignment_writer != NULL && alignment_writer->IsOpen())
     alignment_writer->Write(utt, alignment);
-  
+
   if (scores_writer != NULL && scores_writer->IsOpen())
     scores_writer->Write(utt, -(weight.Value1()+weight.Value2()));
-}
 
+  Vector<BaseFloat> per_frame_loglikes;
+  if (per_frame_acwt_writer != NULL && per_frame_acwt_writer->IsOpen()) {
+    GetPerFrameAcousticCosts(decoded, &per_frame_loglikes);
+    per_frame_loglikes.Scale(-1 / acoustic_scale);
+    per_frame_acwt_writer->Write(utt, per_frame_loglikes);
+  }
+}
 
 } // end namespace kaldi.
diff --git a/src/decoder/decoder-wrappers.h b/src/decoder/decoder-wrappers.h
@@ -69,7 +69,8 @@ void AlignUtteranceWrapper(
     int32 *num_error,
     int32 *num_retried,
     double *tot_like,
-    int64 *frame_count);
+    int64 *frame_count,
+    BaseFloatVectorWriter *per_frame_acwt_writer = NULL);
 
 
 
diff --git a/src/gmmbin/gmm-align-compiled.cc b/src/gmmbin/gmm-align-compiled.cc
@@ -39,8 +39,8 @@ int main(int argc, char *argv[]) {
 
     const char *usage =
         "Align features given [GMM-based] models.\n"
-        "Usage:   gmm-align-compiled [options] model-in graphs-rspecifier "
-        "feature-rspecifier alignments-wspecifier [scores-wspecifier]\n"
+        "Usage:   gmm-align-compiled [options] <model-in> <graphs-rspecifier> "
+        "<feature-rspecifier> <alignments-wspecifier> [scores-wspecifier]\n"
         "e.g.: \n"
         " gmm-align-compiled 1.mdl ark:graphs.fsts scp:train.scp ark:1.ali\n"
         "or:\n"
@@ -52,6 +52,7 @@ int main(int argc, char *argv[]) {
     BaseFloat acoustic_scale = 1.0;
     BaseFloat transition_scale = 1.0;
     BaseFloat self_loop_scale = 1.0;
+    std::string per_frame_acwt_wspecifier;
 
     align_config.Register(&po);
     po.Register("transition-scale", &transition_scale,
@@ -60,13 +61,16 @@ int main(int argc, char *argv[]) {
                 "Scaling factor for acoustic likelihoods");
     po.Register("self-loop-scale", &self_loop_scale,
                 "Scale of self-loop versus non-self-loop log probs [relative to acoustics]");
+    po.Register("write-per-frame-acoustic-loglikes", &per_frame_acwt_wspecifier,
+                "Wspecifier for table of vectors containing the acoustic log-likelihoods "
+                "per frame for each utterance. E.g. ark:foo/per_frame_logprobs.1.ark");
     po.Read(argc, argv);
 
     if (po.NumArgs() < 4 || po.NumArgs() > 5) {
       po.PrintUsage();
       exit(1);
     }
-    
+
     std::string model_in_filename = po.GetArg(1),
         fst_rspecifier = po.GetArg(2),
         feature_rspecifier = po.GetArg(3),
@@ -86,6 +90,7 @@ int main(int argc, char *argv[]) {
     RandomAccessBaseFloatMatrixReader feature_reader(feature_rspecifier);
     Int32VectorWriter alignment_writer(alignment_wspecifier);
     BaseFloatWriter scores_writer(scores_wspecifier);
+    BaseFloatVectorWriter per_frame_acwt_writer(per_frame_acwt_wspecifier);
 
     int num_done = 0, num_err = 0, num_retry = 0;
     double tot_like = 0.0;
@@ -118,12 +123,13 @@ int main(int argc, char *argv[]) {
 
         DecodableAmDiagGmmScaled gmm_decodable(am_gmm, trans_model, features,
                                                acoustic_scale);
-         
+
+        KALDI_LOG << utt;
         AlignUtteranceWrapper(align_config, utt,
                               acoustic_scale, &decode_fst, &gmm_decodable,
                               &alignment_writer, &scores_writer,
                               &num_done, &num_err, &num_retry,
-                              &tot_like, &frame_count);
+                              &tot_like, &frame_count, &per_frame_acwt_writer);
       }
     }
     KALDI_LOG << "Overall log-likelihood per frame is " << (tot_like/frame_count)
@@ -137,5 +143,3 @@ int main(int argc, char *argv[]) {
     return -1;
   }
 }
-
-
diff --git a/src/lat/lattice-functions.cc b/src/lat/lattice-functions.cc
@@ -33,6 +33,48 @@ namespace kaldi {
 using std::map;
 using std::vector;
 
+void GetPerFrameAcousticCosts(const Lattice &nbest, Vector<BaseFloat> *per_frame_loglikes) {
+  using namespace fst;
+  typedef Lattice::Arc::Weight Weight;
+  vector<BaseFloat> loglikes;
+
+  int32 cur_state = nbest.Start();
+  int32 prev_frame = -1;
+  BaseFloat eps_acwt = 0.0;
+  while(1) {
+    Weight w = nbest.Final(cur_state);
+    if (w != Weight::Zero()) {
+      KALDI_ASSERT(nbest.NumArcs(cur_state) == 0);
+      if (per_frame_loglikes != NULL)  {
+        SubVector<BaseFloat> subvec(&(loglikes[0]), loglikes.size());
+        Vector<BaseFloat> vec(subvec);
+        *per_frame_loglikes = vec;
+      }
+      break;
+    } else {
+      KALDI_ASSERT(nbest.NumArcs(cur_state) == 1);
+      fst::ArcIterator<Lattice> iter(nbest, cur_state);
+      const Lattice::Arc &arc = iter.Value();
+      BaseFloat acwt = arc.weight.Value2();
+      if (arc.ilabel != 0) {
+        if (eps_acwt > 0) {
+          acwt += eps_acwt;
+          eps_acwt = 0.0;
+        }
+        loglikes.push_back(acwt);
+        prev_frame++;
+      } else if (acwt == acwt){
+        if (prev_frame > -1) {
+          loglikes[prev_frame] += acwt;
+        } else {
+          eps_acwt += acwt;
+        }
+      }
+      cur_state = arc.nextstate;
+    }
+  }
+}
+
 int32 LatticeStateTimes(const Lattice &lat, vector<int32> *times) {
   if (!lat.Properties(fst::kTopSorted, true))
     KALDI_ERR << "Input lattice must be topologically sorted.";
diff --git a/src/lat/lattice-functions.h b/src/lat/lattice-functions.h
@@ -36,6 +36,24 @@
 
 namespace kaldi {
 
+/**
+   This function extracts the per-frame log likelihoods from a linear
+   lattice (which we refer to as an 'nbest' lattice elsewhere in Kaldi code).
+   The dimension of *per_frame_loglikes will be set to the
+   number of input symbols in 'nbest'.  The elements of
+   '*per_frame_loglikes' will be set to the .Value2() elements of the lattice
+   weights, which represent the acoustic costs; you may want to scale this
+   vector afterward by -1/acoustic_scale to get the original loglikes.
+   If there are acoustic costs on input-epsilon arcs or the final-prob in 'nbest'
+   (and this should not normally be the case in situations where it makes
+   sense to call this function), they will be included to the cost of the
+   preceding input symbol, or the following input symbol for input-epsilons
+   encountered prior to any input symbol.  If 'nbest' has no input symbols,
+   'per_frame_loglikes' will be set to the empty vector.
+**/
+void GetPerFrameAcousticCosts(const Lattice &nbest,
+                              Vector<BaseFloat> *per_frame_loglikes);
+
 /// This function iterates over the states of a topologically sorted lattice and
 /// counts the time instance corresponding to each state. The times are returned
 /// in a vector of integers 'times' which is resized to have a size equal to the
diff --git a/src/nnet2bin/nnet-align-compiled.cc b/src/nnet2bin/nnet-align-compiled.cc
@@ -40,8 +40,8 @@ int main(int argc, char *argv[]) {
 
     const char *usage =
         "Align features given neural-net-based model\n"
-        "Usage:   nnet-align-compiled [options] model-in graphs-rspecifier "
-        "feature-rspecifier alignments-wspecifier\n"
+        "Usage:   nnet-align-compiled [options] <model-in> <graphs-rspecifier> "
+        "<feature-rspecifier> <alignments-wspecifier>\n"
         "e.g.: \n"
         " nnet-align-compiled 1.mdl ark:graphs.fsts scp:train.scp ark:1.ali\n"
         "or:\n"
@@ -54,6 +54,7 @@ int main(int argc, char *argv[]) {
     BaseFloat acoustic_scale = 1.0;
     BaseFloat transition_scale = 1.0;
     BaseFloat self_loop_scale = 1.0;
+    std::string per_frame_acwt_wspecifier;
 
     align_config.Register(&po);
     po.Register("transition-scale", &transition_scale,
@@ -63,6 +64,9 @@ int main(int argc, char *argv[]) {
     po.Register("self-loop-scale", &self_loop_scale,
                 "Scale of self-loop versus non-self-loop "
                 "log probs [relative to acoustics]");
+    po.Register("write-per-frame-acoustic-loglikes", &per_frame_acwt_wspecifier,
+                "Wspecifier for table of vectors containing the acoustic log-likelihoods "
+                "per frame for each utterance. E.g. ark:foo/per_frame_logprobs.1.ark");
     po.Register("use-gpu", &use_gpu,
                 "yes|no|optional|wait, only has effect if compiled with CUDA");
     po.Read(argc, argv);
@@ -75,7 +79,7 @@ int main(int argc, char *argv[]) {
 #if HAVE_CUDA==1
     CuDevice::Instantiate().SelectGpuId(use_gpu);
 #endif
-    
+
     std::string model_in_filename = po.GetArg(1),
         fst_rspecifier = po.GetArg(2),
         feature_rspecifier = po.GetArg(3),
@@ -100,6 +104,7 @@ int main(int argc, char *argv[]) {
       RandomAccessBaseFloatCuMatrixReader feature_reader(feature_rspecifier);
       Int32VectorWriter alignment_writer(alignment_wspecifier);
       BaseFloatWriter scores_writer(scores_wspecifier);
+      BaseFloatVectorWriter per_frame_acwt_writer(per_frame_acwt_wspecifier);
 
       for (; !fst_reader.Done(); fst_reader.Next()) {
         std::string utt = fst_reader.Key();
@@ -135,7 +140,7 @@ int main(int argc, char *argv[]) {
                               acoustic_scale, &decode_fst, &nnet_decodable,
                               &alignment_writer, &scores_writer,
                               &num_done, &num_err, &num_retry,
-                              &tot_like, &frame_count);
+                              &tot_like, &frame_count, &per_frame_acwt_writer);
       }
       KALDI_LOG << "Overall log-likelihood per frame is " << (tot_like/frame_count)
                 << " over " << frame_count<< " frames.";
@@ -152,5 +157,3 @@ int main(int argc, char *argv[]) {
     return -1;
   }
 }
-
-
diff --git a/src/nnet3bin/nnet3-align-compiled.cc b/src/nnet3bin/nnet3-align-compiled.cc
@@ -56,6 +56,7 @@ int main(int argc, char *argv[]) {
     std::string use_gpu = "yes";
     BaseFloat transition_scale = 1.0;
     BaseFloat self_loop_scale = 1.0;
+    std::string per_frame_acwt_wspecifier;
 
     std::string ivector_rspecifier,
         online_ivector_rspecifier,
@@ -71,6 +72,9 @@ int main(int argc, char *argv[]) {
     po.Register("self-loop-scale", &self_loop_scale,
                 "Scale of self-loop versus non-self-loop "
                 "log probs [relative to acoustics]");
+    po.Register("write-per-frame-acoustic-loglikes", &per_frame_acwt_wspecifier,
+                "Wspecifier for table of vectors containing the acoustic log-likelihoods "
+                "per frame for each utterance. E.g. ark:foo/per_frame_logprobs.1.ark");
     po.Register("ivectors", &ivector_rspecifier, "Rspecifier for "
                 "iVectors as vectors (i.e. not estimated online); per utterance "
                 "by default, or per speaker if you provide the --utt2spk option.");
@@ -126,7 +130,7 @@ int main(int argc, char *argv[]) {
       RandomAccessBaseFloatMatrixReader feature_reader(feature_rspecifier);
       Int32VectorWriter alignment_writer(alignment_wspecifier);
       BaseFloatWriter scores_writer(scores_wspecifier);
-
+      BaseFloatVectorWriter per_frame_acwt_writer(per_frame_acwt_wspecifier);
 
       for (; !fst_reader.Done(); fst_reader.Next()) {
         std::string utt = fst_reader.Key();
@@ -185,7 +189,7 @@ int main(int argc, char *argv[]) {
                               &decode_fst, &nnet_decodable,
                               &alignment_writer, &scores_writer,
                               &num_done, &num_err, &num_retry,
-                              &tot_like, &frame_count);
+                              &tot_like, &frame_count, &per_frame_acwt_writer);
       }
       KALDI_LOG << "Overall log-likelihood per frame is "
                 << (tot_like/frame_count)
diff --git a/src/sgmm2bin/sgmm2-align-compiled.cc b/src/sgmm2bin/sgmm2-align-compiled.cc
