Skip to content

Commit f10bc7e

Browse files
chenguoguochenguoguo
committed
Moved ReadIntegerArray() and WriteIntegerArray() to const-arpa-lm.cc; changed ReadInternalDeprecated() to ReadInternalOldFormat()
1 parent 7166563 commit f10bc7e

File tree

4 files changed

+108
-118
lines changed

4 files changed

+108
-118
lines changed

‎src/base/io-funcs-inl.h‎

Lines changed: 0 additions & 103 deletions
Original file line numberDiff line numberDiff line change
@@ -179,109 +179,6 @@ template<class T> inline void ReadIntegerVector(std::istream &is,
179179
}
180180

181181

182-
template<class T> inline void WriteIntegerArray(std::ostream &os, bool binary,
183-
const int64 array_sz,
184-
const T* const array) {
185-
// Compile time assertion that this is not called with a wrong type.
186-
KALDI_ASSERT_IS_INTEGER_TYPE(T);
187-
KALDI_ASSERT(array != NULL);
188-
if (binary) {
189-
char sz = sizeof(T);
190-
os.write(&sz, 1); // this is currently just a check.
191-
os.write(reinterpret_cast<const char *>(&array_sz), sizeof(array_sz));
192-
if (array_sz != 0) {
193-
os.write(reinterpret_cast<const char *>(array),
194-
sizeof(T) * array_sz);
195-
}
196-
} else {
197-
// focus here is on prettiness of text form rather than efficiency of
198-
// reading-in. reading-in is dominated by low-level operations anyway:
199-
// for efficiency use binary.
200-
os << "[ ";
201-
for (int64 i = 0; i < array_sz; ++i) {
202-
if (sizeof(T) == 1) { // To be consistent with WriteIntegerVector().
203-
os << static_cast<int16>(array[i]) << " ";
204-
} else {
205-
os << array[i] << " ";
206-
}
207-
}
208-
os << "]\n";
209-
}
210-
if (os.fail()) {
211-
throw std::runtime_error("Write failure in WriteIntegerType.");
212-
}
213-
}
214-
215-
216-
template<class T> inline void ReadIntegerArray(std::istream &is, bool binary,
217-
const int64 array_sz,
218-
T* const array) {
219-
KALDI_ASSERT_IS_INTEGER_TYPE(T);
220-
KALDI_ASSERT(array != NULL);
221-
if (binary) {
222-
int sz = is.peek();
223-
if (sz == sizeof(T)) {
224-
is.get();
225-
} else { // this is currently just a check.
226-
KALDI_ERR << "ReadIntegerArray: expected to see type of size "
227-
<< sizeof(T) << ", saw instead " << sz << ", at file position "
228-
<< is.tellg();
229-
}
230-
231-
int64 this_array_sz;
232-
is.read(reinterpret_cast<char *>(&this_array_sz), sizeof(this_array_sz));
233-
if (this_array_sz != array_sz) {
234-
KALDI_ERR << "ReadIntegerArray: expected to see array size of "
235-
<< array_sz << ", saw instead " << this_array_sz;
236-
}
237-
238-
if (is.fail()) {
239-
goto bad;
240-
}
241-
if (array_sz > 0) {
242-
is.read(reinterpret_cast<char *>(array), sizeof(T) * array_sz);
243-
}
244-
} else {
245-
is >> std::ws;
246-
if (is.peek() != static_cast<int>('[')) {
247-
KALDI_ERR << "ReadIntegerInteger: expected to see [, saw "
248-
<< is.peek() << ", at file position " << is.tellg();
249-
}
250-
is.get(); // consume the '['.
251-
is >> std::ws; // consume whitespace.
252-
T* array_pointer = array;
253-
while (is.peek() != static_cast<int>(']')) {
254-
if (sizeof(T) == 1) { // read/write chars as numbers.
255-
int16 next_t;
256-
is >> next_t >> std::ws;
257-
if (is.fail()) {
258-
goto bad;
259-
} else {
260-
*array_pointer = (T)next_t;
261-
array_pointer++;
262-
}
263-
} else {
264-
T next_t;
265-
is >> next_t >> std::ws;
266-
if (is.fail()) {
267-
goto bad;
268-
} else {
269-
*array_pointer = next_t;
270-
array_pointer++;
271-
}
272-
}
273-
}
274-
is.get(); // get the final ']'.
275-
}
276-
if (!is.fail()) {
277-
return;
278-
}
279-
bad:
280-
KALDI_ERR << "ReadIntegerArray: read failure at file position "
281-
<< is.tellg();
282-
}
283-
284-
285182
// Initialize an opened stream for writing by writing an optional binary
286183
// header and modifying the floating-point precision.
287184
inline void InitKaldiOutputStream(std::ostream &os, bool binary) {

‎src/base/io-funcs.h‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -182,17 +182,6 @@ template<class T> inline void WriteIntegerVector(std::ostream &os, bool binary,
182182
template<class T> inline void ReadIntegerVector(std::istream &is, bool binary,
183183
std::vector<T> *v);
184184

185-
/// Function for writing array of integer types.
186-
template<class T> inline void WriteIntegerArray(std::ostream &os, bool binary,
187-
const int64 array_sz,
188-
const T* const array);
189-
190-
/// Function for reading array of integer types. We assume it is the caller's
191-
/// responsibility to create <array> with size <array_sz>.
192-
template<class T> inline void ReadIntegerArray(std::istream &is, bool binary,
193-
const int64 array_sz,
194-
T* const array);
195-
196185
/// The WriteToken functions are for writing nonempty sequences of non-space
197186
/// characters. They are not for general strings.
198187
void WriteToken(std::ostream &os, bool binary, const char *token);

‎src/lm/const-arpa-lm.cc‎

Lines changed: 106 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,110 @@
2929

3030
namespace kaldi {
3131

32+
// Function for writing array of integer types.
33+
template<class T> inline void WriteIntegerArray(std::ostream &os, bool binary,
34+
const int64 array_sz,
35+
const T* const array) {
36+
// Compile time assertion that this is not called with a wrong type.
37+
KALDI_ASSERT_IS_INTEGER_TYPE(T);
38+
KALDI_ASSERT(array != NULL);
39+
if (binary) {
40+
char sz = sizeof(T);
41+
os.write(&sz, 1); // this is currently just a check.
42+
os.write(reinterpret_cast<const char *>(&array_sz), sizeof(array_sz));
43+
if (array_sz != 0) {
44+
os.write(reinterpret_cast<const char *>(array),
45+
sizeof(T) * array_sz);
46+
}
47+
} else {
48+
// focus here is on prettiness of text form rather than efficiency of
49+
// reading-in. reading-in is dominated by low-level operations anyway:
50+
// for efficiency use binary.
51+
os << "[ ";
52+
for (int64 i = 0; i < array_sz; ++i) {
53+
if (sizeof(T) == 1) { // To be consistent with WriteIntegerVector().
54+
os << static_cast<int16>(array[i]) << " ";
55+
} else {
56+
os << array[i] << " ";
57+
}
58+
}
59+
os << "]\n";
60+
}
61+
if (os.fail()) {
62+
throw std::runtime_error("Write failure in WriteIntegerType.");
63+
}
64+
}
65+
66+
// Function for reading array of integer types. We assume it is the caller's
67+
// responsibility to create <array> with size <array_sz>.
68+
template<class T> inline void ReadIntegerArray(std::istream &is, bool binary,
69+
const int64 array_sz,
70+
T* const array) {
71+
KALDI_ASSERT_IS_INTEGER_TYPE(T);
72+
KALDI_ASSERT(array != NULL);
73+
if (binary) {
74+
int sz = is.peek();
75+
if (sz == sizeof(T)) {
76+
is.get();
77+
} else { // this is currently just a check.
78+
KALDI_ERR << "ReadIntegerArray: expected to see type of size "
79+
<< sizeof(T) << ", saw instead " << sz << ", at file position "
80+
<< is.tellg();
81+
}
82+
83+
int64 this_array_sz;
84+
is.read(reinterpret_cast<char *>(&this_array_sz), sizeof(this_array_sz));
85+
if (this_array_sz != array_sz) {
86+
KALDI_ERR << "ReadIntegerArray: expected to see array size of "
87+
<< array_sz << ", saw instead " << this_array_sz;
88+
}
89+
90+
if (is.fail()) {
91+
goto bad;
92+
}
93+
if (array_sz > 0) {
94+
is.read(reinterpret_cast<char *>(array), sizeof(T) * array_sz);
95+
}
96+
} else {
97+
is >> std::ws;
98+
if (is.peek() != static_cast<int>('[')) {
99+
KALDI_ERR << "ReadIntegerInteger: expected to see [, saw "
100+
<< is.peek() << ", at file position " << is.tellg();
101+
}
102+
is.get(); // consume the '['.
103+
is >> std::ws; // consume whitespace.
104+
T* array_pointer = array;
105+
while (is.peek() != static_cast<int>(']')) {
106+
if (sizeof(T) == 1) { // read/write chars as numbers.
107+
int16 next_t;
108+
is >> next_t >> std::ws;
109+
if (is.fail()) {
110+
goto bad;
111+
} else {
112+
*array_pointer = (T)next_t;
113+
array_pointer++;
114+
}
115+
} else {
116+
T next_t;
117+
is >> next_t >> std::ws;
118+
if (is.fail()) {
119+
goto bad;
120+
} else {
121+
*array_pointer = next_t;
122+
array_pointer++;
123+
}
124+
}
125+
}
126+
is.get(); // get the final ']'.
127+
}
128+
if (!is.fail()) {
129+
return;
130+
}
131+
bad:
132+
KALDI_ERR << "ReadIntegerArray: read failure at file position "
133+
<< is.tellg();
134+
}
135+
32136
// Auxiliary struct for converting ConstArpaLm format langugae model to Arpa
33137
// format.
34138
struct ArpaLine {
@@ -714,7 +818,7 @@ void ConstArpaLm::Read(std::istream &is, bool binary) {
714818

715819
int first_char = is.peek();
716820
if (first_char == 4) { // Old on-disk format starts with length of int32.
717-
ReadInternalDeprecated(is, binary);
821+
ReadInternalOldFormat(is, binary);
718822
} else { // New on-disk format starts with token <ConstArpaLm>.
719823
ReadInternal(is, binary);
720824
}
@@ -786,7 +890,7 @@ void ConstArpaLm::ReadInternal(std::istream &is, bool binary) {
786890
initialized_ = true;
787891
}
788892

789-
void ConstArpaLm::ReadInternalDeprecated(std::istream &is, bool binary) {
893+
void ConstArpaLm::ReadInternalOldFormat(std::istream &is, bool binary) {
790894
KALDI_ASSERT(!initialized_);
791895
if (!binary) {
792896
KALDI_ERR << "text-mode reading is not implemented for ConstArpaLm.";

‎src/lm/const-arpa-lm.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -254,7 +254,7 @@ class ConstArpaLm {
254254
}
255255

256256
// Reads the ConstArpaLm format language model. It calls ReadInternal() or
257-
// ReadInternalDeprecated() to do the actual reading.
257+
// ReadInternalOldFormat() to do the actual reading.
258258
void Read(std::istream &is, bool binary);
259259

260260
// Writes the language model in ConstArpaLm format.
@@ -285,7 +285,7 @@ class ConstArpaLm {
285285
// that handles the old on-disk format. We keep this for back-compatibility
286286
// purpose. We have modified the Write() function so for all the new on-disk
287287
// format, ReadInternal() will be called.
288-
void ReadInternalDeprecated(std::istream &is, bool binary);
288+
void ReadInternalOldFormat(std::istream &is, bool binary);
289289

290290
// Loops up n-gram probability for given word sequence. Backoff is handled by
291291
// recursively calling this function.

0 commit comments

Comments
 (0)