44 #include <sphinxbase/byteorder.h>
46 #include "ngram_model_internal.h"
47 #include "ngrams_raw.h"
50 ngram_ord_comparator(
const void *a_raw,
const void *b_raw)
56 while (a_w_ptr < a->order && b_w_ptr < b->order) {
57 if (a->words[a_w_ptr] == b->words[b_w_ptr]) {
62 if (a->words[a_w_ptr] < b->words[b_w_ptr])
67 return a->order - b->order;
72 logmath_t * lmath,
int order,
int order_max,
78 char *wptr[NGRAM_MAX_ORDER + 1];
84 E_ERROR(
"Unexpected end of ARPA file. Failed to read %d-gram\n",
88 words_expected = order + 1;
91 NGRAM_MAX_ORDER + 1)) < words_expected) {
92 E_ERROR(
"Format error; %d-gram ignored: %s\n", order, (*li)->buf);
96 raw_ngram->order = order;
98 if (order == order_max) {
99 raw_ngram->prob =
atof_c(wptr[0]);
100 if (raw_ngram->prob > 0) {
101 E_WARN(
"%d-gram '%s' has positive probability\n", order, wptr[1]);
102 raw_ngram->prob = 0.0f;
108 float weight, backoff;
112 E_WARN(
"%d-gram '%s' has positive probability\n", order, wptr[1]);
113 raw_ngram->prob = 0.0f;
120 if (n == order + 1) {
121 raw_ngram->backoff = 0.0f;
124 backoff =
atof_c(wptr[order + 1]);
130 (uint32 *)
ckd_calloc(order,
sizeof(*raw_ngram->words));
131 for (word_out = raw_ngram->words + order - 1, i = 1;
132 word_out >= raw_ngram->words; --word_out, i++) {
141 int order,
int order_max)
143 char expected_header[20];
146 sprintf(expected_header,
"\\%d-grams:", order);
147 while (*li && strcmp((*li)->buf, expected_header) != 0) {
152 E_ERROR(
"Failed to find '%s', language model file truncated\n", expected_header);
157 for (i = 0; i < count; i++) {
158 if (read_ngram_instance(li, wid, lmath, order, order_max,
159 &((*raw_ngrams)[i])) < 0)
163 qsort(*raw_ngrams, count,
sizeof(
ngram_raw_t), &ngram_ord_comparator);
177 for (order_it = 2; order_it <= order; order_it++) {
178 if (ngrams_raw_read_order(&raw_ngrams[order_it - 2], li, wid, lmath,
179 counts[order_it - 1], order_it, order) < 0)
185 E_ERROR(
"ARPA file ends without end-mark\n");
186 ngrams_raw_free(raw_ngrams, counts, order);
190 if (strcmp((*li)->buf,
"\\end\\") != 0) {
192 (
"Finished reading ARPA file. Expecting end mark but found '%s'\n",
201 read_dmp_weight_array(FILE * fp,
logmath_t * lmath, uint8 do_swap,
208 fread(&k,
sizeof(k), 1, fp);
213 fread(tmp_weight_arr,
sizeof(*tmp_weight_arr), k, fp);
214 for (i = 0; i < k; i++) {
216 SWAP_INT32(&tmp_weight_arr[i].l);
218 tmp_weight_arr[i].f =
222 for (i = 0; i < counts; i++) {
223 if (weight_idx == 0) {
225 tmp_weight_arr[(int) raw_ngrams[i].prob].f;
227 raw_ngrams[i].backoff =
228 tmp_weight_arr[(int) raw_ngrams[i].backoff].f;
234 #define BIGRAM_SEGMENT_SIZE 9
237 ngrams_raw_read_dmp(FILE * fp,
logmath_t * lmath, uint32 * counts,
238 int order, uint32 * unigram_next, uint8 do_swap)
241 uint16 *bigrams_next;
248 sizeof(*raw_ngrams[0]));
250 (uint16 *)
ckd_calloc((
size_t) (counts[1] + 1),
251 sizeof(*bigrams_next));
253 for (j = 0; j <= (int32) counts[1]; j++) {
254 uint16 wid, prob_idx, bo_idx;
257 fread(&wid,
sizeof(wid), 1, fp);
260 raw_ngram->order = 2;
261 while (ngram_idx < counts[0] && j == unigram_next[ngram_idx]) {
265 if (j != counts[1]) {
267 (uint32 *)
ckd_calloc(2,
sizeof(*raw_ngram->words));
268 raw_ngram->words[0] = (uint32) wid;
269 raw_ngram->words[1] = (uint32) ngram_idx - 1;
272 fread(&prob_idx,
sizeof(prob_idx), 1, fp);
273 fread(&bo_idx,
sizeof(bo_idx), 1, fp);
274 fread(&bigrams_next[j],
sizeof(bigrams_next[j]), 1, fp);
276 SWAP_INT16(&prob_idx);
278 SWAP_INT16(&bigrams_next[j]);
281 if (j != counts[1]) {
282 raw_ngram->prob = prob_idx + 0.5f;
283 raw_ngram->backoff = bo_idx + 0.5f;
287 if (ngram_idx < counts[0]) {
288 E_ERROR(
"Corrupted model, not enough unigrams %d %d\n", ngram_idx, counts[0]);
290 ngrams_raw_free(raw_ngrams, counts, order);
298 sizeof(*raw_ngrams[1]));
299 for (j = 0; j < (int32) counts[2]; j++) {
300 uint16 wid, prob_idx;
303 fread(&wid,
sizeof(wid), 1, fp);
304 fread(&prob_idx,
sizeof(prob_idx), 1, fp);
307 SWAP_INT16(&prob_idx);
310 raw_ngram->order = 3;
312 (uint32 *)
ckd_calloc(3,
sizeof(*raw_ngram->words));
313 raw_ngram->words[0] = (uint32) wid;
314 raw_ngram->prob = prob_idx + 0.5f;
319 read_dmp_weight_array(fp, lmath, do_swap, (int32) counts[1],
325 read_dmp_weight_array(fp, lmath, do_swap, (int32) counts[1],
328 read_dmp_weight_array(fp, lmath, do_swap, (int32) counts[2],
331 fread(&k,
sizeof(k), 1, fp);
334 tseg_base = (int32 *)
ckd_calloc(k,
sizeof(int32));
335 fread(tseg_base,
sizeof(int32), k, fp);
337 for (j = 0; j < (uint32) k; j++) {
338 SWAP_INT32(&tseg_base[j]);
342 for (j = 1; j <= counts[1]; j++) {
343 uint32 next_ngram_idx =
344 (uint32) (tseg_base[j >> BIGRAM_SEGMENT_SIZE] +
346 while (ngram_idx < next_ngram_idx) {
347 raw_ngrams[1][ngram_idx].words[1] =
348 raw_ngrams[0][j - 1].words[0];
349 raw_ngrams[1][ngram_idx].words[2] =
350 raw_ngrams[0][j - 1].words[1];
356 if (ngram_idx < counts[2]) {
357 E_ERROR(
"Corrupted model, some trigrams have no corresponding bigram\n");
359 ngrams_raw_free(raw_ngrams, counts, order);
366 qsort(raw_ngrams[0], (
size_t) counts[1],
sizeof(*raw_ngrams[0]),
367 &ngram_ord_comparator);
369 qsort(raw_ngrams[1], (
size_t) counts[2],
sizeof(*raw_ngrams[1]),
370 &ngram_ord_comparator);
376 ngrams_raw_free(
ngram_raw_t ** raw_ngrams, uint32 * counts,
int order)
381 for (order_it = 0; order_it < order - 1; order_it++) {
382 for (num = 0; num < counts[order_it + 1]; num++) {
383 ckd_free(raw_ngrams[order_it][num].words);
SPHINXBASE_EXPORT int32 hash_table_lookup_int32(hash_table_t *h, const char *key, int32 *val)
Look up a 32-bit integer value in a hash table.
Miscellaneous useful string functions.
#define ckd_calloc(n, sz)
Macros to simplify the use of above functions.
#define E_ERROR(...)
Print error message to error log.
Sphinx's memory allocation/deallocation routines.
SPHINXBASE_EXPORT void ckd_free(void *ptr)
Test and free a 1-D array.
SPHINXBASE_EXPORT double atof_c(char const *str)
Locale independent version of atof().
SPHINXBASE_EXPORT lineiter_t * lineiter_next(lineiter_t *li)
Move to the next line in the file.
Implementation of logging routines.
#define E_WARN(...)
Print warning message to error log.
SPHINXBASE_EXPORT float logmath_log10_to_log_float(logmath_t *lmath, float64 log_p)
Convert base 10 log (in floating point) to float log in base B.
SPHINXBASE_EXPORT int32 str2words(char *line, char **wptr, int32 n_wptr)
Convert a line to an array of "words", based on whitespace separators.
file IO related operations.