PocketSphinx  5prealpha
dict.c
1 /* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
2 /* ====================================================================
3  * Copyright (c) 1999-2004 Carnegie Mellon University. All rights
4  * reserved.
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  *
10  * 1. Redistributions of source code must retain the above copyright
11  * notice, this list of conditions and the following disclaimer.
12  *
13  * 2. Redistributions in binary form must reproduce the above copyright
14  * notice, this list of conditions and the following disclaimer in
15  * the documentation and/or other materials provided with the
16  * distribution.
17  *
18  * This work was supported in part by funding from the Defense Advanced
19  * Research Projects Agency and the National Science Foundation of the
20  * United States of America, and the CMU Sphinx Speech Consortium.
21  *
22  * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND
23  * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
24  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
25  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
26  * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
27  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
28  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
32  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33  *
34  * ====================================================================
35  *
36  */
37 
38 /* System headers. */
39 #include <string.h>
40 
41 /* SphinxBase headers. */
42 #include <sphinxbase/pio.h>
43 #include <sphinxbase/strfuncs.h>
44 
45 /* Local headers. */
46 #include "dict.h"
47 
48 
49 #define DELIM " \t\n" /* Set of field separator characters */
50 #define DEFAULT_NUM_PHONE (MAX_S3CIPID+1)
51 
52 #if WIN32
53 #define snprintf sprintf_s
54 #endif
55 
56 extern const char *const cmu6_lts_phone_table[];
57 
58 static s3cipid_t
59 dict_ciphone_id(dict_t * d, const char *str)
60 {
61  if (d->nocase)
62  return bin_mdef_ciphone_id_nocase(d->mdef, str);
63  else
64  return bin_mdef_ciphone_id(d->mdef, str);
65 }
66 
67 
68 const char *
69 dict_ciphone_str(dict_t * d, s3wid_t wid, int32 pos)
70 {
71  assert(d != NULL);
72  assert((wid >= 0) && (wid < d->n_word));
73  assert((pos >= 0) && (pos < d->word[wid].pronlen));
74 
75  return bin_mdef_ciphone_str(d->mdef, d->word[wid].ciphone[pos]);
76 }
77 
78 
79 s3wid_t
80 dict_add_word(dict_t * d, char const *word, s3cipid_t const * p, int32 np)
81 {
82  int32 len;
83  dictword_t *wordp;
84  s3wid_t newwid;
85  char *wword;
86 
87  if (d->n_word >= d->max_words) {
88  E_INFO("Reallocating to %d KiB for word entries\n",
89  (d->max_words + S3DICT_INC_SZ) * sizeof(dictword_t) / 1024);
90  d->word =
91  (dictword_t *) ckd_realloc(d->word,
92  (d->max_words +
93  S3DICT_INC_SZ) * sizeof(dictword_t));
94  d->max_words = d->max_words + S3DICT_INC_SZ;
95  }
96 
97  wordp = d->word + d->n_word;
98  wordp->word = (char *) ckd_salloc(word); /* Freed in dict_free */
99 
100  /* Determine base/alt wids */
101  wword = ckd_salloc(word);
102  if ((len = dict_word2basestr(wword)) > 0) {
103  int32 w;
104 
105  /* Truncated to a baseword string; find its ID */
106  if (hash_table_lookup_int32(d->ht, wword, &w) < 0) {
107  E_ERROR("Missing base word for: %s\n", word);
108  ckd_free(wword);
109  ckd_free(wordp->word);
110  wordp->word = NULL;
111  return BAD_S3WID;
112  }
113 
114  /* Link into alt list */
115  wordp->basewid = w;
116  wordp->alt = d->word[w].alt;
117  d->word[w].alt = d->n_word;
118  } else {
119  wordp->alt = BAD_S3WID;
120  wordp->basewid = d->n_word;
121  }
122  ckd_free(wword);
123 
124  /* Associate word string with d->n_word in hash table */
125  if (hash_table_enter_int32(d->ht, wordp->word, d->n_word) != d->n_word) {
126  ckd_free(wordp->word);
127  wordp->word = NULL;
128  return BAD_S3WID;
129  }
130 
131  /* Fill in word entry, and set defaults */
132  if (p && (np > 0)) {
133  wordp->ciphone = (s3cipid_t *) ckd_malloc(np * sizeof(s3cipid_t)); /* Freed in dict_free */
134  memcpy(wordp->ciphone, p, np * sizeof(s3cipid_t));
135  wordp->pronlen = np;
136  }
137  else {
138  wordp->ciphone = NULL;
139  wordp->pronlen = 0;
140  }
141 
142  newwid = d->n_word++;
143 
144  return newwid;
145 }
146 
147 
148 static int32
149 dict_read(FILE * fp, dict_t * d)
150 {
151  lineiter_t *li;
152  char **wptr;
153  s3cipid_t *p;
154  int32 lineno, nwd;
155  s3wid_t w;
156  int32 i, maxwd;
157  size_t stralloc, phnalloc;
158 
159  maxwd = 512;
160  p = (s3cipid_t *) ckd_calloc(maxwd + 4, sizeof(*p));
161  wptr = (char **) ckd_calloc(maxwd, sizeof(char *)); /* Freed below */
162 
163  lineno = 0;
164  stralloc = phnalloc = 0;
165  for (li = lineiter_start(fp); li; li = lineiter_next(li)) {
166  lineno++;
167  if (0 == strncmp(li->buf, "##", 2)
168  || 0 == strncmp(li->buf, ";;", 2))
169  continue;
170 
171  if ((nwd = str2words(li->buf, wptr, maxwd)) < 0) {
172  /* Increase size of p, wptr. */
173  nwd = str2words(li->buf, NULL, 0);
174  assert(nwd > maxwd); /* why else would it fail? */
175  maxwd = nwd;
176  p = (s3cipid_t *) ckd_realloc(p, (maxwd + 4) * sizeof(*p));
177  wptr = (char **) ckd_realloc(wptr, maxwd * sizeof(*wptr));
178  }
179 
180  if (nwd == 0) /* Empty line */
181  continue;
182  /* wptr[0] is the word-string and wptr[1..nwd-1] the pronunciation sequence */
183  if (nwd == 1) {
184  E_ERROR("Line %d: No pronunciation for word '%s'; ignored\n",
185  lineno, wptr[0]);
186  continue;
187  }
188 
189 
190  /* Convert pronunciation string to CI-phone-ids */
191  for (i = 1; i < nwd; i++) {
192  p[i - 1] = dict_ciphone_id(d, wptr[i]);
193  if (NOT_S3CIPID(p[i - 1])) {
194  E_ERROR("Line %d: Phone '%s' is mising in the acoustic model; word '%s' ignored\n",
195  lineno, wptr[i], wptr[0]);
196  break;
197  }
198  }
199 
200  if (i == nwd) { /* All CI-phones successfully converted to IDs */
201  w = dict_add_word(d, wptr[0], p, nwd - 1);
202  if (NOT_S3WID(w))
203  E_ERROR
204  ("Line %d: Failed to add the word '%s' (duplicate?); ignored\n",
205  lineno, wptr[0]);
206  else {
207  stralloc += strlen(d->word[w].word);
208  phnalloc += d->word[w].pronlen * sizeof(s3cipid_t);
209  }
210  }
211  }
212  E_INFO("Dictionary size %d, allocated %d KiB for strings, %d KiB for phones\n",
213  dict_size(d), (int)stralloc / 1024, (int)phnalloc / 1024);
214  ckd_free(p);
215  ckd_free(wptr);
216 
217  return 0;
218 }
219 
220 int
221 dict_write(dict_t *dict, char const *filename, char const *format)
222 {
223  FILE *fh;
224  int i;
225 
226  if ((fh = fopen(filename, "w")) == NULL) {
227  E_ERROR_SYSTEM("Failed to open '%s'", filename);
228  return -1;
229  }
230  for (i = 0; i < dict->n_word; ++i) {
231  char *phones;
232  int j, phlen;
233  if (!dict_real_word(dict, i))
234  continue;
235  for (phlen = j = 0; j < dict_pronlen(dict, i); ++j)
236  phlen += strlen(dict_ciphone_str(dict, i, j)) + 1;
237  phones = ckd_calloc(1, phlen);
238  for (j = 0; j < dict_pronlen(dict, i); ++j) {
239  strcat(phones, dict_ciphone_str(dict, i, j));
240  if (j != dict_pronlen(dict, i) - 1)
241  strcat(phones, " ");
242  }
243  fprintf(fh, "%-30s %s\n", dict_wordstr(dict, i), phones);
244  ckd_free(phones);
245  }
246  fclose(fh);
247  return 0;
248 }
249 
250 
251 dict_t *
252 dict_init(cmd_ln_t *config, bin_mdef_t * mdef)
253 {
254  FILE *fp, *fp2;
255  int32 n;
256  lineiter_t *li;
257  dict_t *d;
258  s3cipid_t sil;
259  char const *dictfile = NULL, *fillerfile = NULL;
260 
261  if (config) {
262  dictfile = cmd_ln_str_r(config, "-dict");
263  fillerfile = cmd_ln_str_r(config, "_fdict");
264  }
265 
266  /*
267  * First obtain #words in dictionary (for hash table allocation).
268  * Reason: The PC NT system doesn't like to grow memory gradually. Better to allocate
269  * all the required memory in one go.
270  */
271  fp = NULL;
272  n = 0;
273  if (dictfile) {
274  if ((fp = fopen(dictfile, "r")) == NULL) {
275  E_ERROR_SYSTEM("Failed to open dictionary file '%s' for reading", dictfile);
276  return NULL;
277  }
278  for (li = lineiter_start(fp); li; li = lineiter_next(li)) {
279  if (0 != strncmp(li->buf, "##", 2)
280  && 0 != strncmp(li->buf, ";;", 2))
281  n++;
282  }
283  fseek(fp, 0L, SEEK_SET);
284  }
285 
286  fp2 = NULL;
287  if (fillerfile) {
288  if ((fp2 = fopen(fillerfile, "r")) == NULL) {
289  E_ERROR_SYSTEM("Failed to open filler dictionary file '%s' for reading", fillerfile);
290  fclose(fp);
291  return NULL;
292  }
293  for (li = lineiter_start(fp2); li; li = lineiter_next(li)) {
294  if (0 != strncmp(li->buf, "##", 2)
295  && 0 != strncmp(li->buf, ";;", 2))
296  n++;
297  }
298  fseek(fp2, 0L, SEEK_SET);
299  }
300 
301  /*
302  * Allocate dict entries. HACK!! Allow some extra entries for words not in file.
303  * Also check for type size restrictions.
304  */
305  d = (dict_t *) ckd_calloc(1, sizeof(dict_t)); /* freed in dict_free() */
306  d->refcnt = 1;
307  d->max_words =
308  (n + S3DICT_INC_SZ < MAX_S3WID) ? n + S3DICT_INC_SZ : MAX_S3WID;
309  if (n >= MAX_S3WID) {
310  E_ERROR("Number of words in dictionaries (%d) exceeds limit (%d)\n", n,
311  MAX_S3WID);
312  fclose(fp);
313  fclose(fp2);
314  ckd_free(d);
315  return NULL;
316  }
317 
318  E_INFO("Allocating %d * %d bytes (%d KiB) for word entries\n",
319  d->max_words, sizeof(dictword_t),
320  d->max_words * sizeof(dictword_t) / 1024);
321  d->word = (dictword_t *) ckd_calloc(d->max_words, sizeof(dictword_t)); /* freed in dict_free() */
322  d->n_word = 0;
323  if (mdef)
324  d->mdef = bin_mdef_retain(mdef);
325 
326  /* Create new hash table for word strings; case-insensitive word strings */
327  if (config && cmd_ln_exists_r(config, "-dictcase"))
328  d->nocase = cmd_ln_boolean_r(config, "-dictcase");
329  d->ht = hash_table_new(d->max_words, d->nocase);
330 
331  /* Digest main dictionary file */
332  if (fp) {
333  E_INFO("Reading main dictionary: %s\n", dictfile);
334  dict_read(fp, d);
335  fclose(fp);
336  E_INFO("%d words read\n", d->n_word);
337  }
338 
339  if (dict_wordid(d, S3_START_WORD) != BAD_S3WID) {
340  E_ERROR("Remove sentence start word '<s>' from the dictionary\n");
341  dict_free(d);
342  return NULL;
343  }
344  if (dict_wordid(d, S3_FINISH_WORD) != BAD_S3WID) {
345  E_ERROR("Remove sentence start word '</s>' from the dictionary\n");
346  dict_free(d);
347  return NULL;
348  }
349  if (dict_wordid(d, S3_SILENCE_WORD) != BAD_S3WID) {
350  E_ERROR("Remove silence word '<sil>' from the dictionary\n");
351  dict_free(d);
352  return NULL;
353  }
354 
355  /* Now the filler dictionary file, if it exists */
356  d->filler_start = d->n_word;
357  if (fillerfile) {
358  E_INFO("Reading filler dictionary: %s\n", fillerfile);
359  dict_read(fp2, d);
360  fclose(fp2);
361  E_INFO("%d words read\n", d->n_word - d->filler_start);
362  }
363  if (mdef)
364  sil = bin_mdef_silphone(mdef);
365  else
366  sil = 0;
367  if (dict_wordid(d, S3_START_WORD) == BAD_S3WID) {
368  dict_add_word(d, S3_START_WORD, &sil, 1);
369  }
370  if (dict_wordid(d, S3_FINISH_WORD) == BAD_S3WID) {
371  dict_add_word(d, S3_FINISH_WORD, &sil, 1);
372  }
373  if (dict_wordid(d, S3_SILENCE_WORD) == BAD_S3WID) {
374  dict_add_word(d, S3_SILENCE_WORD, &sil, 1);
375  }
376 
377  d->filler_end = d->n_word - 1;
378 
379  /* Initialize distinguished word-ids */
380  d->startwid = dict_wordid(d, S3_START_WORD);
381  d->finishwid = dict_wordid(d, S3_FINISH_WORD);
382  d->silwid = dict_wordid(d, S3_SILENCE_WORD);
383 
384  if ((d->filler_start > d->filler_end)
385  || (!dict_filler_word(d, d->silwid))) {
386  E_ERROR("Word '%s' must occur (only) in filler dictionary\n",
387  S3_SILENCE_WORD);
388  dict_free(d);
389  return NULL;
390  }
391 
392  /* No check that alternative pronunciations for filler words are in filler range!! */
393 
394  return d;
395 }
396 
397 
398 s3wid_t
399 dict_wordid(dict_t *d, const char *word)
400 {
401  int32 w;
402 
403  assert(d);
404  assert(word);
405 
406  if (hash_table_lookup_int32(d->ht, word, &w) < 0)
407  return (BAD_S3WID);
408  return w;
409 }
410 
411 
412 int
413 dict_filler_word(dict_t *d, s3wid_t w)
414 {
415  assert(d);
416  assert((w >= 0) && (w < d->n_word));
417 
418  w = dict_basewid(d, w);
419  if ((w == d->startwid) || (w == d->finishwid))
420  return 0;
421  if ((w >= d->filler_start) && (w <= d->filler_end))
422  return 1;
423  return 0;
424 }
425 
426 int
427 dict_real_word(dict_t *d, s3wid_t w)
428 {
429  assert(d);
430  assert((w >= 0) && (w < d->n_word));
431 
432  w = dict_basewid(d, w);
433  if ((w == d->startwid) || (w == d->finishwid))
434  return 0;
435  if ((w >= d->filler_start) && (w <= d->filler_end))
436  return 0;
437  return 1;
438 }
439 
440 
441 int32
442 dict_word2basestr(char *word)
443 {
444  int32 i, len;
445 
446  len = strlen(word);
447  if (word[len - 1] == ')') {
448  for (i = len - 2; (i > 0) && (word[i] != '('); --i);
449 
450  if (i > 0) {
451  /* The word is of the form <baseword>(...); strip from left-paren */
452  word[i] = '\0';
453  return i;
454  }
455  }
456 
457  return -1;
458 }
459 
460 dict_t *
462 {
463  ++d->refcnt;
464  return d;
465 }
466 
467 int
469 {
470  int i;
471  dictword_t *word;
472 
473  if (d == NULL)
474  return 0;
475  if (--d->refcnt > 0)
476  return d->refcnt;
477 
478  /* First Step, free all memory allocated for each word */
479  for (i = 0; i < d->n_word; i++) {
480  word = (dictword_t *) & (d->word[i]);
481  if (word->word)
482  ckd_free((void *) word->word);
483  if (word->ciphone)
484  ckd_free((void *) word->ciphone);
485  }
486 
487  if (d->word)
488  ckd_free((void *) d->word);
489  if (d->ht)
490  hash_table_free(d->ht);
491  if (d->mdef)
492  bin_mdef_free(d->mdef);
493  ckd_free((void *) d);
494 
495  return 0;
496 }
497 
498 void
500 {
501  E_INFO_NOFN("Initialization of dict_t, report:\n");
502  E_INFO_NOFN("Max word: %d\n", d->max_words);
503  E_INFO_NOFN("No of word: %d\n", d->n_word);
504  E_INFO_NOFN("\n");
505 }
dict_t * dict_init(cmd_ln_t *config, bin_mdef_t *mdef)
Initialize a new dictionary.
Definition: dict.c:252
POCKETSPHINX_EXPORT s3wid_t dict_wordid(dict_t *d, const char *word)
Return word id for given word string if present.
Definition: dict.c:399
char * word
Ascii word string.
Definition: dict.h:64
const char * bin_mdef_ciphone_str(bin_mdef_t *m, int32 ci)
In: ciphone id for which name wanted.
Definition: bin_mdef.c:737
int dict_free(dict_t *d)
Release a pointer to a dictionary.
Definition: dict.c:468
int bin_mdef_ciphone_id(bin_mdef_t *m, const char *ciphone)
Context-independent phone lookup.
Definition: bin_mdef.c:691
Operations on dictionary.
#define BAD_S3WID
Dictionary word id.
Definition: s3types.h:90
int32 n_word
#Occupied entries in dict; ie, excluding empty slots
Definition: dict.h:82
const char * dict_ciphone_str(dict_t *d, s3wid_t wid, int32 pos)
Return value: CI phone string for the given word, phone position.
Definition: dict.c:69
dict_t * dict_retain(dict_t *d)
Retain a pointer to an dict_t.
Definition: dict.c:461
int32 filler_end
Last filler word id (read from filler dict)
Definition: dict.h:84
int dict_write(dict_t *dict, char const *filename, char const *format)
Write dictionary to a file.
Definition: dict.c:221
s3wid_t startwid
FOR INTERNAL-USE ONLY.
Definition: dict.h:85
s3wid_t silwid
FOR INTERNAL-USE ONLY.
Definition: dict.h:87
s3wid_t alt
Next alternative pronunciation id, NOT_S3WID if none.
Definition: dict.h:67
int16 s3cipid_t
Size definitions for more semantially meaningful units.
Definition: s3types.h:63
int dict_filler_word(dict_t *d, s3wid_t w)
Return 1 if w is a filler word, 0 if not.
Definition: dict.c:413
a structure for one dictionary word.
Definition: dict.h:63
s3wid_t basewid
Base pronunciation id.
Definition: dict.h:68
s3wid_t finishwid
FOR INTERNAL-USE ONLY.
Definition: dict.h:86
a structure for a dictionary.
Definition: dict.h:76
POCKETSPHINX_EXPORT int dict_real_word(dict_t *d, s3wid_t w)
Test if w is a &quot;real&quot; word, i.e.
Definition: dict.c:427
int32 filler_start
First filler word id (read from filler dict)
Definition: dict.h:83
int bin_mdef_ciphone_id_nocase(bin_mdef_t *m, const char *ciphone)
Case-insensitive context-independent phone lookup.
Definition: bin_mdef.c:714
int bin_mdef_free(bin_mdef_t *m)
Release a pointer to a binary mdef.
Definition: bin_mdef.c:272
int32 max_words
#Entries allocated in dict, including empty slots
Definition: dict.h:81
s3cipid_t * ciphone
Pronunciation.
Definition: dict.h:65
bin_mdef_t * bin_mdef_retain(bin_mdef_t *m)
Retain a pointer to a bin_mdef_t.
Definition: bin_mdef.c:265
dictword_t * word
Array of entries in dictionary.
Definition: dict.h:79
s3wid_t dict_add_word(dict_t *d, char const *word, s3cipid_t const *p, int32 np)
Add a word with the given ciphone pronunciation list to the dictionary.
Definition: dict.c:80
int32 pronlen
Pronunciation length.
Definition: dict.h:66
bin_mdef_t * mdef
Model definition used for phone IDs; NULL if none used.
Definition: dict.h:78
#define dict_size(d)
Packaged macro access to dictionary members.
Definition: dict.h:151
hash_table_t * ht
Hash table for mapping word strings to word ids.
Definition: dict.h:80
void dict_report(dict_t *d)
Report a dictionary structure.
Definition: dict.c:499
int32 dict_word2basestr(char *word)
If the given word contains a trailing &quot;(....)&quot; (i.e., a Sphinx-II style alternative pronunciation spe...
Definition: dict.c:442