added batcher and even made it twice as fast

as it was a minute ago
2019-11-30 22:40:23 -08:00
parent 101248965c
commit 7409eca38b
3 changed files with 38 additions and 22 deletions
--- a/bridge.pyx
+++ b/bridge.pyx
@@ -58,9 +58,9 @@ cdef public int get_tokens(WordList* wl, const char *filename):

 cdef public long vocab_idx_of(Word* w):
    word = w.data.decode('utf-8')
-    if word.lower() in nn.vocab:
-        return nn.vocab.index(word.lower())
-    else:
+    try:
+        return nn.vocab.index(word)
+    except ValueError:
        return -1