added batcher and even made it twice as fast

as it was a minute ago
This commit is contained in:
2019-11-30 22:40:23 -08:00
parent 101248965c
commit 7409eca38b
3 changed files with 38 additions and 22 deletions

View File

@@ -9,7 +9,7 @@ from mynet import load_mnist, onehot
def word_tokenize(s: str):
l = ''.join(c if c.isalpha() else ' ' for c in s)
l = ''.join(c.lower() if c.isalpha() else ' ' for c in s)
return l.split()
@@ -47,7 +47,7 @@ def create_cbow_network(win, vocab, embed):
def token_generator(filename):
with open(filename) as f:
for l in f.readlines(500):
for l in f.readlines():
if not l.isspace():
tok = word_tokenize(l)
if tok: