pavel/fedavg_mpi
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

using adam seems like a sane idea

Browse Source
This commit is contained in:
Pavel Lutskov
2019-12-11 22:44:18 -08:00
parent 598e59ca2e
commit 0372f0ee9c
2 changed files with 5 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
library.py
View File

@@ -39,7 +39,7 @@ def create_mnist_network():
tf.keras.layers.Dense(30, input_shape=(784,), activation='relu'), tf.keras.layers.Dense(30, input_shape=(784,), activation='relu'),
tf.keras.layers.Dense(10, activation='softmax') tf.keras.layers.Dense(10, activation='softmax')
]) ])
model.compile(loss='categorical_crossentropy', optimizer='sgd', model.compile(loss='categorical_crossentropy', optimizer='adam',
metrics=['accuracy']) metrics=['accuracy'])
return model return model
@@ -51,7 +51,7 @@ def create_cbow_network(win, embed):
blowup = tf.keras.layers.Dense(len(vocab), activation='softmax')(cbow) blowup = tf.keras.layers.Dense(len(vocab), activation='softmax')(cbow)
mod = tf.keras.Model(inputs=ctxt, outputs=blowup) mod = tf.keras.Model(inputs=ctxt, outputs=blowup)
mod.compile( mod.compile(
optimizer='sgd', optimizer='adam',
loss='categorical_crossentropy', loss='categorical_crossentropy',
) )
return mod return mod

6
main.c
View File

@@ -323,8 +323,8 @@ void learner() {
int my_batcher_rid = rid % number_of(BATCHER); int my_batcher_rid = rid % number_of(BATCHER);
int batcher = mpi_id_from_role_id(BATCHER, my_batcher_rid); int batcher = mpi_id_from_role_id(BATCHER, my_batcher_rid);
int dispatcher = mpi_id_from_role_id(DISPATCHER, 0); int dispatcher = mpi_id_from_role_id(DISPATCHER, 0);
INFO_PRINTF("%d is Learner %d assigned to batcher %d\n", getpid(), INFO_PRINTF("Learner %d (pid %d) is assigned to pipeline %d\n", rid,
rid, my_batcher_rid); getpid(), my_batcher_rid);
PyObject* net = create_network(WIN, EMB); PyObject* net = create_network(WIN, EMB);
WeightList wl; WeightList wl;
@@ -413,7 +413,7 @@ void dispatcher() {
float delta_t = finish - start; float delta_t = finish - start;
float delta_l = first_loss - crt_loss; float delta_l = first_loss - crt_loss;
INFO_PRINTF( INFO_PRINTF(
"Laptop MPI sgd consecutive_batch W%d E%d " "Laptop MPI adam consecutive_batch W%d E%d "
"BS%d R%d bpe%d LPR%d pp%d," "BS%d R%d bpe%d LPR%d pp%d,"
"%f,%f,%f\n", WIN, EMB, BS, COMM, ITER, lpr, g_argc - 1, "%f,%f,%f\n", WIN, EMB, BS, COMM, ITER, lpr, g_argc - 1,
delta_l / COMM, delta_l / delta_t, min_loss); delta_l / COMM, delta_l / delta_t, min_loss);