pavel/fedavg_mpi
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

tokenizer and batcher are sane and work

Browse Source 
i wonder how much pain it's gonna cause me to change
something in the code
This commit is contained in:
Pavel Lutskov
2019-11-30 22:16:42 -08:00
parent 04c35ed9b6
commit 101248965c
6 changed files with 321 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files

293
main.c
View File

@@ -1,7 +1,8 @@
#include "cythoned/library.h"
#include "cythoned/bridge.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <mpi.h>
#define TAG_IDGAF 0
@@ -10,10 +11,15 @@
#define TAG_WEIGH 3
#define TAG_READY 4
#define TAG_BREAK 5
#define TAG_STLEN 6
#define TAG_SWORD 7
#define TAG_IWORD 8
#define COMM 100
#define ITER 20
#define BS 50
#define EMB 20
#define WIN 2
#define FSPC 1
#define in_range(i, x) (size_t (i) = 0; (i) < (x); (i)++)
@@ -24,109 +30,75 @@
#define INFO_PRINTLN(what) \
do { fprintf(stderr, "%s\n", what); } while(0)
// char_stream -> tokenize -> word_strem -> filter + batch -> slave network
typedef enum{
DATA,
TOKENIZER,
FILTERER,
BATCHER,
SLAVE,
MASTER
} Role;
typedef struct IntQueue IntQueue;
struct IntQueue {
int head;
int tail;
size_t size;
int* data;
};
void queue_from_size(IntQueue* q, size_t s) {
q->data = malloc(s * sizeof(int));
q->size = s+1;
q->head = 0;
q->tail = 0;
}
void push_queue(IntQueue *q, int d) {
// Assuming queue is not full
q->data[q->tail] = d;
q->tail = (q->tail + 1) % q->size;
}
int pop_queue(IntQueue *q) {
int d = q->data[q->head];
q->head = (q->head + 1) % q->size;
return d;
}
int queue_empty(IntQueue *q) {
return q->head == q->tail;
}
int queue_full(IntQueue *q) {
return ((q->tail + 1) % q->size) == q->head;
}
int number_of_nodes() {
int world_size() {
int n;
MPI_Comm_size(MPI_COMM_WORLD, &n);
return n;
}
int number_of_masters() {
return 1;
}
int number_of_readers() {
return 1;
}
int number_of_slaves() {
return number_of_nodes() - number_of_masters() - number_of_readers();
}
int my_id() {
int my_mpi_id() {
int i;
MPI_Comm_rank(MPI_COMM_WORLD, &i);
return i;
}
int master_id(int m) {
return m;
size_t number_of(Role what) {
switch (what) {
case TOKENIZER:
return 1;
case FILTERER:
return 1;
case BATCHER:
return 1;
case SLAVE:
return world_size()
- number_of(TOKENIZER)
- number_of(FILTERER)
- number_of(BATCHER)
- number_of(MASTER);
case MASTER:
return 1;
}
}
int reader_id(int r) {
return r + number_of_masters();
int mpi_id_from_role_id(Role role, int rid) {
int base = 0;
for (Role r = TOKENIZER; r < role; r++) {
base += number_of(r);
}
return rid + base;
}
int slave_id(int s) {
return s + number_of_masters() + number_of_readers();
int role_id_from_mpi_id(Role role, int mid) {
int z = mpi_id_from_role_id(role, 0);
int rid = mid - z;
if (rid >= number_of(role) || rid < 0) {
INFO_PRINTF("%d is not a %d\n", mid, role);
exit(1);
}
return rid;
}
Role map_node() {
int node;
MPI_Comm_rank(MPI_COMM_WORLD, &node);
if (node >= reader_id(0) && node <= reader_id(number_of_readers()-1)) {
return DATA;
}
if (node >= master_id(0) && node <= master_id(number_of_masters()-1)) {
return MASTER;
}
if (node >= slave_id(0) && node <= slave_id(number_of_slaves()-1)) {
return SLAVE;
int node = my_mpi_id();
size_t base = 0;
for (Role r = TOKENIZER; r <= MASTER; r++) {
if (node < number_of(r) + base) return r;
base += number_of(r);
}
exit(1); // this is bad
}
int rid(int id, Role what) {
int z;
switch (what) {
case DATA: z = reader_id(0); break;
case SLAVE: z = slave_id(0); break;
case MASTER: z = master_id(0); break;
}
return id - z;
}
void free_weightlist(WeightList* wl) {
for in_range(i, wl->n_weights) {
free(wl->weights[i].shape);
@@ -135,27 +107,92 @@ void free_weightlist(WeightList* wl) {
free(wl->weights);
}
void data_reader() {
// Reads some data and converts it to a float array
INFO_PRINTF("Starting reader %d\n", getpid());
void free_word(Word* w) {
free(w->data);
w->data = NULL;
w->mem = 0;
}
size_t X_numel = 784 * BS;
size_t y_numel = 10 * BS;
float* X = malloc(X_numel * sizeof(float));
float* y = malloc(y_numel * sizeof(float));
void free_wordlist(WordList* wl) {
for in_range(i, wl->mem) {
free_word(wl->words + i);
}
free(wl->words);
wl->words = NULL;
wl->n_words = 0;
}
void send_word(Word* w, int dest) {
long len = strlen(w->data);
MPI_Send(&len, 1, MPI_LONG, dest, TAG_STLEN, MPI_COMM_WORLD);
MPI_Send(w->data, len + 1, MPI_CHAR, dest, TAG_SWORD, MPI_COMM_WORLD);
}
void recv_word(Word* w, int src) {
long len;
MPI_Recv(&len, 1, MPI_LONG, src, TAG_STLEN, MPI_COMM_WORLD,
MPI_STATUS_IGNORE);
if (w->mem < len + 1) {
w->mem = len + 1;
w->data = realloc(w->data, sizeof(char) * w->mem);
}
MPI_Recv(w->data, len + 1, MPI_CHAR, src, TAG_SWORD, MPI_COMM_WORLD,
MPI_STATUS_IGNORE);
}
void tokenizer(const char* source) {
WordList wl = {0, 0, NULL};
while (get_tokens(&wl, source)) {
for in_range(i, wl.n_words) {
send_word(&wl.words[i], mpi_id_from_role_id(FILTERER, 0));
// printf("OI %s\n", wl.words[i].data);
}
// INFO_PRINTLN("");
}
Word terminator = {0, ""};
send_word(&terminator, mpi_id_from_role_id(FILTERER, 0));
free_wordlist(&wl);
}
void filterer() {
Word w = {0, NULL};
while (1) {
recv_word(&w, role_id_from_mpi_id(TOKENIZER, 0));
if (!strlen(w.data)) {
break;
}
INFO_PRINTF("%s: ", w.data);
long idx = vocab_idx_of(&w);
INFO_PRINTF("%ld\n", idx);
// if (idx != -1) {
// MPI_Send(&idx, 1, MPI_LONG, mpi_id_from_role_id(BATCHER, 0),
// TAG_IWORD, MPI_COMM_WORLD);
// }
}
free_word(&w);
}
void batcher() {
// Reads some data and converts it to a float array
INFO_PRINTF("Starting batcher %d\n", getpid());
int s = 0;
const size_t n_words = BS + WIN + WIN;
float* f_widx = malloc(n_words * sizeof(float));
while (s != -1) {
for in_range(i, n_words) {
long l_wid;
MPI_Recv(&l_wid, 1, MPI_LONG, role_id_from_mpi_id(FILTERER, 0),
TAG_IWORD, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
f_widx[i] = (float)l_wid;
}
MPI_Recv(&s, 1, MPI_INT, MPI_ANY_SOURCE, TAG_READY, MPI_COMM_WORLD,
MPI_STATUS_IGNORE);
if (s != -1) {
mnist_batch(X, y, BS, rid(s, SLAVE), number_of_slaves());
MPI_Send(X, X_numel, MPI_FLOAT, s, TAG_BATCH, MPI_COMM_WORLD);
MPI_Send(y, y_numel, MPI_FLOAT, s, TAG_BATCH, MPI_COMM_WORLD);
MPI_Send(f_widx, n_words, MPI_FLOAT, s, TAG_BATCH, MPI_COMM_WORLD);
}
}
free(X);
free(y);
free(f_widx);
}
void send_weights(const WeightList* wl, int dest, int tag) {
@@ -191,34 +228,39 @@ void slave_node() {
// 3. Do computations
// 4. Send weights back to master
INFO_PRINTF("Starting slave %d\n", getpid());
int me = my_mpi_id();
int me;
MPI_Comm_rank(MPI_COMM_WORLD, &me);
size_t X_numel = 784 * BS;
size_t y_numel = 10 * BS;
float* X = malloc(X_numel * sizeof(float));
float* y = malloc(y_numel * sizeof(float));
PyObject* net = create_network();
PyObject* net = create_network(WIN, EMB);
WeightList wl;
init_weightlist_like(&wl, net);
size_t vocab = out_size(net);
size_t n_words = (BS + WIN + WIN);
size_t X_numel = BS * (WIN + WIN);
size_t y_numel = BS * vocab;
float* X = malloc(X_numel * sizeof(float));
float* y = malloc(y_numel * sizeof(float));
float* f_widx = malloc(n_words * sizeof(float));
for in_range(i, COMM) {
MPI_Send(&me, 1, MPI_INT, master_id(0), TAG_READY, MPI_COMM_WORLD);
recv_weights(&wl, master_id(0), TAG_WEIGH);
MPI_Send(&me, 1, MPI_INT, mpi_id_from_role_id(MASTER, 0),
TAG_READY, MPI_COMM_WORLD);
recv_weights(&wl, mpi_id_from_role_id(MASTER, 0), TAG_WEIGH);
set_net_weights(net, &wl);
for in_range(k, ITER) {
MPI_Send(&me, 1, MPI_INT, reader_id(0), TAG_READY, MPI_COMM_WORLD);
MPI_Recv(X, X_numel, MPI_FLOAT, reader_id(0), TAG_BATCH,
MPI_COMM_WORLD, MPI_STATUS_IGNORE);
MPI_Recv(y, y_numel, MPI_FLOAT, reader_id(0), TAG_BATCH,
MPI_COMM_WORLD, MPI_STATUS_IGNORE);
MPI_Send(&me, 1, MPI_INT, mpi_id_from_role_id(BATCHER, 0),
TAG_READY, MPI_COMM_WORLD);
MPI_Recv(f_widx, n_words, MPI_FLOAT,
mpi_id_from_role_id(BATCHER, 0), TAG_BATCH, MPI_COMM_WORLD,
MPI_STATUS_IGNORE);
c_slices(X, f_widx, BS, WIN);
c_onehot(y, f_widx + WIN, BS);
step_net(net, X, y, BS);
}
printf("%d net: %f\n", my_id(), eval_net(net));
printf("%d net: %f\n", my_mpi_id(), eval_net(net));
update_weightlist(&wl, net);
send_weights(&wl, master_id(0), TAG_WEIGH);
send_weights(&wl, mpi_id_from_role_id(MASTER, 0), TAG_WEIGH);
}
Py_DECREF(net);
free_weightlist(&wl);
@@ -232,12 +274,12 @@ void master_node() {
// 3. Average the weights
PyObject* frank = create_network();
PyObject* frank = create_network(WIN, EMB);
WeightList wl;
init_weightlist_like(&wl, frank);
update_weightlist(&wl, frank);
int spr = number_of_slaves() * FSPC; // Slaves per round
int spr = number_of(SLAVE) * FSPC; // Slaves per round
int s;
WeightList *wls = malloc(sizeof(WeightList) * spr);
@@ -265,33 +307,40 @@ void master_node() {
free_weightlist(&wl);
for in_range(i, spr) free_weightlist(wls + i);
free(wls);
if (rid(my_id(), MASTER) == 0) {
for in_range(r, number_of_readers()) {
int stop = -1;
MPI_Send(&stop, 1, MPI_INT, reader_id(r), TAG_READY,
MPI_COMM_WORLD);
}
}
// if (role_id_from_mpi_id(my_mpi_id(), MASTER) == 0) {
// for in_range(r, number_of(BATCHER)) {
// int stop = -1;
// MPI_Send(&stop, 1, MPI_INT, reader_id(r), TAG_READY,
// MPI_COMM_WORLD);
// }
// }
}
int main (int argc, const char **argv) {
MPI_Init(NULL, NULL);
// Cython Boilerplate
PyImport_AppendInittab("library", PyInit_library);
PyImport_AppendInittab("bridge", PyInit_bridge);
Py_Initialize();
PyRun_SimpleString("import sys\nsys.path.insert(0,'')");
PyObject* library_module = PyImport_ImportModule("library");
PyObject* bridge_module = PyImport_ImportModule("bridge");
// Actual Code
switch (map_node()) {
case DATA: data_reader(); break;
case SLAVE: slave_node(); break;
case MASTER: master_node(); break;
case TOKENIZER:
tokenizer(argv[1]);
break;
case FILTERER:
filterer();
break;
default:
INFO_PRINTLN("DYING HORRIBLY!");
// case SLAVE: slave_node(); break;
// case MASTER: master_node(); break;
}
// Finalizing Boilerplate
Py_DECREF(library_module);
Py_DECREF(bridge_module);
Py_Finalize();
MPI_Finalize();
}