main.cpp 30.2 KB
Newer Older
1
#include "octgrid.h"
Kirill Terekhov's avatar
Kirill Terekhov committed
2
#include "my_glut.h"
3
4
#include "rotate.h"
#include <math.h>
Kirill Terekhov's avatar
Kirill Terekhov committed
5
#include "inmost.h"
6
7
8
9
10

#include <iomanip>
#include <iostream>

#define LOG(level,msg)  { if (log_level >= level) cout << msg << endl; }
11
#define BARRIER MPI_Barrier(MPI_COMM_WORLD);
12
13
14
15
16
17
18
19

using namespace std;

struct grid thegrid;

// Variables for drawing
int draw_edges = 1;
int draw_faces = 1;
20
21
22
int draw_ghost = 1;
int draw_in_motion = 0; // redraw mode. true - always, false - by space press.
int draw_sem = 0;
SilverLife's avatar
SilverLife committed
23
int all_cells_count;
24
25
26
27
28
29
30

int cur_cell = 0; bool draw_all_cells = true;
int cur_face = 0; bool draw_all_faces = true;
int cur_edge = 0; bool draw_all_edges = true;

int from_file = 0;
int redist_after_amr = 0;
31
32

// Variables for refine and coarse
SilverLife's avatar
SilverLife committed
33
int refine_depth = 2;
34
35
double  mx = 0,  my = 0;
double rmx = 0, rmy = 0;
SilverLife's avatar
SilverLife committed
36
double base_radius = 0.01;
37
38
int action = 0;
int log_level = 0;
39
40
41
42
43
44
45
46
47
48
49
50
51
52

// Variables for MPI
int rank;
int size;
int counter = 0; // Count number of steps

// Variables for MPI and drawing
double recv_buff[2];
int wait_message = 0;
MPI_Request recv_req;
const int MAX_NODES_IN_FACE = 10;
const int MAX_PROCESSORS_COUNT = 128;
int current_proc_draw = -1;
void refresh_slaves_grid();
53
54
55
void send_coordinates_to_slaves(int action);

void redistribute_command();
56
57
58

struct drawing_face {
    int nodes_count;
59
    char stat;
60
61
62
63
    double** nodes;

    drawing_face()
    {
64
        stat = Element::Owned;
65
66
67
68
69
70
71
72
73
74
75
        nodes = new double*[MAX_NODES_IN_FACE];
        for (int i = 0; i < MAX_NODES_IN_FACE; i++)
            nodes[i] = new double[3];
    }
};

const int MAX_NODES_COUNT = 500000;
drawing_face** drawing_faces;
int*           drawing_faces_n;
double** sub_edges_nodes;
int*     sub_edges_nodes_n; 
76
char**    sub_edges_ghost;
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91

/// Dump mesh to vtk file in folder "grids"
void dump_to_vtk()
{
	//thegrid.mesh->ResolveShared(); // Resolve duplicate nodes
	//thegrid.mesh->ExchangeGhost(2,NODE); // Construct Ghost cells in 2 layers connected via nodes

    std::stringstream filename;
    filename << "grids/grid_";
    filename << size;
    if( size == 1 )
        filename << ".vtk";
    else
        filename << ".pvtk";
    thegrid.mesh->Save(filename.str());
Kirill Terekhov's avatar
Kirill Terekhov committed
92
	cout << "Process " << ::rank << ": dumped mesh to file" << endl;
93
94
}

SilverLife's avatar
SilverLife committed
95
96
97
98
99
100
101
102
void print_all_cells(grid* g)
{
    for(Mesh::iteratorCell it = thegrid.mesh->BeginCell(); it != thegrid.mesh->EndCell(); it++)
    {
        cout << ::rank << "-" << it->Integer(g->c_tags.base_id) << endl;
    }
}

103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
/// Function provided to octgrid algorithm. Defines transformation from grid to grid for draw.
void transformation(double xyz[3]) 
{
    double tmp[3];
	tmp[0] = xyz[0];
	tmp[1] = xyz[1];
	tmp[2] = xyz[2];
	tmp[0] -= 0.5;
	tmp[0] *= 100.0;
	tmp[1] -= 0.5;
	tmp[1] *= 100.0;
	xyz[0] =  tmp[0];
	xyz[1] =  tmp[1];
	//xyz[2] = 4010.0 + 10.0 * (tmp[2]-0.5);
	xyz[2] = 4010.0 + 10.0 * (tmp[2]-0.5);
}

/// Function provided to octgrid algorithm. Defines transformation from grid for draw to grid.
void rev_transformation(double xyz[3]) 
{
    double tmp[3];
	tmp[0] = xyz[0];
	tmp[1] = xyz[1];
	tmp[2] = xyz[2];
	tmp[0] /= 100.0;
	tmp[0] += 0.5;
	tmp[1] /= 100.0;
	tmp[1] += 0.5;
	xyz[0] =  tmp[0];
	xyz[1] =  tmp[1];
	xyz[2] = 4010.0 + 10.0 * (tmp[2]-0.5);
	xyz[2] = (tmp[2] - 4010.0) / 10.0 + 0.5;
}

/// Function provided to octgrid algorithm. 
/// Defines that cell should be unite. Returns 1 for unite else returns 0.
int cell_should_unite(struct grid * g, Cell cell)
{
	const double r = base_radius;
	int test = 1;	

    double x = cell.RealArrayDF(g->c_tags.center)[0];
    double y = cell.RealArrayDF(g->c_tags.center)[1];

	test &= (x-mx)*(x-mx)+(y-my)*(y-my) > r;
	return test;
}

/// Function provided to octgrid algorithm. 
/// Defines that cell should be split. Returns 1 for split else returns 0.
int cell_should_split(struct grid * g, Cell cell, int level)
{
	double r = base_radius;

    double x = cell.RealArrayDF(g->c_tags.center)[0]; 
    double y = cell.RealArrayDF(g->c_tags.center)[1];
    int c_level = cell.Integer(g->c_tags.level);

	if ((x-mx)*(x-mx)+(y-my)*(y-my) < r)
    {
        if (c_level < refine_depth) return 1;
    }

    for (int level = 2; level <= refine_depth; level++)
    {
	    if ((x-mx)*(x-mx)+(y-my)*(y-my) < r*5*(level-1))
            if (c_level < refine_depth - level + 1) return 1;
    }
    return 0;
}

#if defined( __GRAPHICS__)
int show_region = 0;
int width = 800, height = 600;

/// Function calls while mouse is moving
void motion(int nmx, int nmy) // Mouse
{
    rmx = nmx; rmy = nmy;
	mx = ((nmx/(double)(width))-0.5)*((double)width/(double)height)+0.5;
	my = (1.0 - nmy/(double)height);	
	
185
	if(draw_sem == 0 && draw_in_motion)
186
	{
187
188
189
190
191
        if (size > 0) send_coordinates_to_slaves(0);
        gridAMR(&thegrid, 0); 
        if (redist_after_amr) redistribute_command();
        if (size > 0) refresh_slaves_grid();   
        draw_sem = 1;
192
193
194
195
196
	}

	glutPostRedisplay();
}

197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
void set_color(int i)
{
    switch (i)
    {
        case 0: glColor3f(0.0, 0.0, 1.0); break;
        case 1: glColor3f(0.0, 1.0, 0.0); break;
        case 2: glColor3f(1.0, 0.0, 0.0); break;
        case 3: glColor3f(0.0, 1.0, 1.0); break;
        case 4: glColor3f(1.0, 0.0, 1.0); break;
        case 5: glColor3f(0.0, 0.5, 0.5); break;
        case 6: glColor3f(1.0, 1.0, 0.0); break;
        case 7: glColor3f(0.3, 0.8, 0.5); break;
        case 8: glColor3f(0.5, 0.0, 0.5); break;
        case 9: glColor3f(0.5, 0.5, 0.0); break;
        case 10: glColor3f(1.0, 1.0, 1.0); break;
    }
}

215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
/// Main drawing function. MPI version draws cells of each processor with unique color.
void mpi_draw()
{
	glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
	glLoadIdentity();

    glTranslated(0,0,4010);
    rotate();
    glTranslated(0,0,-4010);

    if (draw_faces)
    {
        // Draw our (main proc) faces
        if (current_proc_draw == -1 || current_proc_draw == 0)
        {
            glColor3f(0.0,   0.0   ,255.0);
231
232
            int c = 0;
            int cf = 0;
233
234
            for(Mesh::iteratorCell it = thegrid.mesh->BeginCell(); it != thegrid.mesh->EndCell(); it++)
            {
235
236
237
238
239
240
241
242
                if (it->GetStatus() == Element::Ghost)
                {
                    if (draw_ghost == 0) continue;
                    glColor3f(1.0,   0.0   ,0.0);
                }
                else if (it->GetStatus() == Element::Shared) glColor3f(1.0,   1.0   ,0.0);
                else                                         glColor3f(0.0,   0.0   ,1.0);

SilverLife's avatar
SilverLife committed
243
                
244

SilverLife's avatar
SilverLife committed
245
246
                if (!draw_all_cells && cur_cell % thegrid.mesh->NumberOfCells() != c++) continue;
                
247
248
249
250
                // Draw faces
                ElementArray<Face> faces = it->getFaces();
                for (ElementArray<Face>::iterator f = faces.begin(); f != faces.end(); f++) 
                {
251
252
253
254
255
256

                    if (f->GetStatus() == Element::Ghost)        glColor3f(0.4,   0.4   ,0.0);
                    else if (f->GetStatus() == Element::Shared)  glColor3f(1.0,   1.0   ,1.0);
                    else                                         glColor3f(0.0,   0.0   ,1.0);

                    if (!draw_all_faces && cur_face % faces.size() != cf++) continue;
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
                    ElementArray<Node> nodes = f->getNodes();
                    glBegin(GL_POLYGON);
                    for (ElementArray<Node>::iterator n = nodes.begin(); n != nodes.end(); n++)
                    {
                        glVertex3dv(&n->RealArray(thegrid.mesh->CoordsTag())[0]);
                    }
                    glEnd();
                }
            }
        }

        // Draw other (slaves procs) faces
        for (int i = 1; i < size; i++)
        {
            if (current_proc_draw != -1 && current_proc_draw != i) continue;
272
            set_color(i);
273
274
275
276
277
278
279
280
281
282
283
284
285
286

            if (i > 10 && i < 13)
            {
                double colo = 40.0 + 215.0*(double(i - 11)/double(size - 11));
                colo /= 255;
                glColor3f(colo, 0.3,0.3 );
            }
            if (i >= 13)
            {
                double colo = 40.0 + 215.0*(double(i - 13)/double(size - 13));
                colo /= 255;
                glColor3f(0.3,0.3,colo);
            }

287
            int cf = 0;
288
289
            for (int j = 1; j < drawing_faces_n[i]; j++)
            {
290
                if (!draw_all_faces && cur_face % drawing_faces_n[i] != cf++) continue;
291
292
293
                glBegin(GL_POLYGON);
                for (int k = 0; k < drawing_faces[i][j].nodes_count; k++)
                {
294
295
296
297
298
299
300
301
                    if (drawing_faces[i][j].stat == Element::Ghost) 
                    {
                        if (draw_ghost == 0) continue;
                        glColor3f(0.4,0.4,0);
                    }
                    else
                        set_color(i);

302
303
304
305
306
307
308
309
310
311
312
313
314
                    glVertex3dv((drawing_faces[i][j].nodes[k]));
                }
                glEnd();
                
            }
        }       
    }

	if (draw_edges)
	{
        // Draw our edges
        glColor3f(0.0f,0.0f,0.0f);
        
315
        if (current_proc_draw == -1 || current_proc_draw == 0)
316
        {
317
            int c = 0;
318
319
            for (Mesh::iteratorEdge f = thegrid.mesh->BeginEdge(); f != thegrid.mesh->EndEdge(); f++)
            {
320
321
322
323
324
325
326
                if (!draw_all_edges && cur_edge % thegrid.mesh->NumberOfEdges() != c++)  {  glEnable(GL_LINE_STIPPLE); glLineStipple(1, 0x0FF0); }
                else                                                                     {  glDisable(GL_LINE_STIPPLE);                          }

                glBegin(GL_LINES);
                if (f->GetStatus() == Element::Ghost)        glColor3f(1.0,   0.0   ,0.0);
                else if (f->GetStatus() == Element::Shared)  glColor3f(0.8,   0.8   ,0.8);
                else                                         glColor3f(0.0,   0.0   ,0.0);
327
328
329
                ElementArray<Node> nodes = f->getNodes();
                glVertex3dv(&nodes[0].RealArray(thegrid.mesh->CoordsTag())[0]);
                glVertex3dv(&nodes[1].RealArray(thegrid.mesh->CoordsTag())[0]);
330
                glEnd();
331
            }
332
333
334
        }
        // Draw other edges
        glLineWidth(2);
335
        glColor3f(0.0,   0.0   ,0.0);
336
337
        for (int i = 1; i < size; i++)
        {
338
            if (current_proc_draw != -1 && current_proc_draw != i) continue;
339
340
            int gc = 0;
            int c = 0;
341
342
            for (int j = 0; j < sub_edges_nodes_n[i]; j+=6)
            {
343
344
345
346
347
348
349
350
351
352
                 if (!draw_all_edges && cur_edge != c++)  { glEnable(GL_LINE_STIPPLE); glLineStipple(1, 0x0FF0); }
                 else { glDisable(GL_LINE_STIPPLE);  }
                 
                 glBegin(GL_LINES);
                 if (sub_edges_ghost[i][gc] == Element::Shared)
                    glColor3f(0.8,   0.8   ,0.8);
                 else if (sub_edges_ghost[i][gc] == Element::Ghost)
                    glColor3f(0.1,   0.0   ,0.0);
                 else
                    glColor3f(0.0,   0.0   ,0.0);
353
354
                glVertex3dv(&(sub_edges_nodes[i][j]));
                glVertex3dv(&(sub_edges_nodes[i][j+3]));
355
356
                gc++;
                glEnd();
357
358
            }
        }       
359
360
361
362
363
364
365
366
367
368
369
370
371
372

    }	
    glEnable(GL_POINT_SMOOTH);
    glBegin(GL_POINTS);
    for(Mesh::iteratorNode it = thegrid.mesh->BeginNode(); it != thegrid.mesh->EndNode(); it++)
    {
        if (it->GetStatus() == Element::Ghost) glColor3f(1, 1, 1);
        else if (it->GetStatus() == Element::Shared) glColor3f(1, 0.0, 0.0);
        else glColor3f(0.0, 0.0, 0.0);
        glVertex3dv(&(it->getAsNode().RealArray(thegrid.mesh->CoordsTag())[0]));
    }
    glEnd();
    glDisable(GL_POINT_SMOOTH);

373
    glutSwapBuffers();
374
    draw_sem = 0;
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
}

/// Main drawing function. 
void draw()
{
	glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
	glLoadIdentity();

	glTranslated(0,0,4010);
	rotate();
	glTranslated(0,0,-4010);

	if (draw_faces)
	{
		int i = 0;
		for (Mesh::iteratorFace f = thegrid.mesh->BeginFace(); f != thegrid.mesh->EndFace(); f++)
		{
			double c[3];
			ElementArray<Node> nodes = f->getNodes();
			glColor3f(0.0f,0.0f,1.0f);	
			glBegin(GL_POLYGON);
			for (ElementArray<Node>::iterator n = nodes.begin(); n != nodes.end(); n++)
				glVertex3dv(&n->RealArray(thegrid.mesh->CoordsTag())[0]);
			glEnd();
		}
	}
	
	if (draw_edges)
	{
        glLineWidth(2);
		glColor3f(0.0f,0.0f,0.0f);
		{
			glBegin(GL_LINES);
			for (Mesh::iteratorEdge f = thegrid.mesh->BeginEdge(); f != thegrid.mesh->EndEdge(); f++)
			{
				ElementArray<Node> nodes = f->getNodes();
				glVertex3dv(&nodes[0].RealArray(thegrid.mesh->CoordsTag())[0]);
				glVertex3dv(&nodes[1].RealArray(thegrid.mesh->CoordsTag())[0]);
			}
			glEnd();
		}
        glLineWidth(1);
	}	
 
    glutSwapBuffers();
420
    draw_sem = 0;
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
}


void reshape(int w, int h)
{
	double aspect = (double)w/(double)h;
	width = w;
	height = h;
	glMatrixMode (GL_PROJECTION);
	glLoadIdentity ();
	glOrtho(-50*aspect,50*aspect,-50,50,-5000,5000);
	glMatrixMode (GL_MODELVIEW);
	glViewport(0, 0, w, h);
}

/// Send mpuse coordinates to slaves. Slaves receive the message 'm', execute "refine" with received coordinates.
437
void send_coordinates_to_slaves(int action)
438
439
440
441
442
443
444
445
446
447
{
    char buff[MAX_PROCESSORS_COUNT][10];
    MPI_Request req[MAX_PROCESSORS_COUNT];

    for (int i = 1; i < size; i++)
    {
        LOG(2,"Main process: send coordinates to " << i)
        buff[i][0] = 'm'; // Special key, means refine 
        *((double*)(buff[i] + 1)) = mx;
        *((double*)(buff[i] + 1 + sizeof(double))) = my;
448
449
        *((int*)(buff[i] + 1 + 2*sizeof(double))) = action;
        MPI_Isend(buff[i], 1 + 2 * sizeof(double) + sizeof(int), MPI_CHAR, i, 0, INMOST_MPI_COMM_WORLD, req + i);
450
451
452
453
454
455
456
    }
}

/// OpenGL drawing execute only on one processor. In our case it's process with rank = 0 (Master). Every time when grid should be redraw, master sends
/// special command 'm' to slaves. Slave receive the message 'm', sends his cells to master.
void refresh_slaves_grid()
{
Kirill Terekhov's avatar
Kirill Terekhov committed
457
	if (::rank == 0) { // Send command to slaves for they will sent grid to us
458
        char buff1[10][2];
459
460
461
        MPI_Request req[10];
        for (int i = 1; i < size; i++)
        {
462
463
            buff1[i][0] = 'r'; // Special key, means send mesh
            MPI_Isend(buff1[i], 1, MPI_CHAR, i, 0, INMOST_MPI_COMM_WORLD, req + i);
464
465
466
467
468
469
470
471
472
        }
    }

    // Waiting response from slaves
    char buff[MAX_NODES_COUNT * sizeof(double)];
    MPI_Status status;
    int count_e;
    int count_df = 0;
    char nodes_count;
473
    char stat;
474
475
476
477
478

    for (int i = 1; i < size; i++)
    {
        // First receive array of vertices of faces
        MPI_Recv(buff, MAX_NODES_COUNT * sizeof(double),MPI_CHAR,i,0,INMOST_MPI_COMM_WORLD, &status);
479
480
        count_df = *((int*)buff);
        LOG(2,"Main process: received faces from " << i << " with " << count_df << " faces")
481
482

        drawing_faces_n[i] = count_df;
483
        int offset = sizeof(int);
484
485
486
        for (int j = 0; j < count_df; j++)
        {
            nodes_count = *(buff + offset++); // Read count of vertex in face
487
            stat = *(buff + offset++);
488
            drawing_faces[i][j].nodes_count = nodes_count;
489
            drawing_faces[i][j].stat = stat;
490
491
492
493
494
495
496
497
498
499
500
501
            for (int k = 0; k < nodes_count; k++)
            {
                drawing_faces[i][j].nodes[k][0] = *((double*)(buff + offset)); 
                drawing_faces[i][j].nodes[k][1] = *((double*)(buff + offset + sizeof(double)  )); 
                drawing_faces[i][j].nodes[k][2] = *((double*)(buff + offset + sizeof(double)*2)); 
                offset += sizeof(double)*3;
            }
        }

        // Now receive array of vertices of edges
        MPI_Recv(buff, MAX_NODES_COUNT * sizeof(double),MPI_CHAR,i,0,INMOST_MPI_COMM_WORLD, &status);
        count_e = *((int*)buff);
502
        sub_edges_nodes_n[i] = (count_e/7)*6;
503
504
        LOG(2,"Main process: received edges from " << i << " with " << count_e << " doubles")

505
506
507
        int c = 0;
        int gc = 0;
        while (c+gc < count_e)
508
        {
509
510
511
512
513
514
515
            sub_edges_ghost[i][gc] = *((double*)(buff + sizeof(int) + sizeof(double)*(c+gc)));
            gc++;
            for (int j = 0; j < 6 ; j++)
            {
                sub_edges_nodes[i][c] = *((double*)(buff + sizeof(int) + sizeof(double)*(c+gc))); 
                c++;
            }
516
517
518
519
520
521
        }
    }
}

void prepare_to_correct_brothers()
{
Kirill Terekhov's avatar
Kirill Terekhov committed
522
	correct_brothers(&thegrid,size,::rank, 0);
523
524
525
526
527
528
    thegrid.mesh->RemoveGhost();
    thegrid.mesh->Redistribute(); 
    thegrid.mesh->ReorderEmpty(CELL|FACE|EDGE|NODE);
    thegrid.mesh->AssignGlobalID(CELL | EDGE | FACE | NODE);
}

529
/// Redistribute grid by  partitioner
530
531
void redistribute(int type)
{
SilverLife's avatar
SilverLife committed
532
533
    thegrid.mesh->RemoveGhost();

Kirill Terekhov's avatar
Kirill Terekhov committed
534
	LOG(2,"Process " << ::rank << ": redistribute. Cells: " << thegrid.mesh->NumberOfCells())
535
536
537
    Partitioner * part = new Partitioner(thegrid.mesh);
    
    // Specify the partitioner
SilverLife's avatar
SilverLife committed
538
    type = 1;
539
540
541
    if (type == 0) part->SetMethod(Partitioner::Parmetis, Partitioner::Partition);
    if (type == 1) part->SetMethod(Partitioner::Parmetis, Partitioner::Repartition);
    if (type == 2) part->SetMethod(Partitioner::Parmetis, Partitioner::Refine);
542
543
544
545
546
547
548
549
550
    
    try
    {
        part->Evaluate();
    }
    catch (INMOST::ErrorType er)
    {
        cout << "Exception: " << er << endl;
    }
551
552
553
    catch(...)
    {
    }
554
555
    delete part;

SilverLife's avatar
SilverLife committed
556
    thegrid.mesh->RemoveGhost();
Kirill Terekhov's avatar
Kirill Terekhov committed
557
	correct_brothers(&thegrid,size,::rank, 2);
558

559
560
561
562
563
564
565
566
    try
    {
        thegrid.mesh->Redistribute(); 
    }
    catch (INMOST::ErrorType er)
    {
        cout << "Exception: " << er << endl;
    }
567
568
569
570
571
    catch(...)
    {
    }


572
573
    thegrid.mesh->RemoveGhost();
    thegrid.mesh->ReorderEmpty(CELL|FACE|EDGE|NODE);
574
    //thegrid.mesh->AssignGlobalID(CELL | EDGE | FACE | NODE);
Kirill Terekhov's avatar
Kirill Terekhov committed
575
	LOG(2,"Process " << ::rank << ": redistribute completed")
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
}

/// Prepare to redistribute. Master sends special command to slaves which means redistribute
void redistribute_command()
{
    char buff[MAX_PROCESSORS_COUNT][10];
    MPI_Request req[MAX_PROCESSORS_COUNT];
    int type = 2;
    
    if (counter == 5)
    {
        counter = 0;
        type = 1;
    }
   
    for (int i = 1; i < size; i++)
    {
Kirill Terekhov's avatar
Kirill Terekhov committed
593
		LOG(3,"Master: send redistribute command to slave " << ::rank)
594
595
596
597
598
599
600
        buff[i][0] = 'x'; // Special key, means redistribute
        buff[i][1] = type + '0'; // Special key, means redistribute
        MPI_Isend(buff[i], 2, MPI_CHAR, i, 0, INMOST_MPI_COMM_WORLD, req + i);
    }       
    
    redistribute(type);
    counter++;
601

602
603
604
605
606
607
608
609
610
611
}


void keyboard(unsigned char key, int x, int y)
{
	(void) x, (void) y;
	if( key == 27 )
	{
		exit(-1);
	}
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
    if( key == '+')
    {
        cur_cell++;
		glutPostRedisplay();
    }
    if( key == '-')
    {
        cur_cell--;
		glutPostRedisplay();
    }
    if( key == 'q')
    {
        draw_all_cells = !draw_all_cells;
		glutPostRedisplay();
    }
    if( key == 'a')
    {
        draw_all_faces = !draw_all_faces;
		glutPostRedisplay();
    }
    if( key == 'w')
    {
        draw_all_edges = !draw_all_edges;
		glutPostRedisplay();
    }
    if( key == '6')
    {
        cur_edge++;
		glutPostRedisplay();
    }
    if( key == '4')
    {
        cur_edge--;
		glutPostRedisplay();
    }
    if( key == '>')
    {
        cur_face++;
		glutPostRedisplay();
    }
    if( key == '<')
    {
        cur_face--;
		glutPostRedisplay();
    }
    if( key == 'g' || key == 'G')
    {
        draw_ghost = !draw_ghost;
		glutPostRedisplay();
    }
    if( key == 'o' || key == 'O')
    {
        if (action == 0) action = 3;
        else action = 0;
		glutPostRedisplay();
    }
    if( key == 'p' || key == 'P')
    {
        redist_after_amr = !redist_after_amr; 
        draw_in_motion = !draw_in_motion;
		glutPostRedisplay();
    }
    if( key == 'c' || key == 'C')
    {
        char buff[10][1];
        MPI_Request req[10];
        for (int i = 1; i < size; i++)
        {
            buff[i][0] = 'c'; // Special key, means command
            MPI_Isend(buff[i], 1, MPI_CHAR, i, 0, INMOST_MPI_COMM_WORLD, req + i);
        } 
        command(&thegrid);
        
		glutPostRedisplay();
    }
    if( key == 'u' || key == 'U')
    {
        char buff[10][1];
        MPI_Request req[10];
        for (int i = 1; i < size; i++)
        {
            buff[i][0] = 'u'; // Special key, means remove_ghost
            MPI_Isend(buff[i], 1, MPI_CHAR, i, 0, INMOST_MPI_COMM_WORLD, req + i);
        } 
        thegrid.mesh->RemoveGhost();
		glutPostRedisplay();
    }
699
700
	if( key == ' ' ) 
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
701
		if (::rank == 0)
702
        {
703
            send_coordinates_to_slaves(action);
704
705
		}

706
707
708
709
        gridAMR(&thegrid, action);
        //thegrid.mesh->AssignGlobalID(CELL | EDGE | FACE | NODE);
        if (redist_after_amr)
            redistribute_command();
710
711
712
713
		glutPostRedisplay();
	}
    if( key == '[' ) 
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
714
		if (::rank == 0)
715
        {
716
            send_coordinates_to_slaves(action);
717
718
719
		}

        gridAMR(&thegrid, 1);
720
        //thegrid.mesh->AssignGlobalID(CELL | EDGE | FACE | NODE);
721
722
723
724
725
726
727
728
		glutPostRedisplay();
	}
    if( key == 'r' || key == 'R'|| key == ' ') 
    {
        refresh_slaves_grid();   
    }
    if( key == 'f' ) 
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
729
		if (::rank == 0) { // Send dump command to other process
730
731
732
733
734
735
736
737
738
739
740
741
742
743
            dump_to_vtk();
            char buff[10][2];
            MPI_Request req[10];
            for (int i = 1; i < size; i++)
            {
                buff[i][0] = 'f'; // Special key, means dump file
                MPI_Isend(buff[i], 1, MPI_CHAR, i, 0, INMOST_MPI_COMM_WORLD, req + i);
            }
        }
		glutPostRedisplay();
	}
    if( key == 'x' || key == 'X' )
    {
        redistribute_command();
744
745
        refresh_slaves_grid();   
	glutPostRedisplay();
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
    }
    if( key == 'z' || key == 'Z' )
    {
        char buff[10][10];
        MPI_Request req[10];

        for (int i = 1; i < size; i++)
        {
            buff[i][0] = 'z'; // Special key, means correct_brothers
            MPI_Isend(buff[i], 1, MPI_CHAR, i, 0, INMOST_MPI_COMM_WORLD, req + i);
        }       
        prepare_to_correct_brothers();
    }
	if( key == '*' || key == '8' )
    {
        current_proc_draw++;
        if (current_proc_draw == size) current_proc_draw = -1;
	    glutPostRedisplay();
    }
	if( key == '/' || key == '?' )
    {
        current_proc_draw--;
        if (current_proc_draw == -2) current_proc_draw = size - 1;
	    glutPostRedisplay();
    }
	if( key == 'e' || key == 'E' )
	{
        draw_edges = !draw_edges;
	}
775
776
777
778
	if( key == 't' || key == 'T' )
	{
        draw_faces = !draw_faces;
	}
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
    if( key == 'i' || key == 'I' )
    {
        draw_in_motion = !draw_in_motion ;
    }
}

void idle(void)
{
}
#endif

/// Main loop for not main process
void NotMainProcess()
{
    char buff[256];
    MPI_Status status;
    int length;

    while (1) {
        MPI_Recv(buff, 256, MPI_CHAR, 0,0, INMOST_MPI_COMM_WORLD, &status);

Kirill Terekhov's avatar
Kirill Terekhov committed
800
		LOG(2, "Process " << ::rank << ": received message '" << buff[0] << "'")
801
802
803
804
805
        
        if (buff[0] == 'm') // Need to refine, mouse coordinates come
        {
            mx = *((double*)(buff + 1));
            my = *((double*)(buff + 1 + sizeof(double)));
806
807
808
809
            int action = *((int*)(buff + 1 + 2*sizeof(double)));

            gridAMR(&thegrid, action); 
            //thegrid.mesh->AssignGlobalID(CELL | EDGE | FACE | NODE);
810
811
812
813
814
815
816
        }
        if (buff[0] == 'x') // Need to redistribute 
        {
            char type_C = buff[1];
            int  type = type_C - '0';
            redistribute(type);
        }
817
818
819
820
821
822
823
824
        if (buff[0] == 'u') // Need remove ghosts
        {
            thegrid.mesh->RemoveGhost();
        }
        if (buff[0] == 'c') // Need command
        {
            command(&thegrid);
        }
825
826
827
828
829
830
831
832
833
834
835
836
837
838
        if (buff[0] == 'z') // Need to correct brothers 
        {
            prepare_to_correct_brothers();
        }
        if (buff[0] == 'f') // Dump mesh to file
        {
            dump_to_vtk();
        }
        if (buff[0] == 'r') // Master wants to redraw mesh. Now this procces need send own grid to master.
        {
            // First fill the buffer for sent
            char buff_f[MAX_NODES_COUNT * sizeof(double)];
            char buff_e[MAX_NODES_COUNT * sizeof(double)];
            int count_e = 0;
839
            int offset = sizeof(int);
840
841
842
            int count_df = 0; // Number of transmitted faces in array buff_f
           
            // Fill array buff_f. Strcuture of array:
843
            // [Faces count] [Vertices' count in face] [Status] [Points] [Vertices' count in face] [Status] [Points]...
844
845
846
847
848
849
            for(Mesh::iteratorCell it = thegrid.mesh->BeginCell(); it != thegrid.mesh->EndCell(); it++)
            {
                ElementArray<Face> faces = it->getFaces();
                for(ElementArray<Face>::iterator f = faces.begin(); f != faces.end(); f++) 
                {
                    count_df++;
850

851
                    ElementArray<Node> nodes = f->getNodes();
852
853
854
855
856
                    if (nodes.size() == 0) continue;

                    *(buff_f + offset++) = (char)nodes.size();
                    *(buff_f + offset++) = (char)it->GetStatus();

857
858
859
860
861
862
863
864
865
                    for(ElementArray<Node>::iterator n = nodes.begin(); n != nodes.end(); n++)
                    {
                        *((double*)(buff_f + offset                   )) = n->RealArray(thegrid.mesh->CoordsTag())[0];
                        *((double*)(buff_f + offset + sizeof(double)  )) = n->RealArray(thegrid.mesh->CoordsTag())[1];
                        *((double*)(buff_f + offset + sizeof(double)*2)) = n->RealArray(thegrid.mesh->CoordsTag())[2];
                        offset  += 3*sizeof(double);
                    }
                }
            }
866
            *((int*)buff_f) = count_df;
867
868
869
870
871

            // Fill array buff_e
            for(Mesh::iteratorEdge f = thegrid.mesh->BeginEdge(); f != thegrid.mesh->EndEdge(); f++)
			{
                ElementArray<Node> nodes = f->getNodes();
872
				*((double*)(buff_e + sizeof(int) + sizeof(double)*count_e++)) = (double)f->GetStatus();
873
874
875
876
877
878
879
880
881
882
				*((double*)(buff_e + sizeof(int) + sizeof(double)*count_e++)) = nodes[0].RealArray(thegrid.mesh->CoordsTag())[0];
				*((double*)(buff_e + sizeof(int) + sizeof(double)*count_e++)) = nodes[0].RealArray(thegrid.mesh->CoordsTag())[1];
				*((double*)(buff_e + sizeof(int) + sizeof(double)*count_e++)) = nodes[0].RealArray(thegrid.mesh->CoordsTag())[2];
				*((double*)(buff_e + sizeof(int) + sizeof(double)*count_e++)) = nodes[1].RealArray(thegrid.mesh->CoordsTag())[0];
				*((double*)(buff_e + sizeof(int) + sizeof(double)*count_e++)) = nodes[1].RealArray(thegrid.mesh->CoordsTag())[1];
				*((double*)(buff_e + sizeof(int) + sizeof(double)*count_e++)) = nodes[1].RealArray(thegrid.mesh->CoordsTag())[2];
			}
            *((int*)buff_e) = count_e;

            // Now we are ready to transmit the data. Transmit with 2 steps
883
			LOG(2, "Process " << ::rank << ": send buffer with faces = " << count_df << ". Edges " << count_e)
884
885
886
887
888
889
            MPI_Send(buff_f, offset                             ,MPI_CHAR,0,0,INMOST_MPI_COMM_WORLD);
            MPI_Send(buff_e,count_e*sizeof(double) + sizeof(int),MPI_CHAR,0,0,INMOST_MPI_COMM_WORLD);
        }
    }
}

890
void parse_arguments(int argc, char** argv, int* n, double* R, int* L)
SilverLife's avatar
SilverLife committed
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
{
  if (argc < 2) return;

  string str;
  string str1, str2;
  for (int i = 1; i < argc; i++)
  {
    str = argv[i];
    size_t pos = str.find('=');
    if (pos == string::npos)
    {
      cout << "Invalid argument: " << str << endl;
      continue;
    }

    str1 = str.substr(0, pos);
    str2 = str.substr(pos + 1);

    if (str1 == "-n")
    {
      for (int j = 0; j < 3; j++)
      {
        pos = str2.find('x');
        if (j < 2 && pos == string::npos) 
        {
          cout << "Invalid command: " << str2 << endl;
          break;
        }
        str1 = str2.substr(0, pos);
        n[j] = atoi(str1.c_str());
        str2 = str2.substr(pos + 1);
      }
    }
    else if (str1 == "-r")
    {

927
928
929
930
931
	    *R = atof(str2.c_str());
    }
    else if (str1 == "-l")
    {
	    *L = atoi(str2.c_str());
SilverLife's avatar
SilverLife committed
932
933
934
935
936
937
938
939
940
941
942
943
    }
    else
    {
      cout << "Invalid command: " << str1 << endl;
    }
  }
}

void print_help()
{
  cout << "Example of Octree refine on redistributed grid" << endl;
  cout << "Command arguments:" << endl;
944
945
946
  cout << "   -n=10x10x1 - grid size" << endl;
  cout << "   -r=0.01    - refine radius" << endl;
  cout << "   -l=2       - refine level"  << endl;
SilverLife's avatar
SilverLife committed
947
948
949
  cout << endl;
  cout << "Hotkeys:" << endl;
  cout << "   Space - refine grid around mouse cursor" << endl;
950
  //cout << "       r - redraw grid" << endl;
SilverLife's avatar
SilverLife committed
951
952
953
954
955
  cout << "       f - dump grid to file (see grids folder)" << endl;
  cout << "       x - redistribute grid" << endl;
}


956
957
958
int main(int argc, char ** argv)
{
	int i;
959
	int n[3] = {10,10,1};
960

SilverLife's avatar
SilverLife committed
961

962
963
964
965
966
	thegrid.transformation = transformation;
	thegrid.rev_transformation = rev_transformation;
	thegrid.cell_should_split = cell_should_split;
	thegrid.cell_should_unite = cell_should_unite;
    Mesh::Initialize(&argc,&argv);
Kirill Terekhov's avatar
Kirill Terekhov committed
967
	MPI_Comm_rank(MPI_COMM_WORLD, &::rank);
968

SilverLife's avatar
SilverLife committed
969
    if (::rank == 0) print_help();
970
    parse_arguments(argc, argv, n, &base_radius,&refine_depth);
SilverLife's avatar
SilverLife committed
971
972
    all_cells_count = n[0]*n[1]*n[2] * 2;

973
974
975
    gridInit(&thegrid,n);
    Partitioner::Initialize(&argc,&argv);

Kirill Terekhov's avatar
Kirill Terekhov committed
976
977
	::size = thegrid.mesh->GetProcessorsNumber();
	::rank = thegrid.mesh->GetProcessorRank();
978
979
980

    //dump_to_vtk();

Kirill Terekhov's avatar
Kirill Terekhov committed
981
	if (::rank != 0)
982
983
984
985
986
987
988
989
990
991
992
    {
        NotMainProcess();
    }
    else
    {

        #if defined(USE_MPI)
        drawing_faces     = new drawing_face*[size-1];
        drawing_faces_n   = new int[size-1];
        sub_edges_nodes   = new double*[size-1];
        sub_edges_nodes_n = new int[size-1];
993
994
        sub_edges_ghost   = new char*[size-1];
        
995
996
997
998
        for (int i = 0; i < size; i++) {
            drawing_faces_n[i] = 0;
            drawing_faces[i] = new drawing_face[30000];
            sub_edges_nodes_n[i] = 0;
999
            sub_edges_ghost[i] = new char[25000];
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
            sub_edges_nodes[i] = new double[75000];
        }
        #endif

        quatinit();
        glutInit(&argc,argv);
        glutInitDisplayMode(GLUT_DOUBLE | GLUT_RGBA | GLUT_DEPTH);
        glutInitWindowSize(width, height);
        glutInitWindowPosition (100, 100);
        glutCreateWindow("Graph");

        glDepthFunc(GL_LEQUAL);
        glClearDepth(1.f);
        glEnable(GL_DEPTH_TEST);
        //glPolygonMode(GL_FRONT_AND_BACK, GL_LINE);

        glHint(GL_POLYGON_SMOOTH_HINT,GL_NICEST);
        glHint(GL_LINE_SMOOTH_HINT,GL_NICEST);
        glHint(GL_PERSPECTIVE_CORRECTION_HINT,GL_NICEST);
        glHint(GL_POINT_SMOOTH_HINT,GL_NICEST);

        glClearColor (1.0f, 1.0f, 1.0f, 1.f);
        if (size > 1) glutDisplayFunc(mpi_draw);
        else glutDisplayFunc(draw);
        glutReshapeFunc(reshape);

        glutKeyboardFunc(keyboard);
        glutMotionFunc(rotate_clickmotion);
        glutMouseFunc(rotate_click);
        //glutPassiveMotionFunc(rotate_motion);
        glutPassiveMotionFunc(motion);
        glutIdleFunc(idle);

        glutPostRedisplay();
        glutMainLoop();
    }
}