parallel.cpp 277 KB
Newer Older
Kirill Terekhov's avatar
Kirill Terekhov committed
1
#ifdef _MSC_VER //kill some warnings
2
#ifndef _SCL_SECURE_NO_WARNINGS
Kirill Terekhov's avatar
Kirill Terekhov committed
3
4
#define _SCL_SECURE_NO_WARNINGS
#endif
5
#endif
Kirill Terekhov's avatar
Kirill Terekhov committed
6

Kirill Terekhov's avatar
Kirill Terekhov committed
7
8
9
10
11
12
13
#include "inmost.h"


#if defined(USE_MESH)
#include <iostream>
#include <fstream>
#include <sstream>
14
#include <string>
Kirill Terekhov's avatar
Kirill Terekhov committed
15
#include <queue>
Kirill Terekhov's avatar
Kirill Terekhov committed
16

Kirill Terekhov's avatar
Kirill Terekhov committed
17
//using namespace std;
Kirill Terekhov's avatar
Kirill Terekhov committed
18

Kirill Terekhov's avatar
fix    
Kirill Terekhov committed
19
#if defined(USE_PARALLEL_STORAGE)
20
bool allow_pack_by_gids = true;
Kirill Terekhov's avatar
fix    
Kirill Terekhov committed
21
22
23
24
#else // USE_PARALLEL_STORAGE
bool allow_pack_by_gids = false;
#endif // USE_PARALLEL_STORAGE

25

Kirill Terekhov's avatar
Kirill Terekhov committed
26
27
#if defined(USE_MPI)
static INMOST_DATA_BIG_ENUM_TYPE pmid = 0;
Kirill Terekhov's avatar
fix    
Kirill Terekhov committed
28
#endif // USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
29

30
31
32
33
34
35
36
37
static std::string NameSlash(std::string input)
{
	for(unsigned l = input.size(); l > 0; --l)
		if( input[l-1] == '/' || input[l-1] == '\\' )
			return std::string(input.c_str() + l);
	return input;
}

Kirill Terekhov's avatar
Kirill Terekhov committed
38
39
40
41
#if defined(USE_PARALLEL_WRITE_TIME)
#define REPORT_MPI(x) {WriteTab(out_time) << "<MPI><![CDATA[" << #x << "]]></MPI>" << std::endl; x;}
#define REPORT_STR(x) {WriteTab(out_time) << "<TEXT><![CDATA[" << x << "]]></TEXT>" << std::endl;}
#define REPORT_VAL(str,x) {WriteTab(out_time) << "<VALUE name=\"" << str << "\"> <CONTENT><![CDATA[" << x << "]]></CONTENT> <CODE><![CDATA[" << #x << "]]></CODE></VALUE>" << std::endl;}
Kirill Terekhov's avatar
Kirill Terekhov committed
42
#define ENTER_FUNC() double all_time = Timer(); {WriteTab(out_time) << "<FUNCTION name=\"" << __FUNCTION__ << "\" id=\"func" << func_id++ << "\">" << std::endl; Enter();}
43
44
45
#define ENTER_BLOCK() { double btime = Timer(); WriteTab(out_time) << "<FUNCTION name=\"" << __FUNCTION__ << ":" << NameSlash(__FILE__) << ":" << __LINE__ << "\" id=\"func" << GetFuncID()++ << "\">" << std::endl; Enter();
#define EXIT_BLOCK() WriteTab(out_time) << "<TIME>" << Timer() - btime << "</TIME>" << std::endl; Exit(); WriteTab(out_time) << "</FUNCTION>" << std::endl;}

Kirill Terekhov's avatar
Kirill Terekhov committed
46
47
#define EXIT_FUNC() {WriteTab(out_time) << "<TIME>" << Timer() - all_time << "</TIME>" << std::endl; Exit(); WriteTab(out_time) << "</FUNCTION>" << std::endl;}
#define EXIT_FUNC_DIE() {WriteTab(out_time) << "<TIME>" << -1 << "</TIME>" << std::endl; Exit(); WriteTab(out_time) << "</FUNCTION>" << std::endl;}
Kirill Terekhov's avatar
fix    
Kirill Terekhov committed
48
#else // USE_PARALLEL_WRITE_TIME
Kirill Terekhov's avatar
Kirill Terekhov committed
49
#define REPORT_MPI(x) x
Kirill Terekhov's avatar
Kirill Terekhov committed
50
51
52
#define REPORT_STR(x) {}
#define REPORT_VAL(str,x) {}
#define ENTER_FUNC() {}
53
54
#define ENTER_BLOCK()
#define EXIT_BLOCK()
Kirill Terekhov's avatar
Kirill Terekhov committed
55
56
#define EXIT_FUNC() {}
#define EXIT_FUNC_DIE()  {}
Kirill Terekhov's avatar
fix    
Kirill Terekhov committed
57
#endif // USE_PARALLEL_WRITE_TIME
Kirill Terekhov's avatar
Kirill Terekhov committed
58

Kirill Terekhov's avatar
Kirill Terekhov committed
59
60
61
62
63
64
65
66

#if defined(__LINUX__) || defined(__linux__) || defined(__APPLE__)
#include <unistd.h>
#define PROCESSID getpid()
#else
#define PROCESSID -1
#endif

Kirill Terekhov's avatar
Kirill Terekhov committed
67
68
namespace INMOST
{
69
    //static int block_recursion = 0;
Kirill Terekhov's avatar
Kirill Terekhov committed
70
	/*
71
72
73
    std::string ro()
    {
        int rank = 0;
74
#ifdef USE_MPI
75
        MPI_Comm_rank(MPI_COMM_WORLD,&rank);
76
#endif
77
78
79
80
81
        std::stringstream ss;
        for (int i = 0; i < rank; i++)
            ss << "   ";
        return ss.str();
    }
Kirill Terekhov's avatar
Kirill Terekhov committed
82
    */
Kirill Terekhov's avatar
Fixes    
Kirill Terekhov committed
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
	//////////////////////////////
	/// REDUCTION FUNCTIONS    ///
	//////////////////////////////
	void DefaultUnpack(const Tag & tag, const Element & element, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
	{
		if( size == 0 )
		{
			if( tag.isDefined(element->GetElementType()) ) 
			{
				if( tag.isSparse(element->GetElementType()) )
				{
					if( element->HaveData(tag) )
						element->DelData(tag); 
				}
				else if( tag.GetSize() == ENUMUNDEF )
					element->SetDataSize(tag,size);
				else
				{
					std::cerr << "received zero size for dense nonzero tag" << std::endl;
					assert(false);
				}
			}
			return;
		}
		if( !element->HaveData(tag) )
			element->SetDataSize(tag,size);
		else if( size != element->GetDataSize(tag) )
		{
			if( tag.GetSize() == ENUMUNDEF )
				element->SetDataSize(tag,size);
			else
			{
				assert(false);
			}
		}
		element->SetData(tag,0,size,data);
	}


	void UnpackOnSkin(const Tag & tag, const Element & e, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
	{
		if( size )
		{
126
			const Storage::integer * recv = static_cast<const Storage::integer *>(static_cast<const void *>(data));
Kirill Terekhov's avatar
Fixes    
Kirill Terekhov committed
127
			Storage::integer_array arr = e->IntegerArray(tag);
128
			//for(int k = 0; k < size; ++k)
Kirill Terekhov's avatar
Fixes    
Kirill Terekhov committed
129
130
131
132
133
134
135
136
137
			arr.push_back(recv[0]);
		}
	}


	void UnpackSkin(const Tag & tag, const Element & e, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
	{
		if( size )
		{
Kirill Terekhov's avatar
Kirill Terekhov committed
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
			for(INMOST_DATA_ENUM_TYPE k = 0; k < size; k+=2 )
			{
				bool flag = true;
				const Storage::integer * recv = static_cast<const Storage::integer *>(static_cast<const void *>(data))+k;
				Storage::integer_array arr = e->IntegerArray(tag);
				for(Storage::integer_array::iterator it = arr.begin(); it != arr.end(); it+=2)
				{
					if( *it == recv[0] )
					{
						flag = false;
						break;
					}
				}
				if( flag ) 
				{
					arr.push_back(recv[0]);
					arr.push_back(recv[1]);
				}
			}
Kirill Terekhov's avatar
Fixes    
Kirill Terekhov committed
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
		}
	}
	


	void DeleteUnpack(const Tag & tag, const Element & e, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
	{
		if( size ) 
		{
			int old_size;
			if( e->HaveData(tag) ) old_size = e->GetDataSize(tag);
			else old_size = 0;
			e->SetDataSize(tag,old_size+size);
			e->SetData(tag,old_size,size,data);
		}
	}
	
	void RedistUnpack(const Tag & tag,const Element & e, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
	{
		if( size ) 
		{
			Storage::integer_array p1 = e->IntegerArray(tag);
			const Storage::integer * p2 = static_cast<const Storage::integer *>(static_cast<const void *>(data));
			dynarray<Storage::integer,64> result(p1.size()+size);
181
182
			result.resize(std::set_union(p1.begin(),p1.end(),p2,p2+size,result.begin())-result.begin());
			p1.replace(p1.begin(),p1.end(),result.begin(),result.end());
Kirill Terekhov's avatar
Fixes    
Kirill Terekhov committed
183
184
185
186
187
188
189
190
191
192
193
194
195
196
		}
	}

	void UnpackLayersMarker(const Tag & tag, const Element & e, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
	{
		if( size )
		{
			int old_size;
			if( e->HaveData(tag) ) old_size = e->GetDataSize(tag);
			else old_size = 0;
			e->SetDataSize(tag,old_size+size);
			e->SetData(tag,old_size,size,data);
		}
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
197
/*
198
199
    void Mesh::CheckFaces()
    {
Kirill Terekhov's avatar
Kirill Terekhov committed
200
        //std::cout << "Check faces" << endl;
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216

        for(Mesh::iteratorFace it = BeginFace(); it != EndFace(); ++it) 
        {
            std::set<int> set_nodes;
            ElementArray<Node> nodes = it->getNodes();
            bool suc = true;
            for (ElementArray<Node>::iterator node = nodes.begin(); node != nodes.end(); node++)
                if (set_nodes.find(node->LocalID()) != set_nodes.end())
                {
                    suc = false;
                    break;
                }
                else
                {
                    set_nodes.insert(node->LocalID());
                }
Kirill Terekhov's avatar
Kirill Terekhov committed
217
218
            //if (suc) std::cout << "-=== Good face: " << setw(2) << it->LocalID();
            //else std::cout << "-=== Error face: " << setw(2) << it->LocalID();
219

Kirill Terekhov's avatar
Kirill Terekhov committed
220
221
222
223
            //cout << ". Nodes = " << nodes.size() << ": ";
            //for (ElementArray<Node>::iterator node = nodes.begin(); node != nodes.end(); node++)
             //   std::cout << node->LocalID() << " ";
            //std::cout << endl;
224
225
        }
    }
Kirill Terekhov's avatar
Kirill Terekhov committed
226
*/
227

228
	static void OperationMinDistance(const Tag & tag, const Element & element, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
Kirill Terekhov's avatar
Kirill Terekhov committed
229
230
231
232
233
234
235
236
237
238
	{
		(void)size;
		assert(size == 2);
		INMOST_DATA_INTEGER_TYPE * idata = (INMOST_DATA_INTEGER_TYPE *)data;
		Storage::integer_array rdata = element->IntegerArray(tag);
		if (idata[1] < rdata[1])
		{
			rdata[0] = idata[0];
			rdata[1] = idata[1];
		}
239
240
241
242
243
		else if(idata[1] == rdata[1] && idata[0] < rdata[0])
		{
			rdata[0] = idata[0];
			rdata[1] = idata[1];
		}
Kirill Terekhov's avatar
Kirill Terekhov committed
244
	}
245
246
247
248
249
250
251
252
253
	
	static void OperationMinOwner(const Tag & tag, const Element & element, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
	{
		(void)size;
		assert(size == 1);
		INMOST_DATA_INTEGER_TYPE idata = *(INMOST_DATA_INTEGER_TYPE *)data;
		element->Integer(tag) = std::min(element->Integer(tag),idata);
	}
	
Kirill Terekhov's avatar
Kirill Terekhov committed
254
255
	void Mesh::EquilibrateGhost(bool only_new)
	{
256
		static int counter = 0;
Kirill Terekhov's avatar
Kirill Terekhov committed
257
258
259
		if( GetProcessorsNumber() == 1 ) return;
		ENTER_FUNC();
#if defined(USE_MPI)
260
261
		//std::cout << "before_equiv"+std::to_string(counter)+".pvtk" << std::endl;
		//Save("before_equiv"+std::to_string(counter)+".pvtk");
262
263
		ElementType bridge = FACE;
		if( tag_bridge.isValid() ) bridge = ElementType(Integer(GetHandle(),tag_bridge));
264
		TagIntegerArray tag = CreateTag("TEMPORARY_OWNER_DISTANCE_TAG",DATA_INTEGER,CELL,CELL,2);
Kirill Terekhov's avatar
Kirill Terekhov committed
265
		std::queue<HandleType> cells_queue;
266
267
		ENTER_BLOCK();
		//TODO: compute graph distance only in new elements!!!!
Kirill Terekhov's avatar
Kirill Terekhov committed
268
269
270
271
		// Push nearest to owned cell into queue
		for(int k = 0; k < CellLastLocalID(); k++) if( isValidCell(k) )
		{
			Cell c = CellByLocalID(k);
Kirill Terekhov's avatar
Kirill Terekhov committed
272
			if (!c.Hidden() && c.GetStatus() != Element::Owned)
Kirill Terekhov's avatar
Kirill Terekhov committed
273
			{
274
275
				assert( c.GetStatus() == Element::Shared || c.GetStatus() == Element::Ghost );
				tag[c][0] = -1;
Kirill Terekhov's avatar
Kirill Terekhov committed
276
				tag[c][1] = INT_MAX; 
277
278
				//ElementArray<Cell> cells = c.NeighbouringCells();
				ElementArray<Cell> cells = c.BridgeAdjacencies2Cell(bridge);
Kirill Terekhov's avatar
Kirill Terekhov committed
279
280
281
282
283
284
285
286
287
				for(unsigned l = 0; l < cells.size(); l++) if (cells[l].GetStatus() == Element::Owned)
				{
					cells_queue.push(c.GetHandle());
					tag[c][0] = cells[l].Integer(tag_owner);
					tag[c][1] = 1;
					break;
				}
			}
		}
288
		EXIT_BLOCK();
Kirill Terekhov's avatar
Kirill Terekhov committed
289
		int cur_dist = 1; // For assert
290
		ENTER_BLOCK();
Kirill Terekhov's avatar
Kirill Terekhov committed
291
292
293
294
295
296
		while (!cells_queue.empty())
		{
			Cell c(this,cells_queue.front());
			cells_queue.pop();
			if (cur_dist > tag[c][1]) assert(0); 
			if (cur_dist < tag[c][1]) cur_dist++;
297
298
			//ElementArray<Cell> cells = c.NeighbouringCells();
			ElementArray<Cell> cells = c.BridgeAdjacencies2Cell(bridge);
Kirill Terekhov's avatar
Kirill Terekhov committed
299
300
			for(unsigned l = 0; l < cells.size(); l++) if (cells[l].GetStatus() != Element::Owned)
			{
301
				assert( cells[l].GetStatus() == Element::Shared || cells[l].GetStatus() == Element::Ghost );
Kirill Terekhov's avatar
Kirill Terekhov committed
302
303
304
305
306
307
308
309
				if (tag[cells[l]][1] > tag[c][1] + 1)
				{
					tag[cells[l]][0] = tag[c][0];
					tag[cells[l]][1] = tag[c][1] + 1;
					cells_queue.push(cells[l].GetHandle());
				}
			}
		}
310
311
312
313
314
315
316
317
		EXIT_BLOCK();
		
		//static int equiv = 0;
		//std::string file;
		//file = "aequiv"+std::to_string(equiv)+".pvtk";
		//std::cout << "Save " << file << std::endl;
		//Save(file);
		
318
		//CheckCentroids(__FILE__,__LINE__);
Kirill Terekhov's avatar
Kirill Terekhov committed
319
320
		ReduceData(tag, CELL, 0, OperationMinDistance);
		ExchangeData(tag, CELL, 0);
321
		
322
323
		//std::cout << "comput_equiv"+std::to_string(counter)+".pvtk" << std::endl;
		//Save("comput_equiv"+std::to_string(counter)+".pvtk");
324
325
326
327
		//file = "bequiv"+std::to_string(equiv)+".pvtk";
		//std::cout << "Save " << file << std::endl;
		//Save(file);
		//equiv++;
328
329
330

		CheckGhostSharedCount(__FILE__,__LINE__);
		CheckCentroids(__FILE__,__LINE__);
331
332
		
		ENTER_BLOCK();
333
334
335
336
337
338
339
340
341
342
343
344
		//TagInteger tag_new_owner = CreateTag("TEMPORARY_NEW_OWNER",DATA_INTEGER,CELL,NONE,1);
		//for(int k = 0; k < CellLastLocalID(); k++) if( isValidCell(k) )
		//{
		//	Cell c = CellByLocalID(k);
		//	if( !c.Hidden() && c.GetStatus() != Element::Owned )
		//	{
		//		assert( c.GetStatus() == Element::Shared || c.GetStatus() == Element::Ghost );
		//		tag_new_owner[c] = tag[c][0];
		//	}
		//}
		//ExchangeData(tag_new_owner,CELL,0);

Kirill Terekhov's avatar
Kirill Terekhov committed
345
		for(int k = 0; k < CellLastLocalID(); k++) if( isValidCell(k) )
Kirill Terekhov's avatar
Kirill Terekhov committed
346
		{
Kirill Terekhov's avatar
Kirill Terekhov committed
347
348
			Cell c = CellByLocalID(k);
			if( !c.Hidden() && c.GetStatus() != Element::Owned )
Kirill Terekhov's avatar
Kirill Terekhov committed
349
			{
350
				assert( c.GetStatus() == Element::Shared || c.GetStatus() == Element::Ghost );
Kirill Terekhov's avatar
Kirill Terekhov committed
351
352
353
				//if ( !only_new || it->nbAdjElements(NODE | EDGE | FACE | CELL, NewMarker()) != 0)
				{
					int new_owner = tag[c][0];
354
355
356
357
358
359
360
361
362
363
364
					if( new_owner == -1 ) continue;
					//int new_owner = tag_new_owner[c];

					Storage::integer_array procs = c.IntegerArray(ProcessorsTag());
					if( !std::binary_search(procs.begin(),procs.end(),tag[c][0]) ) 
					{
						std::cout << "new owner " << tag[c][0] << " dist " << tag[c][1] << " is not among processors ";
						for(int k = 0; k < procs.size();++k) std::cout << procs[k] << " ";
						std::cout << " for cell " << c.LocalID() << " " << c.GlobalID() << std::endl;
					}
					
Kirill Terekhov's avatar
Kirill Terekhov committed
365
366
367
368
369
370
371
372
					c.IntegerDF(tag_owner) = new_owner;
					if (GetProcessorRank() == new_owner)
						c.SetStatus(Element::Shared);
					else
						c.SetStatus(Element::Ghost);
				}
			}
		}
373
374
375
		//DeleteTag(tag_new_owner);
		//ExchangeData(tag_owner,CELL,0);

376
377
		RecomputeParallelStorage(CELL);
		CheckGhostSharedCount(__FILE__,__LINE__,CELL);
378
		EXIT_BLOCK();
379
380
381
382
		DeleteTag(tag);
		
		
		
383
		ENTER_BLOCK();
384
385
		
		TagInteger new_owner = CreateTag("TEMPORARY_NEW_OWNER",DATA_INTEGER,NODE|EDGE|FACE,NODE|EDGE|FACE,1);
Kirill Terekhov's avatar
Kirill Terekhov committed
386
387
		for(ElementType etype = FACE; etype >= NODE; etype = PrevElementType(etype))
		{
388
389
			REPORT_VAL("etype ", ElementTypeName(etype));
			ENTER_BLOCK();
Kirill Terekhov's avatar
Kirill Terekhov committed
390
391
392
393
394
			for(int k = 0; k < LastLocalID(etype); k++) if( isValidElement(etype,k) )
			{
				Element e = ElementByLocalID(etype,k);
				if( !e.Hidden() && e.GetStatus() != Element::Owned )
				{
395
					assert( e.GetStatus() == Element::Shared || e.GetStatus() == Element::Ghost );
396
					//if( !only_new || e.nbAdjElements(NODE | EDGE | FACE | CELL,NewMarker()) )
Kirill Terekhov's avatar
Kirill Terekhov committed
397
398
					{
						Element::adj_type & hc = HighConn(e.GetHandle());
399
						new_owner[e] = INT_MAX;
Kirill Terekhov's avatar
Kirill Terekhov committed
400
401
402
						for(int l = 0; l < hc.size(); ++l)
						{
							if( !GetMarker(hc[l],HideMarker()) )
403
								new_owner[e] = std::min(new_owner[e],Integer(hc[l],tag_owner));
Kirill Terekhov's avatar
Kirill Terekhov committed
404
						}
405
406
407
					}
				}
			}
408
			EXIT_BLOCK();
409
410
			ReduceData(new_owner,etype,0,OperationMinOwner);
			ExchangeData(new_owner,etype);
411
			ENTER_BLOCK();
412
413
414
415
416
			for(int k = 0; k < LastLocalID(etype); k++) if( isValidElement(etype,k) )
			{
				Element e = ElementByLocalID(etype,k);
				if( !e.Hidden() && e.GetStatus() != Element::Owned )
				{
417
					assert( e.GetStatus() == Element::Shared || e.GetStatus() == Element::Ghost );
418
419
420
421
					//if( !only_new || e.nbAdjElements(NODE | EDGE | FACE | CELL,NewMarker()) )
					{
						if( new_owner[e] != INT_MAX &&
						   e.IntegerDF(tag_owner) != new_owner[e] )
Kirill Terekhov's avatar
Kirill Terekhov committed
422
						{
423
424
							e.IntegerDF(tag_owner) = new_owner[e];
							if (GetProcessorRank() == new_owner[e])
Kirill Terekhov's avatar
Kirill Terekhov committed
425
426
427
428
429
430
								e.SetStatus(Element::Shared);
							else
								e.SetStatus(Element::Ghost);
						}
					}
				}
Kirill Terekhov's avatar
Kirill Terekhov committed
431
			}
432
433
434
			EXIT_BLOCK();
			RecomputeParallelStorage(etype);
			CheckGhostSharedCount(__FILE__,__LINE__,etype);
Kirill Terekhov's avatar
Kirill Terekhov committed
435
		}
436
		DeleteTag(new_owner);
437
438
439
		//ComputeSharedProcs();
		//std::cout << "after_equiv"+std::to_string(counter)+".pvtk" << std::endl;
		//Save("after_equiv"+std::to_string(counter)+".pvtk");
440
		EXIT_BLOCK();
441
		//RecomputeParallelStorage(CELL | EDGE | FACE | NODE);
442
#endif
Kirill Terekhov's avatar
Kirill Terekhov committed
443
		EXIT_FUNC();
444
		counter++;
Kirill Terekhov's avatar
Kirill Terekhov committed
445
	}
Kirill Terekhov's avatar
Fixes    
Kirill Terekhov committed
446
447
	
	void UnpackSyncMarkerOR(const Tag & tag, const Element & element, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
Kirill Terekhov's avatar
Kirill Terekhov committed
448
	{
Kirill Terekhov's avatar
Fixes    
Kirill Terekhov committed
449
		(void) size;
Kirill Terekhov's avatar
Kirill Terekhov committed
450
451
452
		element->Bulk(tag) |= *data;
	}

Kirill Terekhov's avatar
Fixes    
Kirill Terekhov committed
453
	void UnpackSyncMarkerXOR(const Tag & tag, const Element & element, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
Kirill Terekhov's avatar
Kirill Terekhov committed
454
	{
Kirill Terekhov's avatar
Fixes    
Kirill Terekhov committed
455
		(void) size;
Kirill Terekhov's avatar
Kirill Terekhov committed
456
457
458
459
		element->Bulk(tag) ^= *data;
	}

	
Kirill Terekhov's avatar
Fixes    
Kirill Terekhov committed
460
	void UnpackSyncMarkerAND(const Tag & tag, const Element & element, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
Kirill Terekhov's avatar
Kirill Terekhov committed
461
	{
Kirill Terekhov's avatar
Fixes    
Kirill Terekhov committed
462
		(void) size;
Kirill Terekhov's avatar
Kirill Terekhov committed
463
464
465
		element->Bulk(tag) &= *data;
	}

Kirill Terekhov's avatar
Fixes    
Kirill Terekhov committed
466
467
468
	//////////////////////////////
	/// REDUCTION FUNCTIONS END///
	//////////////////////////////
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
	bool Mesh::HaveGlobalID(ElementType type) const
	{
		assert(OneType(type));
		Tag test;
		if( GlobalIDTag().isValid() ) test = GlobalIDTag();
		//else if( HaveTag("GLOBAL_ID") ) test = GetTag("GLOBAL_ID") );
		if( test.isValid() )
		{
			//bool ret = true;
			//for(ElementType etype = NODE; etype <= MESH; etype = NextElementType(etype) ) if( etype & types )
			//	ret &= test.isDefined(etype);
			//return ret;
			return test.isDefined(type);
		}
		else return false;
	}
Kirill Terekhov's avatar
Fixes    
Kirill Terekhov committed
485

Kirill Terekhov's avatar
Kirill Terekhov committed
486

Kirill Terekhov's avatar
Kirill Terekhov committed
487
	void Mesh::SynchronizeMarker(MarkerType marker, ElementType mask, SyncBitOp op)
Kirill Terekhov's avatar
Kirill Terekhov committed
488
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
489
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
490
491
492
493
#if defined(USE_MPI)
		if( m_state == Mesh::Parallel )
		{
			Tag t = CreateTag("TEMP_SYNC_MARKER",DATA_BULK,mask,mask,1);
Kirill Terekhov's avatar
Kirill Terekhov committed
494
			
Kirill Terekhov's avatar
Kirill Terekhov committed
495
496
497
			//workaround for old gcc compiler
			const Element::Status SGhost = Element::Ghost;
			const Element::Status SAny = Element::Any;
Kirill Terekhov's avatar
Kirill Terekhov committed
498
			Element::Status Expr = (Element::Shared | ((op != SYNC_BIT_SET) ? SGhost : SAny));
Kirill Terekhov's avatar
Kirill Terekhov committed
499
			
Kirill Terekhov's avatar
Kirill Terekhov committed
500
			for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); ++it)
Kirill Terekhov's avatar
Kirill Terekhov committed
501
				if( it->GetMarker(marker) && (it->GetStatus() & Expr) )
Kirill Terekhov's avatar
Kirill Terekhov committed
502
					it->Bulk(t) = 1;
Kirill Terekhov's avatar
Kirill Terekhov committed
503
			
Kirill Terekhov's avatar
Kirill Terekhov committed
504
505
506
507
			
			
			switch(op)
			{
Kirill Terekhov's avatar
Kirill Terekhov committed
508
509
510
				case SYNC_BIT_SET:
					ExchangeData(t,mask,0);
					for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); ++it)
Kirill Terekhov's avatar
Kirill Terekhov committed
511
					{
Kirill Terekhov's avatar
Kirill Terekhov committed
512
513
514
515
						if( it->GetStatus() == Element::Ghost )
						{
							if( it->HaveData(t) ) it->SetMarker(marker); else it->RemMarker(marker);
						}
Kirill Terekhov's avatar
Kirill Terekhov committed
516
					}
Kirill Terekhov's avatar
Kirill Terekhov committed
517
518
519
520
					break;
				case SYNC_BIT_OR:
					ReduceData(t,mask,0,UnpackSyncMarkerOR);
					for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); ++it)
Kirill Terekhov's avatar
Kirill Terekhov committed
521
					{
Kirill Terekhov's avatar
Kirill Terekhov committed
522
523
524
525
						if( it->GetStatus() & (Element::Ghost | Element::Shared) )
						{
							if( !it->GetMarker(marker) && it->HaveData(t) ) it->SetMarker(marker);
						}
Kirill Terekhov's avatar
Kirill Terekhov committed
526
					}
Kirill Terekhov's avatar
Kirill Terekhov committed
527
528
529
530
					break;
				case SYNC_BIT_AND:
					ReduceData(t,mask,0,UnpackSyncMarkerAND);
					for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); ++it)
Kirill Terekhov's avatar
Kirill Terekhov committed
531
					{
Kirill Terekhov's avatar
Kirill Terekhov committed
532
533
534
535
						if( it->GetStatus() & (Element::Ghost | Element::Shared))
						{
							if( it->HaveData(t) && it->Bulk(t) ) it->SetMarker(marker); else it->RemMarker(marker);
						}
Kirill Terekhov's avatar
Kirill Terekhov committed
536
					}
Kirill Terekhov's avatar
Kirill Terekhov committed
537
538
539
540
					break;
				case SYNC_BIT_XOR:
					ReduceData(t,mask,0,UnpackSyncMarkerXOR);
					for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); ++it)
Kirill Terekhov's avatar
Kirill Terekhov committed
541
					{
Kirill Terekhov's avatar
Kirill Terekhov committed
542
543
544
545
						if( it->GetStatus() & (Element::Ghost | Element::Shared))
						{
							if( it->HaveData(t) && it->Bulk(t) ) it->SetMarker(marker); else it->RemMarker(marker);
						}
Kirill Terekhov's avatar
Kirill Terekhov committed
546
					}
Kirill Terekhov's avatar
Kirill Terekhov committed
547
					break;
Kirill Terekhov's avatar
Kirill Terekhov committed
548
549
			}
			
Kirill Terekhov's avatar
Kirill Terekhov committed
550
			
Kirill Terekhov's avatar
Kirill Terekhov committed
551
552
			DeleteTag(t,mask);
		}
Kirill Terekhov's avatar
Fixes    
Kirill Terekhov committed
553
554
555
556
557
#else//USE_MPI
		(void) marker;
		(void) mask;
		(void) op;
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
558
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
559
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
560
	
Kirill Terekhov's avatar
Kirill Terekhov committed
561
562
	ElementType Mesh::SynchronizeElementType(ElementType etype)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
563
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
564
565
		ElementType etypeout = etype;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
566
		REPORT_MPI(MPI_Allreduce(&etype,&etypeout,1,INMOST_MPI_DATA_BULK_TYPE,MPI_BOR,GetCommunicator()));
Kirill Terekhov's avatar
Fixes    
Kirill Terekhov committed
567
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
568
569
		return etypeout;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
570
571
	Storage::integer Mesh::TotalNumberOf(ElementType mask)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
572
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
573
574
575
576
577
		Storage::integer number = 0, ret = 0;
		for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); it++)
			if( it->GetStatus() != Element::Ghost ) number++;
		ret = number;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
578
		REPORT_MPI(MPI_Allreduce(&number,&ret,1,INMOST_MPI_DATA_INTEGER_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Fixes    
Kirill Terekhov committed
579
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
580
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
581
582
		return ret;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
583
	Storage::integer Mesh::EnumerateSet(const ElementSet & set, const Tag & num_tag, Storage::integer start, bool define_sparse)
Kirill Terekhov's avatar
Kirill Terekhov committed
584
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
585
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
586
587
588
589
		Storage::integer shift = 0, ret = 0;
		ElementType mask = CELL | FACE | EDGE | NODE;
#if defined(USE_MPI)
		Storage::integer number = 0;
Kirill Terekhov's avatar
Kirill Terekhov committed
590
591
		for(ElementSet::iterator it = set.Begin(); it != set.End(); it++)
			if( it->GetStatus() != Element::Ghost && (define_sparse || it->HaveData(num_tag)) )
Kirill Terekhov's avatar
Kirill Terekhov committed
592
				number++;
Kirill Terekhov's avatar
Kirill Terekhov committed
593
		REPORT_MPI(MPI_Scan(&number,&shift,1,INMOST_MPI_DATA_INTEGER_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
594
		shift -= number;
Kirill Terekhov's avatar
Fixes    
Kirill Terekhov committed
595
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
596
		shift += start;
Kirill Terekhov's avatar
Kirill Terekhov committed
597
		for(ElementSet::iterator it = set.Begin(); it != set.End(); it++)
598
			if( (it->GetStatus() == Element::Owned || it->GetStatus() == Element::Any) && (define_sparse || it->HaveData(num_tag)) )
Kirill Terekhov's avatar
Kirill Terekhov committed
599
				it->Integer(num_tag) = shift++;
Kirill Terekhov's avatar
Kirill Terekhov committed
600
601
		for(ElementSet::iterator it = set.Begin(); it != set.End(); it++)
			if( it->GetStatus() == Element::Shared && (define_sparse || it->HaveData(num_tag)) )
Kirill Terekhov's avatar
Kirill Terekhov committed
602
				it->Integer(num_tag) = shift++;
Kirill Terekhov's avatar
Kirill Terekhov committed
603
		ExchangeData(num_tag,mask,0);
Kirill Terekhov's avatar
Kirill Terekhov committed
604
605
		ret = shift;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
606
		REPORT_MPI(MPI_Bcast(&ret,1,INMOST_MPI_DATA_INTEGER_TYPE,GetProcessorsNumber()-1,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
607
		//MPI_Allreduce(&shift,&ret,1,INMOST_DATA_INTEGER_TYPE,MPI_MAX,comm);
Kirill Terekhov's avatar
Fixes    
Kirill Terekhov committed
608
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
609
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
610
611
		return ret;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
612
	Storage::integer Mesh::Enumerate(const HandleType * set, enumerator n, const Tag & num_tag, Storage::integer start, bool define_sparse)
Kirill Terekhov's avatar
Kirill Terekhov committed
613
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
614
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
615
616
617
618
		Storage::integer shift = 0, ret = 0;
		ElementType mask = CELL | FACE | EDGE | NODE;
#if defined(USE_MPI)
		Storage::integer number = 0;
Kirill Terekhov's avatar
Kirill Terekhov committed
619
620
		for(const HandleType * it = set; it != set+n; ++it)
			if( GetStatus(*it) != Element::Ghost && (define_sparse || HaveData(*it,num_tag))) number++;
Kirill Terekhov's avatar
Kirill Terekhov committed
621
		REPORT_MPI(MPI_Scan(&number,&shift,1,INMOST_MPI_DATA_INTEGER_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
622
		shift -= number;
Kirill Terekhov's avatar
Fixes    
Kirill Terekhov committed
623
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
624
		shift += start;
Kirill Terekhov's avatar
Kirill Terekhov committed
625
		for(const HandleType * it = set; it != set+n; ++it)
626
			if( (GetStatus(*it) == Element::Owned || GetStatus(*it) == Element::Any) && (define_sparse || HaveData(*it,num_tag))) Integer(*it,num_tag) = shift++;
Kirill Terekhov's avatar
Kirill Terekhov committed
627
		for(const HandleType * it = set; it != set+n; ++it)
628
			if( GetStatus(*it) == Element::Shared && (define_sparse || HaveData(*it,num_tag))) Integer(*it,num_tag) = shift++;
Kirill Terekhov's avatar
Kirill Terekhov committed
629
		ExchangeData(num_tag,mask,0);
Kirill Terekhov's avatar
Kirill Terekhov committed
630
631
		ret = shift;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
632
		REPORT_MPI(MPI_Bcast(&ret,1,INMOST_MPI_DATA_INTEGER_TYPE,GetProcessorsNumber()-1,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
633
		//MPI_Allreduce(&shift,&ret,1,INMOST_DATA_INTEGER_TYPE,MPI_MAX,comm);
Kirill Terekhov's avatar
Fixes    
Kirill Terekhov committed
634
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
635
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
636
637
		return ret;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
638
	Storage::integer Mesh::Enumerate(ElementType mask, Tag num_tag, Storage::integer start, bool define_sparse)
Kirill Terekhov's avatar
Kirill Terekhov committed
639
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
640
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
641
642
643
644
		Storage::integer shift = 0, ret = 0;
#if defined(USE_MPI)
		Storage::integer number = 0;
		for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); it++)
Kirill Terekhov's avatar
Kirill Terekhov committed
645
			if( it->GetStatus() != Element::Ghost && (define_sparse || it->HaveData(num_tag)) )
Kirill Terekhov's avatar
Kirill Terekhov committed
646
				number++;
Kirill Terekhov's avatar
Kirill Terekhov committed
647
		REPORT_MPI(MPI_Scan(&number,&shift,1,INMOST_MPI_DATA_INTEGER_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
648
		shift -= number;
Kirill Terekhov's avatar
Fixes    
Kirill Terekhov committed
649
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
650
651
		shift += start;
		for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); it++)
652
			if( (it->GetStatus() == Element::Owned || it->GetStatus() == Element::Any) && (define_sparse || it->HaveData(num_tag)) )
Kirill Terekhov's avatar
Kirill Terekhov committed
653
654
				it->Integer(num_tag) = shift++;
		for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); it++)
655
			if( it->GetStatus() == Element::Shared && (define_sparse || it->HaveData(num_tag)) )
Kirill Terekhov's avatar
Kirill Terekhov committed
656
				it->Integer(num_tag) = shift++;
Kirill Terekhov's avatar
Kirill Terekhov committed
657
		ExchangeData(num_tag,mask,0);
Kirill Terekhov's avatar
Kirill Terekhov committed
658
659
		ret = shift;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
660
		REPORT_MPI(MPI_Bcast(&ret,1,INMOST_MPI_DATA_INTEGER_TYPE,GetProcessorsNumber()-1,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
661
		//MPI_Allreduce(&shift,&ret,1,INMOST_DATA_INTEGER_TYPE,MPI_MAX,comm);
Kirill Terekhov's avatar
Fixes    
Kirill Terekhov committed
662
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
663
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
664
665
666
667
668
		return ret;
	}
	
	Storage::real Mesh::Integrate(Storage::real input)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
669
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
670
671
		Storage::real output = input;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
672
		REPORT_MPI(MPI_Allreduce(&input,&output,1,INMOST_MPI_DATA_REAL_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Fixes    
Kirill Terekhov committed
673
674
675
#else//USE_MPI
		(void) input;
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
676
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
677
678
		return output;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
679
680
	
	
Kirill Terekhov's avatar
Kirill Terekhov committed
681
	
Kirill Terekhov's avatar
Kirill Terekhov committed
682
683
684
	
	Storage::integer Mesh::Integrate(Storage::integer input)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
685
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
686
687
		Storage::integer output = input;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
688
		REPORT_MPI(MPI_Allreduce(&input,&output,1,INMOST_MPI_DATA_INTEGER_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Fixes    
Kirill Terekhov committed
689
690
691
#else//USE_MPI
		(void) input;
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
692
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
693
694
		return output;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
695
	
Kirill Terekhov's avatar
Kirill Terekhov committed
696
697
	void Mesh::Integrate(Storage::real * input, Storage::integer size)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
698
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
699
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
700
701
		static dynarray<Storage::real,64> temp;
		temp.resize(size);
Kirill Terekhov's avatar
Kirill Terekhov committed
702
		memcpy(temp.data(),input,sizeof(Storage::real)*size);
Kirill Terekhov's avatar
Kirill Terekhov committed
703
		REPORT_MPI(MPI_Allreduce(temp.data(),input,size,INMOST_MPI_DATA_REAL_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
704
#else//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
705
706
		(void) input;
		(void) size;
Kirill Terekhov's avatar
Kirill Terekhov committed
707
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
708
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
709
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
710
	
Kirill Terekhov's avatar
Kirill Terekhov committed
711
712
	void Mesh::Integrate(Storage::integer * input, Storage::integer size)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
713
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
714
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
715
716
		static dynarray<Storage::integer,64> temp;
		temp.resize(size);
Kirill Terekhov's avatar
Kirill Terekhov committed
717
		memcpy(temp.data(),input,sizeof(Storage::integer)*size);
Kirill Terekhov's avatar
Kirill Terekhov committed
718
		REPORT_MPI(MPI_Allreduce(temp.data(),input,size,INMOST_MPI_DATA_INTEGER_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
719
720
#else//USE_MPI
		(void) input;
Kirill Terekhov's avatar
Kirill Terekhov committed
721
		(void) size;
Kirill Terekhov's avatar
Kirill Terekhov committed
722
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
723
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
724
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
725
726
727
	
	Storage::integer Mesh::ExclusiveSum(Storage::integer input)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
728
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
729
730
		Storage::integer output = 0;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
731
		REPORT_MPI(MPI_Scan(&input,&output,1,INMOST_MPI_DATA_INTEGER_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
732
		output -= input;
Kirill Terekhov's avatar
Fixes    
Kirill Terekhov committed
733
734
735
#else//USE_MPI
		(void) input;
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
736
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
737
738
739
		return output;
	}
	
Kirill Terekhov's avatar
Kirill Terekhov committed
740
	Storage::real Mesh::Integrate(const Tag & t, enumerator entry, ElementType mask)
Kirill Terekhov's avatar
Kirill Terekhov committed
741
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
742
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
743
744
		Storage::real output = 0, input = 0;
		for(iteratorElement it = BeginElement(mask); it != EndElement(); it++)
Kirill Terekhov's avatar
Kirill Terekhov committed
745
			if( GetStatus(*it) != Element::Ghost && HaveData(*it,t) )
Kirill Terekhov's avatar
Kirill Terekhov committed
746
747
748
749
			{
				real_array arr = RealArray(*it,t);
				if( arr.size() > entry ) input += arr[entry];
			}
Kirill Terekhov's avatar
Kirill Terekhov committed
750
751
		output = input;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
752
		REPORT_MPI(MPI_Allreduce(&input,&output,1,INMOST_MPI_DATA_REAL_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Fixes    
Kirill Terekhov committed
753
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
754
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
755
756
757
		return output;
	}
	
758
759
	Storage::real Mesh::AggregateMax(Storage::real input)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
760
		ENTER_FUNC();
761
762
		Storage::real output = input;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
763
		REPORT_MPI(MPI_Allreduce(&input,&output,1,INMOST_MPI_DATA_REAL_TYPE,MPI_MAX,comm));
Kirill Terekhov's avatar
Fixes    
Kirill Terekhov committed
764
765
766
#else //USE_MPI
		(void) input;
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
767
		EXIT_FUNC();
768
769
		return output;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
770
	
771
772
	Storage::integer Mesh::AggregateMax(Storage::integer input)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
773
		ENTER_FUNC();
774
775
		Storage::integer output = input;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
776
		REPORT_MPI(MPI_Allreduce(&input,&output,1,INMOST_MPI_DATA_INTEGER_TYPE,MPI_MAX,comm));
Kirill Terekhov's avatar
Fixes    
Kirill Terekhov committed
777
778
779
#else //USE_MPI
		(void) input;
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
780
		EXIT_FUNC();
781
782
		return output;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
783
	
Kirill Terekhov's avatar
Kirill Terekhov committed
784
785
	void Mesh::AggregateMax(Storage::real * input, Storage::integer size)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
786
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
787
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
788
789
		static dynarray<Storage::real,64> temp;
		temp.resize(size);
Kirill Terekhov's avatar
Kirill Terekhov committed
790
		memcpy(temp.data(),input,sizeof(Storage::real)*size);
Kirill Terekhov's avatar
Kirill Terekhov committed
791
		REPORT_MPI(MPI_Allreduce(temp.data(),input,size,INMOST_MPI_DATA_REAL_TYPE,MPI_MAX,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
792
793
#else//USE_MPI
		(void) input;
Kirill Terekhov's avatar
Kirill Terekhov committed
794
		(void) size;
Kirill Terekhov's avatar
Kirill Terekhov committed
795
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
796
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
797
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
798
	
Kirill Terekhov's avatar
Kirill Terekhov committed
799
800
	void Mesh::AggregateMax(Storage::integer * input, Storage::integer size)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
801
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
802
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
803
804
		static dynarray<Storage::integer,64> temp;
		temp.resize(size);
Kirill Terekhov's avatar
Kirill Terekhov committed
805
		memcpy(temp.data(),input,sizeof(Storage::integer)*size);
Kirill Terekhov's avatar
Kirill Terekhov committed
806
		REPORT_MPI(MPI_Allreduce(temp.data(),input,size,INMOST_MPI_DATA_INTEGER_TYPE,MPI_MAX,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
807
808
#else//USE_MPI
		(void) input;
Kirill Terekhov's avatar
Kirill Terekhov committed
809
		(void) size;
Kirill Terekhov's avatar
Kirill Terekhov committed
810
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
811
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
812
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
813
	
Kirill Terekhov's avatar
Kirill Terekhov committed
814
815
	Storage::real Mesh::AggregateMin(Storage::real input)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
816
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
817
818
		Storage::real output = input;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
819
		REPORT_MPI(MPI_Allreduce(&input,&output,1,INMOST_MPI_DATA_REAL_TYPE,MPI_MIN,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
820
821
822
#else //USE_MPI
		(void) input;
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
823
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
824
825
826
827
828
		return output;
	}
	
	Storage::integer Mesh::AggregateMin(Storage::integer input)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
829
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
830
831
		Storage::integer output = input;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
832
		REPORT_MPI(MPI_Allreduce(&input,&output,1,INMOST_MPI_DATA_INTEGER_TYPE,MPI_MIN,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
833
834
835
#else //USE_MPI
		(void) input;
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
836
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
837
838
		return output;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
839
	
Kirill Terekhov's avatar
Kirill Terekhov committed
840
841
	void Mesh::AggregateMin(Storage::real * input, Storage::integer size)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
842
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
843
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
844
845
		static dynarray<Storage::real,64> temp;
		temp.resize(size);
Kirill Terekhov's avatar
Kirill Terekhov committed
846
		memcpy(temp.data(),input,sizeof(Storage::real)*size);
Kirill Terekhov's avatar
Kirill Terekhov committed
847
		REPORT_MPI(MPI_Allreduce(temp.data(),input,size,INMOST_MPI_DATA_REAL_TYPE,MPI_MIN,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
848
849
#else//USE_MPI
		(void) input;
Kirill Terekhov's avatar
Kirill Terekhov committed
850
		(void) size;
Kirill Terekhov's avatar
Kirill Terekhov committed
851
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
852
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
853
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
854
	
Kirill Terekhov's avatar
Kirill Terekhov committed
855
856
	void Mesh::AggregateMin(Storage::integer * input, Storage::integer size)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
857
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
858
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
859
860
		static dynarray<Storage::integer,64> temp;
		temp.resize(size);
Kirill Terekhov's avatar
Kirill Terekhov committed
861
		memcpy(temp.data(),input,sizeof(Storage::integer)*size);
Kirill Terekhov's avatar
Kirill Terekhov committed
862
		REPORT_MPI(MPI_Allreduce(temp.data(),input,size,INMOST_MPI_DATA_INTEGER_TYPE,MPI_MIN,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
863
864
#else//USE_MPI
		(void) input;
Kirill Terekhov's avatar
Kirill Terekhov committed
865
		(void) size;
Kirill Terekhov's avatar
Kirill Terekhov committed
866
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
867
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
868
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
869
870
	
	
Kirill Terekhov's avatar
Kirill Terekhov committed
871
	INMOST_MPI_Comm Mesh::GetCommunicator() const
Kirill Terekhov's avatar
Kirill Terekhov committed
872
873
874
	{
#if defined(USE_MPI)
		return comm;
Kirill Terekhov's avatar
Fixes    
Kirill Terekhov committed
875
#else //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
876
		return 0;
Kirill Terekhov's avatar
Fixes    
Kirill Terekhov committed
877
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
878
879
	}
	
Kirill Terekhov's avatar
Kirill Terekhov committed
880
	int Mesh::GetProcessorRank() const
Kirill Terekhov's avatar
Kirill Terekhov committed
881
882
883
884
885
	{
#if defined(USE_MPI)
		int rank;
		MPI_Comm_rank(comm,&rank);
		return rank;
Kirill Terekhov's avatar
Fixes    
Kirill Terekhov committed
886
#else //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
887
		return 0;
Kirill Terekhov's avatar
Fixes    
Kirill Terekhov committed
888
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
889
890
	}
	
Kirill Terekhov's avatar
Kirill Terekhov committed
891
	int Mesh::GetProcessorsNumber() const
Kirill Terekhov's avatar
Kirill Terekhov committed
892
893
894
895
896
	{
#if defined(USE_MPI)
		int size;
		MPI_Comm_size(comm,&size);
		return size;
Kirill Terekhov's avatar
Fixes    
Kirill Terekhov committed
897
#else //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
898
		return 1;
Kirill Terekhov's avatar
Fixes    
Kirill Terekhov committed
899
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
900
901
902
903
904
905
906
907
	}	
	
	void Mesh::Initialize(int * argc, char *** argv)
	{
#if defined(USE_MPI)
		int test;
		MPI_Initialized(&test);
        if( test == 0 ) MPI_Init(argc,argv);
Kirill Terekhov's avatar
Fixes    
Kirill Terekhov committed
908
909
910
911
#else //USE_MPI
		(void) argc;
		(void) argv;
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
912
913
914
#if defined(USE_PARALLEL_WRITE_TIME)
		atexit(Mesh::AtExit);
#endif
Kirill Terekhov's avatar
Kirill Terekhov committed
915
916
917
918
919
920
921
922
	}
	void Mesh::Finalize()
	{
#if defined(USE_MPI)
		int test = 0;
		MPI_Finalized(&test);
		if( !test )
			MPI_Finalize();
Kirill Terekhov's avatar
Fixes    
Kirill Terekhov committed
923
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
924
925
	}
	
Kirill Terekhov's avatar
Kirill Terekhov committed
926
    INMOST_MPI_Group Mesh::GetGroup() const
Kirill Terekhov's avatar
Fixes    
Kirill Terekhov committed
927
928
929
930
931
932
933
    {
        INMOST_MPI_Group ret = INMOST_MPI_GROUP_EMPTY;
#if defined(USE_MPI)
        MPI_Comm_group(GetCommunicator(), &ret);
#endif
        return ret;
    }
Kirill Terekhov's avatar
Kirill Terekhov committed
934
935
936
937
	
	void Mesh::SetCommunicator(INMOST_MPI_Comm _comm)
	{
		ENTER_FUNC();
938
939
		tag_shared = CreateTag("PROTECTED_STATUS",DATA_BULK,  ESET |CELL | FACE | EDGE | NODE,NONE,1);
		tag_owner = CreateTag("OWNER_PROCESSOR",DATA_INTEGER, ESET | CELL | FACE | EDGE | NODE,NONE,1);
940
		tag_processors = CreateTag("PROCESSORS_LIST",DATA_INTEGER, ESET | NODE | EDGE | FACE | CELL,NONE);
Kirill Terekhov's avatar
Kirill Terekhov committed
941
942
		tag_layers = CreateTag("LAYERS",DATA_INTEGER,MESH,NONE,1);
		tag_bridge = CreateTag("BRIDGE",DATA_INTEGER,MESH,NONE,1);
943
944
		tag_sendto = CreateTag("PROTECTED_SENDTO",DATA_INTEGER, ESET | CELL | FACE | EDGE | NODE, ESET | CELL | FACE | EDGE | NODE);
		tag_an_id = CreateTag("PROTECTED_ID",DATA_INTEGER, CELL | FACE | EDGE | NODE, CELL | FACE | EDGE | NODE);
Kirill Terekhov's avatar
Kirill Terekhov committed
945
946

#if defined(USE_MPI)
947
		randomizer = Random();
Kirill Terekhov's avatar
Kirill Terekhov committed
948
949
950
951
952
953
954
955
956
957
958
		
		parallel_strategy = 1;
		parallel_file_strategy = 1;

		

		Mesh::Initialize(NULL,NULL);
		//~ MPI_Comm_dup(_comm,&comm);
		comm = _comm;
		{
			INMOST_DATA_BIG_ENUM_TYPE t = pmid;
959
			REPORT_MPI(MPI_Allreduce(&t,&parallel_mesh_unique_id,1,INMOST_MPI_DATA_BIG_ENUM_TYPE,MPI_MAX,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
960
961
962
963
964
			pmid = parallel_mesh_unique_id+1;
		}
		m_state = Mesh::Parallel;


Kirill Terekhov's avatar
Fixes    
Kirill Terekhov committed
965
#if defined(USE_MPI_P2P)
Kirill Terekhov's avatar
Kirill Terekhov committed
966
967
968
969
970
971
972
973
974
975
976
977
978