parallel.cpp 205 KB
Newer Older
Kirill Terekhov's avatar
Kirill Terekhov committed
1
#ifdef _MSC_VER //kill some warnings
2
#ifndef _SCL_SECURE_NO_WARNINGS
Kirill Terekhov's avatar
Kirill Terekhov committed
3 4
#define _SCL_SECURE_NO_WARNINGS
#endif
5
#endif
Kirill Terekhov's avatar
Kirill Terekhov committed
6

Kirill Terekhov's avatar
Kirill Terekhov committed
7 8 9 10 11 12 13
#include "inmost.h"


#if defined(USE_MESH)
#include <iostream>
#include <fstream>
#include <sstream>
14
#include <string>
Kirill Terekhov's avatar
Kirill Terekhov committed
15

16
using namespace std;
Kirill Terekhov's avatar
Kirill Terekhov committed
17 18 19 20 21 22 23 24 25

#if defined(USE_MPI)
static INMOST_DATA_BIG_ENUM_TYPE pmid = 0;
#endif

#if defined(USE_PARALLEL_WRITE_TIME)
#define REPORT_MPI(x) {WriteTab(out_time) << "<MPI><![CDATA[" << #x << "]]></MPI>" << std::endl; x;}
#define REPORT_STR(x) {WriteTab(out_time) << "<TEXT><![CDATA[" << x << "]]></TEXT>" << std::endl;}
#define REPORT_VAL(str,x) {WriteTab(out_time) << "<VALUE name=\"" << str << "\"> <CONTENT><![CDATA[" << x << "]]></CONTENT> <CODE><![CDATA[" << #x << "]]></CODE></VALUE>" << std::endl;}
Kirill Terekhov's avatar
Kirill Terekhov committed
26 27 28
#define ENTER_FUNC() double all_time = Timer(); {WriteTab(out_time) << "<FUNCTION name=\"" << __FUNCTION__ << "\" id=\"func" << func_id++ << "\">" << std::endl; Enter();}
#define EXIT_FUNC() {WriteTab(out_time) << "<TIME>" << Timer() - all_time << "</TIME>" << std::endl; Exit(); WriteTab(out_time) << "</FUNCTION>" << std::endl;}
#define EXIT_FUNC_DIE() {WriteTab(out_time) << "<TIME>" << -1 << "</TIME>" << std::endl; Exit(); WriteTab(out_time) << "</FUNCTION>" << std::endl;}
Kirill Terekhov's avatar
Kirill Terekhov committed
29 30
#else
#define REPORT_MPI(x) x
Kirill Terekhov's avatar
Kirill Terekhov committed
31 32 33 34 35
#define REPORT_STR(x) {}
#define REPORT_VAL(str,x) {}
#define ENTER_FUNC() {}
#define EXIT_FUNC() {}
#define EXIT_FUNC_DIE()  {}
Kirill Terekhov's avatar
Kirill Terekhov committed
36 37 38 39
#endif

namespace INMOST
{
40 41 42 43 44
    static int flag = 0;

    std::string ro()
    {
        int rank = 0;
45
#ifdef USE_MPI
46
        MPI_Comm_rank(MPI_COMM_WORLD,&rank);
47
#endif
48 49 50 51 52 53
        std::stringstream ss;
        for (int i = 0; i < rank; i++)
            ss << "   ";
        return ss.str();
    }
    
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96
	//////////////////////////////
	/// REDUCTION FUNCTIONS    ///
	//////////////////////////////
	void DefaultUnpack(const Tag & tag, const Element & element, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
	{
		if( size == 0 )
		{
			if( tag.isDefined(element->GetElementType()) ) 
			{
				if( tag.isSparse(element->GetElementType()) )
				{
					if( element->HaveData(tag) )
						element->DelData(tag); 
				}
				else if( tag.GetSize() == ENUMUNDEF )
					element->SetDataSize(tag,size);
				else
				{
					std::cerr << "received zero size for dense nonzero tag" << std::endl;
					assert(false);
				}
			}
			return;
		}
		if( !element->HaveData(tag) )
			element->SetDataSize(tag,size);
		else if( size != element->GetDataSize(tag) )
		{
			if( tag.GetSize() == ENUMUNDEF )
				element->SetDataSize(tag,size);
			else
			{
				assert(false);
			}
		}
		element->SetData(tag,0,size,data);
	}


	void UnpackOnSkin(const Tag & tag, const Element & e, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
	{
		if( size )
		{
Kirill Terekhov's avatar
Kirill Terekhov committed
97
      const Storage::integer * recv = static_cast<const Storage::integer *>(static_cast<const void *>(data));
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
98 99 100 101 102 103 104 105 106 107
			Storage::integer_array arr = e->IntegerArray(tag);
			arr.push_back(recv[0]);
		}
	}


	void UnpackSkin(const Tag & tag, const Element & e, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
	{
		if( size )
		{
Kirill Terekhov's avatar
Kirill Terekhov committed
108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127
      for(INMOST_DATA_ENUM_TYPE k = 0; k < size; k+=2 )
      {
			  bool flag = true;
			  const Storage::integer * recv = static_cast<const Storage::integer *>(static_cast<const void *>(data))+k;
			  Storage::integer_array arr = e->IntegerArray(tag);
      
        for(Storage::integer_array::iterator it = arr.begin(); it != arr.end(); it+=2)
        {
          if( *it == recv[0] )
				  {
					  flag = false;
					  break;
				  }
        }
			  if( flag ) 
			  {
				  arr.push_back(recv[0]);
				  arr.push_back(recv[1]);
			  }
      }
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171
		}
	}
	


	void DeleteUnpack(const Tag & tag, const Element & e, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
	{
		if( size ) 
		{
			int old_size;
			if( e->HaveData(tag) ) old_size = e->GetDataSize(tag);
			else old_size = 0;
			e->SetDataSize(tag,old_size+size);
			e->SetData(tag,old_size,size,data);
		}
	}
	
	void RedistUnpack(const Tag & tag,const Element & e, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
	{
		if( size ) 
		{
			Storage::integer_array p1 = e->IntegerArray(tag);
			const Storage::integer * p2 = static_cast<const Storage::integer *>(static_cast<const void *>(data));
			dynarray<Storage::integer,64> result(p1.size()+size);
			dynarray<Storage::integer,64>::iterator end;
			end = std::set_union(p1.begin(),p1.end(),p2,p2+size,result.begin());
			result.resize(end-result.begin());
			p1.clear();
			p1.insert(p1.end(),result.begin(),result.end());
		}
	}

	void UnpackLayersMarker(const Tag & tag, const Element & e, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
	{
		if( size )
		{
			int old_size;
			if( e->HaveData(tag) ) old_size = e->GetDataSize(tag);
			else old_size = 0;
			e->SetDataSize(tag,old_size+size);
			e->SetData(tag,old_size,size,data);
		}
	}
	
Kirill Terekhov's avatar
Kirill Terekhov committed
172

Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
173
	void UnpackSyncMarkerOR(const Tag & tag, const Element & element, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
Kirill Terekhov's avatar
Kirill Terekhov committed
174
	{
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
175
		(void) size;
Kirill Terekhov's avatar
Kirill Terekhov committed
176 177 178
		element->Bulk(tag) |= *data;
	}

Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
179
	void UnpackSyncMarkerXOR(const Tag & tag, const Element & element, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
Kirill Terekhov's avatar
Kirill Terekhov committed
180
	{
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
181
		(void) size;
Kirill Terekhov's avatar
Kirill Terekhov committed
182 183 184 185
		element->Bulk(tag) ^= *data;
	}

	
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
186
	void UnpackSyncMarkerAND(const Tag & tag, const Element & element, const INMOST_DATA_BULK_TYPE * data, INMOST_DATA_ENUM_TYPE size)
Kirill Terekhov's avatar
Kirill Terekhov committed
187
	{
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
188
		(void) size;
Kirill Terekhov's avatar
Kirill Terekhov committed
189 190 191
		element->Bulk(tag) &= *data;
	}

Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
192 193 194
	//////////////////////////////
	/// REDUCTION FUNCTIONS END///
	//////////////////////////////
195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210
	bool Mesh::HaveGlobalID(ElementType type) const
	{
		assert(OneType(type));
		Tag test;
		if( GlobalIDTag().isValid() ) test = GlobalIDTag();
		//else if( HaveTag("GLOBAL_ID") ) test = GetTag("GLOBAL_ID") );
		if( test.isValid() )
		{
			//bool ret = true;
			//for(ElementType etype = NODE; etype <= MESH; etype = NextElementType(etype) ) if( etype & types )
			//	ret &= test.isDefined(etype);
			//return ret;
			return test.isDefined(type);
		}
		else return false;
	}
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
211

Kirill Terekhov's avatar
Kirill Terekhov committed
212

Kirill Terekhov's avatar
Kirill Terekhov committed
213
	void Mesh::SynchronizeMarker(MarkerType marker, ElementType mask, SyncBitOp op)
Kirill Terekhov's avatar
Kirill Terekhov committed
214
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
215
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
216 217 218 219
#if defined(USE_MPI)
		if( m_state == Mesh::Parallel )
		{
			Tag t = CreateTag("TEMP_SYNC_MARKER",DATA_BULK,mask,mask,1);
Kirill Terekhov's avatar
Kirill Terekhov committed
220
			
Kirill Terekhov's avatar
Kirill Terekhov committed
221 222 223
			//workaround for old gcc compiler
			const Element::Status SGhost = Element::Ghost;
			const Element::Status SAny = Element::Any;
Kirill Terekhov's avatar
Kirill Terekhov committed
224
			Element::Status Expr = (Element::Shared | ((op != SYNC_BIT_SET) ? SGhost : SAny));
Kirill Terekhov's avatar
Kirill Terekhov committed
225
			
Kirill Terekhov's avatar
Kirill Terekhov committed
226
			for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); ++it)
Kirill Terekhov's avatar
Kirill Terekhov committed
227
				if( it->GetMarker(marker) && (it->GetStatus() & Expr) )
Kirill Terekhov's avatar
Kirill Terekhov committed
228
					it->Bulk(t) = 1;
Kirill Terekhov's avatar
Kirill Terekhov committed
229
			
Kirill Terekhov's avatar
Kirill Terekhov committed
230 231 232 233
			
			
			switch(op)
			{
Kirill Terekhov's avatar
Kirill Terekhov committed
234 235 236
				case SYNC_BIT_SET:
					ExchangeData(t,mask,0);
					for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); ++it)
Kirill Terekhov's avatar
Kirill Terekhov committed
237
					{
Kirill Terekhov's avatar
Kirill Terekhov committed
238 239 240 241
						if( it->GetStatus() == Element::Ghost )
						{
							if( it->HaveData(t) ) it->SetMarker(marker); else it->RemMarker(marker);
						}
Kirill Terekhov's avatar
Kirill Terekhov committed
242
					}
Kirill Terekhov's avatar
Kirill Terekhov committed
243 244 245 246
					break;
				case SYNC_BIT_OR:
					ReduceData(t,mask,0,UnpackSyncMarkerOR);
					for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); ++it)
Kirill Terekhov's avatar
Kirill Terekhov committed
247
					{
Kirill Terekhov's avatar
Kirill Terekhov committed
248 249 250 251
						if( it->GetStatus() & (Element::Ghost | Element::Shared) )
						{
							if( !it->GetMarker(marker) && it->HaveData(t) ) it->SetMarker(marker);
						}
Kirill Terekhov's avatar
Kirill Terekhov committed
252
					}
Kirill Terekhov's avatar
Kirill Terekhov committed
253 254 255 256
					break;
				case SYNC_BIT_AND:
					ReduceData(t,mask,0,UnpackSyncMarkerAND);
					for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); ++it)
Kirill Terekhov's avatar
Kirill Terekhov committed
257
					{
Kirill Terekhov's avatar
Kirill Terekhov committed
258 259 260 261
						if( it->GetStatus() & (Element::Ghost | Element::Shared))
						{
							if( it->HaveData(t) && it->Bulk(t) ) it->SetMarker(marker); else it->RemMarker(marker);
						}
Kirill Terekhov's avatar
Kirill Terekhov committed
262
					}
Kirill Terekhov's avatar
Kirill Terekhov committed
263 264 265 266
					break;
				case SYNC_BIT_XOR:
					ReduceData(t,mask,0,UnpackSyncMarkerXOR);
					for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); ++it)
Kirill Terekhov's avatar
Kirill Terekhov committed
267
					{
Kirill Terekhov's avatar
Kirill Terekhov committed
268 269 270 271
						if( it->GetStatus() & (Element::Ghost | Element::Shared))
						{
							if( it->HaveData(t) && it->Bulk(t) ) it->SetMarker(marker); else it->RemMarker(marker);
						}
Kirill Terekhov's avatar
Kirill Terekhov committed
272
					}
Kirill Terekhov's avatar
Kirill Terekhov committed
273
					break;
Kirill Terekhov's avatar
Kirill Terekhov committed
274 275
			}
			
Kirill Terekhov's avatar
Kirill Terekhov committed
276
			
Kirill Terekhov's avatar
Kirill Terekhov committed
277 278
			DeleteTag(t,mask);
		}
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
279 280 281 282 283
#else//USE_MPI
		(void) marker;
		(void) mask;
		(void) op;
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
284
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
285
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
286
	
Kirill Terekhov's avatar
Kirill Terekhov committed
287 288
	ElementType Mesh::SynchronizeElementType(ElementType etype)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
289
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
290 291
		ElementType etypeout = etype;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
292
		REPORT_MPI(MPI_Allreduce(&etype,&etypeout,1,INMOST_MPI_DATA_BULK_TYPE,MPI_BOR,GetCommunicator()));
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
293
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
294 295
		return etypeout;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
296 297
	Storage::integer Mesh::TotalNumberOf(ElementType mask)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
298
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
299 300 301 302 303
		Storage::integer number = 0, ret = 0;
		for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); it++)
			if( it->GetStatus() != Element::Ghost ) number++;
		ret = number;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
304
		REPORT_MPI(MPI_Allreduce(&number,&ret,1,INMOST_MPI_DATA_INTEGER_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
305
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
306
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
307 308
		return ret;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
309
	Storage::integer Mesh::EnumerateSet(const ElementSet & set, const Tag & num_tag, Storage::integer start, bool define_sparse)
Kirill Terekhov's avatar
Kirill Terekhov committed
310
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
311
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
312 313 314 315
		Storage::integer shift = 0, ret = 0;
		ElementType mask = CELL | FACE | EDGE | NODE;
#if defined(USE_MPI)
		Storage::integer number = 0;
Kirill Terekhov's avatar
Kirill Terekhov committed
316 317
		for(ElementSet::iterator it = set.Begin(); it != set.End(); it++)
			if( it->GetStatus() != Element::Ghost && (define_sparse || it->HaveData(num_tag)) )
Kirill Terekhov's avatar
Kirill Terekhov committed
318
				number++;
Kirill Terekhov's avatar
Kirill Terekhov committed
319
		REPORT_MPI(MPI_Scan(&number,&shift,1,INMOST_MPI_DATA_INTEGER_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
320
		shift -= number;
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
321
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
322
		shift += start;
Kirill Terekhov's avatar
Kirill Terekhov committed
323 324
		for(ElementSet::iterator it = set.Begin(); it != set.End(); it++)
			if( it->GetStatus() == Element::Owned && (define_sparse || it->HaveData(num_tag)) )
Kirill Terekhov's avatar
Kirill Terekhov committed
325
				it->Integer(num_tag) = shift++;
Kirill Terekhov's avatar
Kirill Terekhov committed
326 327
		for(ElementSet::iterator it = set.Begin(); it != set.End(); it++)
			if( it->GetStatus() == Element::Shared && (define_sparse || it->HaveData(num_tag)) )
Kirill Terekhov's avatar
Kirill Terekhov committed
328
				it->Integer(num_tag) = shift++;
Kirill Terekhov's avatar
Kirill Terekhov committed
329
		ExchangeData(num_tag,mask,0);
Kirill Terekhov's avatar
Kirill Terekhov committed
330 331
		ret = shift;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
332
		REPORT_MPI(MPI_Bcast(&ret,1,INMOST_MPI_DATA_INTEGER_TYPE,GetProcessorsNumber()-1,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
333
		//MPI_Allreduce(&shift,&ret,1,INMOST_DATA_INTEGER_TYPE,MPI_MAX,comm);
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
334
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
335
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
336 337
		return ret;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
338
	Storage::integer Mesh::Enumerate(const HandleType * set, enumerator n, const Tag & num_tag, Storage::integer start, bool define_sparse)
Kirill Terekhov's avatar
Kirill Terekhov committed
339
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
340
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
341 342 343 344
		Storage::integer shift = 0, ret = 0;
		ElementType mask = CELL | FACE | EDGE | NODE;
#if defined(USE_MPI)
		Storage::integer number = 0;
Kirill Terekhov's avatar
Kirill Terekhov committed
345 346
		for(const HandleType * it = set; it != set+n; ++it)
			if( GetStatus(*it) != Element::Ghost && (define_sparse || HaveData(*it,num_tag))) number++;
Kirill Terekhov's avatar
Kirill Terekhov committed
347
		REPORT_MPI(MPI_Scan(&number,&shift,1,INMOST_MPI_DATA_INTEGER_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
348
		shift -= number;
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
349
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
350
		shift += start;
Kirill Terekhov's avatar
Kirill Terekhov committed
351
		for(const HandleType * it = set; it != set+n; ++it)
352
			if( GetStatus(*it) == Element::Owned && (define_sparse || HaveData(*it,num_tag))) Integer(*it,num_tag) = shift++;
Kirill Terekhov's avatar
Kirill Terekhov committed
353
		for(const HandleType * it = set; it != set+n; ++it)
354
			if( GetStatus(*it) == Element::Shared && (define_sparse || HaveData(*it,num_tag))) Integer(*it,num_tag) = shift++;
Kirill Terekhov's avatar
Kirill Terekhov committed
355
		ExchangeData(num_tag,mask,0);
Kirill Terekhov's avatar
Kirill Terekhov committed
356 357
		ret = shift;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
358
		REPORT_MPI(MPI_Bcast(&ret,1,INMOST_MPI_DATA_INTEGER_TYPE,GetProcessorsNumber()-1,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
359
		//MPI_Allreduce(&shift,&ret,1,INMOST_DATA_INTEGER_TYPE,MPI_MAX,comm);
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
360
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
361
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
362 363
		return ret;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
364
	Storage::integer Mesh::Enumerate(ElementType mask, Tag num_tag, Storage::integer start, bool define_sparse)
Kirill Terekhov's avatar
Kirill Terekhov committed
365
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
366
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
367 368 369 370
		Storage::integer shift = 0, ret = 0;
#if defined(USE_MPI)
		Storage::integer number = 0;
		for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); it++)
Kirill Terekhov's avatar
Kirill Terekhov committed
371
			if( it->GetStatus() != Element::Ghost && (define_sparse || it->HaveData(num_tag)) )
Kirill Terekhov's avatar
Kirill Terekhov committed
372
				number++;
Kirill Terekhov's avatar
Kirill Terekhov committed
373
		REPORT_MPI(MPI_Scan(&number,&shift,1,INMOST_MPI_DATA_INTEGER_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
374
		shift -= number;
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
375
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
376 377
		shift += start;
		for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); it++)
378
			if( it->GetStatus() == Element::Owned && (define_sparse || it->HaveData(num_tag)) )
Kirill Terekhov's avatar
Kirill Terekhov committed
379 380
				it->Integer(num_tag) = shift++;
		for(Mesh::iteratorElement it = BeginElement(mask); it != EndElement(); it++)
381
			if( it->GetStatus() == Element::Shared && (define_sparse || it->HaveData(num_tag)) )
Kirill Terekhov's avatar
Kirill Terekhov committed
382
				it->Integer(num_tag) = shift++;
Kirill Terekhov's avatar
Kirill Terekhov committed
383
		ExchangeData(num_tag,mask,0);
Kirill Terekhov's avatar
Kirill Terekhov committed
384 385
		ret = shift;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
386
		REPORT_MPI(MPI_Bcast(&ret,1,INMOST_MPI_DATA_INTEGER_TYPE,GetProcessorsNumber()-1,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
387
		//MPI_Allreduce(&shift,&ret,1,INMOST_DATA_INTEGER_TYPE,MPI_MAX,comm);
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
388
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
389
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
390 391 392 393 394
		return ret;
	}
	
	Storage::real Mesh::Integrate(Storage::real input)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
395
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
396 397
		Storage::real output = input;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
398
		REPORT_MPI(MPI_Allreduce(&input,&output,1,INMOST_MPI_DATA_REAL_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
399 400 401
#else//USE_MPI
		(void) input;
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
402
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
403 404
		return output;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
405 406
	
	
Kirill Terekhov's avatar
Kirill Terekhov committed
407
	
Kirill Terekhov's avatar
Kirill Terekhov committed
408 409 410
	
	Storage::integer Mesh::Integrate(Storage::integer input)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
411
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
412 413
		Storage::integer output = input;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
414
		REPORT_MPI(MPI_Allreduce(&input,&output,1,INMOST_MPI_DATA_INTEGER_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
415 416 417
#else//USE_MPI
		(void) input;
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
418
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
419 420
		return output;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
421
	
Kirill Terekhov's avatar
Kirill Terekhov committed
422 423
	void Mesh::Integrate(Storage::real * input, Storage::integer size)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
424
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
425
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
426 427
		static dynarray<Storage::real,64> temp;
		temp.resize(size);
Kirill Terekhov's avatar
Kirill Terekhov committed
428
		memcpy(temp.data(),input,sizeof(Storage::real)*size);
Kirill Terekhov's avatar
Kirill Terekhov committed
429
		REPORT_MPI(MPI_Allreduce(temp.data(),input,size,INMOST_MPI_DATA_REAL_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
430
#else//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
431 432
		(void) input;
		(void) size;
Kirill Terekhov's avatar
Kirill Terekhov committed
433
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
434
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
435
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
436
	
Kirill Terekhov's avatar
Kirill Terekhov committed
437 438
	void Mesh::Integrate(Storage::integer * input, Storage::integer size)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
439
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
440
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
441 442
		static dynarray<Storage::integer,64> temp;
		temp.resize(size);
Kirill Terekhov's avatar
Kirill Terekhov committed
443
		memcpy(temp.data(),input,sizeof(Storage::integer)*size);
Kirill Terekhov's avatar
Kirill Terekhov committed
444
		REPORT_MPI(MPI_Allreduce(temp.data(),input,size,INMOST_MPI_DATA_INTEGER_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
445 446
#else//USE_MPI
		(void) input;
Kirill Terekhov's avatar
Kirill Terekhov committed
447
		(void) size;
Kirill Terekhov's avatar
Kirill Terekhov committed
448
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
449
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
450
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
451 452 453
	
	Storage::integer Mesh::ExclusiveSum(Storage::integer input)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
454
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
455 456
		Storage::integer output = 0;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
457
		REPORT_MPI(MPI_Scan(&input,&output,1,INMOST_MPI_DATA_INTEGER_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
458
		output -= input;
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
459 460 461
#else//USE_MPI
		(void) input;
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
462
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
463 464 465
		return output;
	}
	
Kirill Terekhov's avatar
Kirill Terekhov committed
466
	Storage::real Mesh::Integrate(const Tag & t, enumerator entry, ElementType mask)
Kirill Terekhov's avatar
Kirill Terekhov committed
467
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
468
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
469 470
		Storage::real output = 0, input = 0;
		for(iteratorElement it = BeginElement(mask); it != EndElement(); it++)
Kirill Terekhov's avatar
Kirill Terekhov committed
471
			if( GetStatus(*it) != Element::Ghost && HaveData(*it,t) )
Kirill Terekhov's avatar
Kirill Terekhov committed
472 473 474 475
			{
				real_array arr = RealArray(*it,t);
				if( arr.size() > entry ) input += arr[entry];
			}
Kirill Terekhov's avatar
Kirill Terekhov committed
476 477
		output = input;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
478
		REPORT_MPI(MPI_Allreduce(&input,&output,1,INMOST_MPI_DATA_REAL_TYPE,MPI_SUM,comm));
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
479
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
480
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
481 482 483
		return output;
	}
	
484 485
	Storage::real Mesh::AggregateMax(Storage::real input)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
486
		ENTER_FUNC();
487 488
		Storage::real output = input;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
489
		REPORT_MPI(MPI_Allreduce(&input,&output,1,INMOST_MPI_DATA_REAL_TYPE,MPI_MAX,comm));
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
490 491 492
#else //USE_MPI
		(void) input;
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
493
		EXIT_FUNC();
494 495
		return output;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
496
	
497 498
	Storage::integer Mesh::AggregateMax(Storage::integer input)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
499
		ENTER_FUNC();
500 501
		Storage::integer output = input;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
502
		REPORT_MPI(MPI_Allreduce(&input,&output,1,INMOST_MPI_DATA_INTEGER_TYPE,MPI_MAX,comm));
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
503 504 505
#else //USE_MPI
		(void) input;
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
506
		EXIT_FUNC();
507 508
		return output;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
509
	
Kirill Terekhov's avatar
Kirill Terekhov committed
510 511
	void Mesh::AggregateMax(Storage::real * input, Storage::integer size)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
512
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
513
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
514 515
		static dynarray<Storage::real,64> temp;
		temp.resize(size);
Kirill Terekhov's avatar
Kirill Terekhov committed
516
		memcpy(temp.data(),input,sizeof(Storage::real)*size);
Kirill Terekhov's avatar
Kirill Terekhov committed
517
		REPORT_MPI(MPI_Allreduce(temp.data(),input,size,INMOST_MPI_DATA_REAL_TYPE,MPI_MAX,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
518 519
#else//USE_MPI
		(void) input;
Kirill Terekhov's avatar
Kirill Terekhov committed
520
		(void) size;
Kirill Terekhov's avatar
Kirill Terekhov committed
521
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
522
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
523
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
524
	
Kirill Terekhov's avatar
Kirill Terekhov committed
525 526
	void Mesh::AggregateMax(Storage::integer * input, Storage::integer size)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
527
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
528
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
529 530
		static dynarray<Storage::integer,64> temp;
		temp.resize(size);
Kirill Terekhov's avatar
Kirill Terekhov committed
531
		memcpy(temp.data(),input,sizeof(Storage::integer)*size);
Kirill Terekhov's avatar
Kirill Terekhov committed
532
		REPORT_MPI(MPI_Allreduce(temp.data(),input,size,INMOST_MPI_DATA_INTEGER_TYPE,MPI_MAX,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
533 534
#else//USE_MPI
		(void) input;
Kirill Terekhov's avatar
Kirill Terekhov committed
535
		(void) size;
Kirill Terekhov's avatar
Kirill Terekhov committed
536
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
537
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
538
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
539
	
Kirill Terekhov's avatar
Kirill Terekhov committed
540 541
	Storage::real Mesh::AggregateMin(Storage::real input)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
542
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
543 544
		Storage::real output = input;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
545
		REPORT_MPI(MPI_Allreduce(&input,&output,1,INMOST_MPI_DATA_REAL_TYPE,MPI_MIN,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
546 547 548
#else //USE_MPI
		(void) input;
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
549
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
550 551 552 553 554
		return output;
	}
	
	Storage::integer Mesh::AggregateMin(Storage::integer input)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
555
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
556 557
		Storage::integer output = input;
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
558
		REPORT_MPI(MPI_Allreduce(&input,&output,1,INMOST_MPI_DATA_INTEGER_TYPE,MPI_MIN,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
559 560 561
#else //USE_MPI
		(void) input;
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
562
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
563 564
		return output;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
565
	
Kirill Terekhov's avatar
Kirill Terekhov committed
566 567
	void Mesh::AggregateMin(Storage::real * input, Storage::integer size)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
568
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
569
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
570 571
		static dynarray<Storage::real,64> temp;
		temp.resize(size);
Kirill Terekhov's avatar
Kirill Terekhov committed
572
		memcpy(temp.data(),input,sizeof(Storage::real)*size);
Kirill Terekhov's avatar
Kirill Terekhov committed
573
		REPORT_MPI(MPI_Allreduce(temp.data(),input,size,INMOST_MPI_DATA_REAL_TYPE,MPI_MIN,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
574 575
#else//USE_MPI
		(void) input;
Kirill Terekhov's avatar
Kirill Terekhov committed
576
		(void) size;
Kirill Terekhov's avatar
Kirill Terekhov committed
577
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
578
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
579
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
580
	
Kirill Terekhov's avatar
Kirill Terekhov committed
581 582
	void Mesh::AggregateMin(Storage::integer * input, Storage::integer size)
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
583
		ENTER_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
584
#if defined(USE_MPI)
Kirill Terekhov's avatar
Kirill Terekhov committed
585 586
		static dynarray<Storage::integer,64> temp;
		temp.resize(size);
Kirill Terekhov's avatar
Kirill Terekhov committed
587
		memcpy(temp.data(),input,sizeof(Storage::integer)*size);
Kirill Terekhov's avatar
Kirill Terekhov committed
588
		REPORT_MPI(MPI_Allreduce(temp.data(),input,size,INMOST_MPI_DATA_INTEGER_TYPE,MPI_MIN,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
589 590
#else//USE_MPI
		(void) input;
Kirill Terekhov's avatar
Kirill Terekhov committed
591
		(void) size;
Kirill Terekhov's avatar
Kirill Terekhov committed
592
#endif//USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
593
		EXIT_FUNC();
Kirill Terekhov's avatar
Kirill Terekhov committed
594
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
595 596
	
	
Kirill Terekhov's avatar
Kirill Terekhov committed
597
	INMOST_MPI_Comm Mesh::GetCommunicator() const
Kirill Terekhov's avatar
Kirill Terekhov committed
598 599 600
	{
#if defined(USE_MPI)
		return comm;
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
601
#else //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
602
		return 0;
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
603
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
604 605
	}
	
Kirill Terekhov's avatar
Kirill Terekhov committed
606
	int Mesh::GetProcessorRank() const
Kirill Terekhov's avatar
Kirill Terekhov committed
607 608 609 610 611
	{
#if defined(USE_MPI)
		int rank;
		MPI_Comm_rank(comm,&rank);
		return rank;
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
612
#else //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
613
		return 0;
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
614
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
615 616
	}
	
Kirill Terekhov's avatar
Kirill Terekhov committed
617
	int Mesh::GetProcessorsNumber() const
Kirill Terekhov's avatar
Kirill Terekhov committed
618 619 620 621 622
	{
#if defined(USE_MPI)
		int size;
		MPI_Comm_size(comm,&size);
		return size;
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
623
#else //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
624
		return 1;
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
625
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
626 627 628 629 630 631 632 633
	}	
	
	void Mesh::Initialize(int * argc, char *** argv)
	{
#if defined(USE_MPI)
		int test;
		MPI_Initialized(&test);
        if( test == 0 ) MPI_Init(argc,argv);
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
634 635 636 637
#else //USE_MPI
		(void) argc;
		(void) argv;
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
638 639 640
#if defined(USE_PARALLEL_WRITE_TIME)
		atexit(Mesh::AtExit);
#endif
Kirill Terekhov's avatar
Kirill Terekhov committed
641 642 643 644 645 646 647 648
	}
	void Mesh::Finalize()
	{
#if defined(USE_MPI)
		int test = 0;
		MPI_Finalized(&test);
		if( !test )
			MPI_Finalize();
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
649
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
650 651
	}
	
Kirill Terekhov's avatar
Kirill Terekhov committed
652
    INMOST_MPI_Group Mesh::GetGroup() const
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
653 654 655 656 657 658 659
    {
        INMOST_MPI_Group ret = INMOST_MPI_GROUP_EMPTY;
#if defined(USE_MPI)
        MPI_Comm_group(GetCommunicator(), &ret);
#endif
        return ret;
    }
Kirill Terekhov's avatar
Kirill Terekhov committed
660 661 662 663
	
	void Mesh::SetCommunicator(INMOST_MPI_Comm _comm)
	{
		ENTER_FUNC();
664 665 666
		tag_shared = CreateTag("PROTECTED_STATUS",DATA_BULK,  ESET |CELL | FACE | EDGE | NODE,NONE,1);
		tag_owner = CreateTag("OWNER_PROCESSOR",DATA_INTEGER, ESET | CELL | FACE | EDGE | NODE,NONE,1);
		tag_processors = CreateTag("PROCESSORS_LIST",DATA_INTEGER, ESET|  MESH | NODE | EDGE | FACE | CELL,NONE);
Kirill Terekhov's avatar
Kirill Terekhov committed
667 668
		tag_layers = CreateTag("LAYERS",DATA_INTEGER,MESH,NONE,1);
		tag_bridge = CreateTag("BRIDGE",DATA_INTEGER,MESH,NONE,1);
669 670
		tag_sendto = CreateTag("PROTECTED_SENDTO",DATA_INTEGER, ESET | CELL | FACE | EDGE | NODE, ESET | CELL | FACE | EDGE | NODE);
		tag_an_id = CreateTag("PROTECTED_ID",DATA_INTEGER, CELL | FACE | EDGE | NODE, CELL | FACE | EDGE | NODE);
Kirill Terekhov's avatar
Kirill Terekhov committed
671 672

#if defined(USE_MPI)
673
		randomizer = Random();
Kirill Terekhov's avatar
Kirill Terekhov committed
674 675 676 677 678 679 680 681 682 683 684
		
		parallel_strategy = 1;
		parallel_file_strategy = 1;

		

		Mesh::Initialize(NULL,NULL);
		//~ MPI_Comm_dup(_comm,&comm);
		comm = _comm;
		{
			INMOST_DATA_BIG_ENUM_TYPE t = pmid;
685
			REPORT_MPI(MPI_Allreduce(&t,&parallel_mesh_unique_id,1,INMOST_MPI_DATA_BIG_ENUM_TYPE,MPI_MAX,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
686 687 688 689 690
			pmid = parallel_mesh_unique_id+1;
		}
		m_state = Mesh::Parallel;


Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
691
#if defined(USE_MPI_P2P)
Kirill Terekhov's avatar
Kirill Terekhov committed
692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709
		int err;
		REPORT_MPI(err = MPI_Alloc_mem(GetProcessorsNumber()*sizeof(unsigned)*2,MPI_INFO_NULL,&shared_space));
		if( err )
		{
			int errclass;
			MPI_Error_class(err,&errclass);
			std::cout << "Cannot allocate shared space of size " << GetProcessorsNumber()*sizeof(unsigned)*2 << " reason is "; 
			switch(err)
			{
				case MPI_SUCCESS: std::cout << "success"; break;
				case MPI_ERR_INFO: std::cout << "bad info"; break;
				case MPI_ERR_ARG: std::cout << "bad argument"; break;
				case MPI_ERR_NO_MEM: std::cout << "no memory"; break;
			}
			std::cout << std::endl;
			MPI_Abort(comm,err);
		}
		REPORT_MPI(MPI_Win_create(shared_space,sizeof(unsigned)*GetProcessorsNumber()*2,sizeof(unsigned),MPI_INFO_NULL,comm,&window));
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
710 711 712 713
#endif //USE_MPI_P2P
#else //USE_MPI
		(void) _comm;
#endif //USE_MPI
Kirill Terekhov's avatar
Kirill Terekhov committed
714 715 716 717 718 719
		EXIT_FUNC();
	}
	
#if defined(USE_PARALLEL_WRITE_TIME)
	void Mesh::Enter() { tab++; }
	void Mesh::Exit() {tab--; }
Kirill Terekhov's avatar
Kirill Terekhov committed
720
	std::ostream & Mesh::WriteTab(std::ostream & f)
Kirill Terekhov's avatar
Kirill Terekhov committed
721 722 723 724 725 726 727 728 729
	{
		for(int i = 0; i < tab; i++)
			f << "   ";
		return f;
	}
	std::fstream & Mesh::GetStream()
	{
		return out_time;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
730 731
	void Mesh::FinalizeFile()
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
732 733
    //std::stringstream str;
    if( tab > 1 )
Kirill Terekhov's avatar
Kirill Terekhov committed
734
		{
Kirill Terekhov's avatar
Kirill Terekhov committed
735
      out_time << "<TEXT><![CDATA[Died!]]></TEXT>\n";// << std::endl;
Kirill Terekhov's avatar
Kirill Terekhov committed
736
		}
Kirill Terekhov's avatar
Kirill Terekhov committed
737
    while(tab > 1)
Kirill Terekhov's avatar
Kirill Terekhov committed
738
		{
Kirill Terekhov's avatar
Kirill Terekhov committed
739 740
      out_time << "<TIME>-1</TIME>\n</FUNCTION>\n";// << std::endl; 
      Exit(); 
Kirill Terekhov's avatar
Kirill Terekhov committed
741 742
		}
		out_time << "</Debug>" << std::endl;
Kirill Terekhov's avatar
Kirill Terekhov committed
743 744 745
    //out_time << str;
    //out_time.flush();
    out_time.close();
Kirill Terekhov's avatar
Kirill Terekhov committed
746
	}
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
747
#endif //USE_PARALLEL_WRITE_TIME
Kirill Terekhov's avatar
Kirill Terekhov committed
748 749 750
	
	
	
Kirill Terekhov's avatar
Kirill Terekhov committed
751
	void determine_my_procs_low(Mesh * m, HandleType h, dynarray<Storage::integer,64> & result, dynarray<Storage::integer,64> & intersection)
Kirill Terekhov's avatar
Kirill Terekhov committed
752
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
753 754 755
		Element::adj_type const & subelements = m->LowConn(h);
		Element::adj_type::const_iterator i = subelements.begin();
		Storage::integer_array p = m->IntegerArrayDV(*i,m->ProcessorsTag());
Kirill Terekhov's avatar
Kirill Terekhov committed
756 757 758 759 760
		result.clear();
		result.insert(result.end(),p.begin(),p.end());
		i++;
		while(i != subelements.end())
		{
Kirill Terekhov's avatar
Kirill Terekhov committed
761 762
			Storage::integer_array q = m->IntegerArrayDV(*i,m->ProcessorsTag());
			intersection.resize(std::max(static_cast<unsigned>(result.size()),static_cast<unsigned>(q.size())));
Kirill Terekhov's avatar
Kirill Terekhov committed
763 764 765 766 767 768 769
			dynarray<Storage::integer,64>::iterator qt = std::set_intersection(result.begin(),result.end(),q.begin(),q.end(),intersection.begin());
			intersection.resize(qt-intersection.begin());
			result.swap(intersection);
			i++;
		}
	}
	
Kirill Terekhov's avatar
Kirill Terekhov committed
770
	void determine_my_procs_high(Mesh * m, HandleType h, const Tag & procs, dynarray<Storage::integer,64> & result, dynarray<Storage::integer,64> & intersection)
Kirill Terekhov's avatar
Kirill Terekhov committed
771
	{
Kirill Terekhov's avatar
Kirill Terekhov committed
772
		Element::adj_type const & overelements = m->HighConn(h);
Kirill Terekhov's avatar
Kirill Terekhov committed
773
		if( overelements.empty() ) return;
Kirill Terekhov's avatar
Kirill Terekhov committed
774
		Element::adj_type::const_iterator i = overelements.begin();
Kirill Terekhov's avatar
Kirill Terekhov committed
775 776 777
		result.clear();
		while(i != overelements.end())
		{
Kirill Terekhov's avatar
Kirill Terekhov committed
778
			Storage::integer_array q = m->IntegerArrayDV(*i,procs);
Kirill Terekhov's avatar
Kirill Terekhov committed
779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874
			intersection.resize(result.size()+q.size());
			dynarray<Storage::integer,64>::iterator qt = std::set_union(result.begin(),result.end(),q.begin(),q.end(),intersection.begin());
			intersection.resize(qt-intersection.begin());
			result.swap(intersection);
			i++;
		}
	}
	
	/*
	void determine_my_procs_low(adjacent<Element> & subelements,Storage::integer_array & my_procs,Tag & procs_tag)
	{
		if( subelements.empty() ) return;
		adjacent<Element>::iterator i = subelements.begin();
		std::vector<Storage::integer> result;
		Storage::integer_array p = i->IntegerArray(procs_tag);
		result.insert(result.begin(),p.begin(),p.end());
		i++;
		while(i != subelements.end())
		{
			Storage::integer_array q = i->IntegerArray(procs_tag);
			std::vector<Storage::integer> intersection(result.size());
			std::vector<Storage::integer>::iterator qt = std::set_intersection(result.begin(),result.end(),q.begin(),q.end(),intersection.begin());
			intersection.resize(qt-intersection.begin());
			result.swap(intersection);
			i++;
		}
		my_procs.clear();
		my_procs.insert(my_procs.begin(),result.begin(),result.end());
	}
	
	void determine_my_procs_high(adjacent<Element> & overelements,Storage::integer_array & my_procs,Tag & procs_tag)
	{
		if( overelements.empty() ) return;
		adjacent<Element>::iterator i = overelements.begin();
		std::vector<Storage::integer> result;
		Storage::integer_array p = i->IntegerArray(procs_tag);
		result.insert(result.begin(),p.begin(),p.end());
		i++;
		while(i != overelements.end())
		{
			Storage::integer_array q = i->IntegerArray(procs_tag);
			std::vector<Storage::integer> intersection(result.size()+q.size());
			std::vector<Storage::integer>::iterator qt = std::set_union(result.begin(),result.end(),q.begin(),q.end(),intersection.begin());
			intersection.resize(qt-intersection.begin());
			result.swap(intersection);
			i++;
		}
		my_procs.clear();
		my_procs.insert(my_procs.begin(),result.begin(),result.end());
	}
	*/
	/*
	void Mesh::MarkShared(ElementType mask)
	{
		for(ElementType etype = NODE; etype <= CELL; etype = etype << 1 ) if( etype & mask )
		for(Mesh::iteratorElement it = BeginElement(etype); it != EndElement(); it++)
		{
			int owner;
			Storage::integer_array v = it->IntegerArrayDV(tag_processors);
			std::sort(v.begin(),v.end());
			if( v.empty() )
			{
				owner = mpirank;
				v.push_back(mpirank);
			}
			else
				owner = std::min(mpirank,v[0]);
			
			it->IntegerDF(tag_owner) = owner;
			
			if( mpirank == owner )
			{
				if( v.size() == 1 )
					it->BulkDF(tag_shared) = Element::Owned;
				else
					it->BulkDF(tag_shared) = Element::Shared;
			}
			else
				it->BulkDF(tag_shared) = Element::Ghost;
		}
		
		ComputeSharedProcs();
		RecomputeParallelStorage(mask);
		AssignGlobalID(mask);
		
		//have to do it for all types
#if defined(USE_PARALLEL_STORAGE)
		for(parallel_storage::iterator it = shared_elements.begin(); it != shared_elements.end(); it++)
			qsort(&it->second[0][0],it->second[0].size(),sizeof(Element *),CompareElementsCGID);
		for(parallel_storage::iterator it = ghost_elements.begin(); it != ghost_elements.end(); it++)			
			qsort(&it->second[0][0],it->second[0].size(),sizeof(Element *),CompareElementsCGID);
#endif
	}
	*/
	
	
875
	__INLINE bool point_in_bbox(Storage::real * p, Storage::real bbox[6], unsigned dim, Storage::real eps)
Kirill Terekhov's avatar
Kirill Terekhov committed
876 877 878
	{
		bool ret = true;
		for(unsigned k = 0; k < dim; k++) 
879
			ret &= (p[k] > bbox[k]-eps && p[k] < bbox[dim+k]+eps);
Kirill Terekhov's avatar
Kirill Terekhov committed
880 881 882 883 884 885
		return ret;
	}
	
	__INLINE bool compare_coord(Storage::real * a, Storage::real * b, INMOST_DATA_ENUM_TYPE dim, Storage::real eps)
	{
		for(INMOST_DATA_ENUM_TYPE i = 0; i <  dim; i++)
Kirill Terekhov's avatar
Kirill Terekhov committed
886
			if( ::fabs(a[i]-b[i]) > eps ) return a[i] <= b[i];
Kirill Terekhov's avatar
Kirill Terekhov committed
887 888
		return true;
	}
Kirill Terekhov's avatar
Kirill Terekhov committed
889 890 891 892 893

	class MappingComparator
	{
	public: bool operator () (const std::pair<int,int> & a, const std::pair<int,int> & b) {return a.first < b.first;}
	};
894
    
Kirill Terekhov's avatar
Kirill Terekhov committed
895
	
896 897
	
	void Mesh::ResolveShared(bool only_new)
Kirill Terekhov's avatar
Kirill Terekhov committed
898 899
	{
		ENTER_FUNC();
900
		
Kirill Terekhov's avatar
Kirill Terekhov committed
901 902
#if defined(USE_MPI)
		if( m_state == Mesh::Serial ) SetCommunicator(INMOST_MPI_COMM_WORLD);
903
		if( tag_global_id.isValid() ) tag_global_id = DeleteTag(tag_global_id,CELL | EDGE | FACE | NODE);
Kirill Terekhov's avatar
Kirill Terekhov committed
904
		integer dim = GetDimensions();
Kirill Terekhov's avatar
Kirill Terekhov committed
905 906
		int sendsize;
		int mpirank = GetProcessorRank(),mpisize = GetProcessorsNumber();
907
        int rank = mpirank;
Kirill Terekhov's avatar
Kirill Terekhov committed
908 909 910
#if defined(USE_PARALLEL_STORAGE)
		shared_elements.clear();
		ghost_elements.clear();
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
911
#endif //USE_PARALLEL_STORAGE
Kirill Terekhov's avatar
Kirill Terekhov committed
912 913
		//determine which bboxes i intersect
		dynarray<int,64> procs;
Kirill Terekhov's avatar
Kirill Terekhov committed
914 915 916 917 918
		Storage::real bbox[6]; //local bounding box
		std::vector<Storage::real> bboxs(mpisize*6);
		//Compute local bounding box containing nodes.
		//Will be more convinient to compute (or store)
		//and communicate local octree over all the nodes.
Kirill Terekhov's avatar
Kirill Terekhov committed
919
		for(integer k = 0; k < dim; k++)
Kirill Terekhov's avatar
Kirill Terekhov committed
920 921 922 923
		{
			bbox[k] = 1e20;
			bbox[k+dim] = -1e20;
		}
Kirill Terekhov's avatar
Kirill Terekhov committed
924 925 926
#if defined(USE_OMP)
#pragma omp parallel
#endif
Kirill Terekhov's avatar
Kirill Terekhov committed
927
		{
Kirill Terekhov's avatar
Kirill Terekhov committed
928
			real bbox0[6];
Kirill Terekhov's avatar
Kirill Terekhov committed
929
			for(integer k = 0; k < dim; k++)
Kirill Terekhov's avatar
Kirill Terekhov committed
930
			{
Kirill Terekhov's avatar
Kirill Terekhov committed
931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953
				bbox0[k] = 1e20;
				bbox0[k+dim] = -1e20;
			}
#if defined(USE_OMP)
#pragma omp for
#endif
			for(integer nit = 0; nit < NodeLastLocalID(); ++nit) if( isValidNode(nit) )
			{
				Node it = NodeByLocalID(nit);
				Storage::real_array arr = it->Coords();
				for(integer k = 0; k < dim; k++)
				{
					if( arr[k] < bbox0[k] ) bbox0[k] = arr[k];
					if( arr[k] > bbox0[k+dim] ) bbox0[k+dim] = arr[k];
				}
			}
#if defined(USE_OMP)
#pragma omp critical
#endif
			{
				for(integer k = 0; k < dim; k++)
				{
					if( bbox0[k] < bbox[k] ) bbox[k] = bbox0[k];
Kirill Terekhov's avatar
Kirill Terekhov committed
954
					if( bbox0[k+dim] > bbox[k+dim] ) bbox[k+dim] = bbox0[k+dim];
Kirill Terekhov's avatar
Kirill Terekhov committed
955
				}
Kirill Terekhov's avatar
Kirill Terekhov committed
956 957
			}
		}
Kirill Terekhov's avatar
Kirill Terekhov committed
958
		REPORT_VAL("dim",dim);
959
		assert(dim*2 <= 6);
Kirill Terekhov's avatar
Kirill Terekhov committed
960
		// write down bounding boxes
Kirill Terekhov's avatar
Kirill Terekhov committed
961
		for(integer k = 0; k < dim; k++)
Kirill Terekhov's avatar
Kirill Terekhov committed
962 963 964 965
		{
			REPORT_VAL("min",bbox[k]);
			REPORT_VAL("max",bbox[dim+k]);
		}
Kirill Terekhov's avatar
Kirill Terekhov committed
966
		// communicate bounding boxes
Kirill Terekhov's avatar
Kirill Terekhov committed
967
		REPORT_MPI(MPI_Allgather(&bbox[0],dim*2,INMOST_MPI_DATA_REAL_TYPE,&bboxs[0],dim*2,INMOST_MPI_DATA_REAL_TYPE,comm));
Kirill Terekhov's avatar
Kirill Terekhov committed
968
		// find all processors that i communicate with
Kirill Terekhov's avatar
Kirill Terekhov committed
969 970 971 972
		for(int k = 0; k < mpisize; k++)
			if( k != mpirank )
			{
				bool flag = true;
Kirill Terekhov's avatar
Kirill Terekhov committed
973
				for(integer q = 0; q < dim; q++)
974
					flag &= !((bbox[q]-GetEpsilon() > bboxs[k*dim*2+q+dim]) || (bbox[dim+q]+GetEpsilon() < bboxs[k*dim*2+q]));
Kirill Terekhov's avatar
Kirill Terekhov committed
975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997
				if( flag ) procs.push_back(k);
			}
		REPORT_VAL("neighbour processors",procs.size());
		
		//~ if( procs.empty() )
		//~ {
			//~ REPORT_STR("no processors around - all elements are owned");
			//~ for(Mesh::iteratorElement it = BeginElement(CELL | EDGE | FACE | NODE); it != EndElement(); it++)
			//~ {
				//~ it->IntegerArrayDV(tag_processors).resize(1);
				//~ it->IntegerDF(tag_owner) = it->IntegerDV(tag_processors) = mpirank;
				//~ it->BulkDF(tag_shared) = Element::Owned;
			//~ }
			//~ ComputeSharedProcs();
			//~ RecomputeParallelStorage(CELL | EDGE | FACE | NODE);
			//~ AssignGlobalID(CELL | EDGE | FACE | NODE);
		//~ }
		//~ else
		{
			bool same_boxes = true, same_box;
			for(int k = 0; k < mpisize && same_boxes; k++)
			{
				same_box = true;
Kirill Terekhov's avatar
Kirill Terekhov committed
998
				for(integer j = 0; j < dim*2; j++)
999
					same_box &= ::fabs(bbox[j] - bboxs[k*dim*2+j]) < GetEpsilon();
Kirill Terekhov's avatar
Kirill Terekhov committed
1000 1001 1002 1003 1004 1005
				same_boxes &= same_box;
			}
			
			if( same_boxes )
			{
				REPORT_STR("All bounding boxes are the same - assuming that mesh is replicated over all nodes");
Kirill Terekhov's avatar
Kirill Terekhov committed
1006 1007
				//for(Mesh::iteratorElement it = BeginElement(CELL | EDGE | FACE | NODE); it != EndElement(); it++)
				for(ElementType etype = NODE; etype <= CELL; etype = NextElementType(etype) )
Kirill Terekhov's avatar
Kirill Terekhov committed
1008
				{
Kirill Terekhov's avatar
Kirill Terekhov committed
1009 1010 1011 1012 1013 1014
#if defined(USE_OMP)
#pragma omp parallel for
#endif
					for(integer eit = 0; eit < LastLocalID(etype); ++eit) if( isValidElement(etype,eit) )
					{
						Element it = ElementByLocalID(etype,eit);
1015 1016
                        if (only_new && GetMarker(it->GetHandle(),NewMarker()) == false) continue;
                        
Kirill Terekhov's avatar
Kirill Terekhov committed
1017 1018 1019 1020 1021 1022 1023 1024 1025 1026
						integer_array arr = it->IntegerArrayDV(tag_processors);
						arr.resize(mpisize);
						for(int k = 0; k < mpisize; k++)
							arr[k] = k;
						it->IntegerDF(tag_owner) = 0;
						if( mpirank == 0 )
							SetStatus(it->GetHandle(),Element::Shared);
						else
							SetStatus(it->GetHandle(),Element::Ghost);
					}
Kirill Terekhov's avatar
Kirill Terekhov committed
1027 1028 1029 1030 1031 1032 1033 1034 1035
				}
				ComputeSharedProcs();
				RecomputeParallelStorage(CELL | EDGE | FACE | NODE);
				AssignGlobalID(CELL | EDGE | FACE | NODE);
#if defined(USE_PARALLEL_STORAGE)
				for(parallel_storage::iterator it = shared_elements.begin(); it != shared_elements.end(); it++)
					for(int i = 0; i < 4; i++)
					{
						if( !it->second[i].empty() )
Kirill Terekhov's avatar
Kirill Terekhov committed
1036 1037
							std::sort(it->second[i].begin(),it->second[i].end(),GlobalIDComparator(this));
							//qsort(&it->second[i][0],it->second[i].size(),sizeof(Element *),CompareElementsCGID);
Kirill Terekhov's avatar
Kirill Terekhov committed
1038
					}
Kirill Terekhov's avatar
Kirill Terekhov committed
1039
				for(parallel_storage::iterator it = ghost_elements.begin(); it != ghost_elements.end(); it++)
Kirill Terekhov's avatar
Kirill Terekhov committed
1040 1041 1042
					for(int i = 0; i < 4; i++)
					{
						if( !it->second[i].empty() )
Kirill Terekhov's avatar
Kirill Terekhov committed
1043 1044
							std::sort(it->second[i].begin(),it->second[i].end(),GlobalIDComparator(this));
							//qsort(&it->second[i][0],it->second[i].size(),sizeof(Element *),CompareElementsCGID);
Kirill Terekhov's avatar
Kirill Terekhov committed
1045
					}
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
1046
#endif //USE_PARALLEL_STORAGE
Kirill Terekhov's avatar
Kirill Terekhov committed
1047 1048 1049
			}
			else
			{
1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062
                
                if (only_new)
                {
		            for(Mesh::iteratorCell it = BeginCell(); it != EndCell(); it++) if (GetMarker(*it,NewMarker()))
                    {
                        ElementArray<Node> nodes = it->getNodes();
                        for (int i = 0; i < nodes.size(); i++)
                        {
                            nodes[i].SetMarker(NewMarker());
                        }
                    }
                }

Kirill Terekhov's avatar
Kirill Terekhov committed
1063 1064 1065
				double time = Timer();
				Storage::real epsilon = GetEpsilon();
			
Kirill Terekhov's avatar
Kirill Terekhov committed
1066
				GeomParam table;
1067
				for(ElementType etype = EDGE; etype <= CELL; etype = NextElementType(etype))
Kirill Terekhov's avatar
Kirill Terekhov committed
1068 1069 1070 1071 1072 1073 1074 1075 1076
					if( !HaveGeometricData(CENTROID,etype) )
						table[CENTROID] |= etype;
				PrepareGeometricData(table);
				
				time = Timer() - time;
			
				REPORT_STR("Prepare geometric data");
				REPORT_VAL("time",time);
			
Kirill Terekhov's avatar
Kirill Terekhov committed
1077 1078 1079 1080 1081
				//for(iteratorNode it = BeginNode(); it != EndNode(); it++)
#if defined(USE_OMP)
#pragma omp parallel for
#endif
				for(integer nit = 0; nit < NodeLastLocalID(); ++nit) if( isValidNode(nit) )
Kirill Terekhov's avatar
Kirill Terekhov committed
1082
				{
Kirill Terekhov's avatar
Kirill Terekhov committed
1083
					Node it = NodeByLocalID(nit);
Kirill Terekhov's avatar
Kirill Terekhov committed
1084 1085 1086 1087 1088 1089
					Storage::integer_array arr = it->IntegerArrayDV(tag_processors);
					arr.resize(1);
					arr[0] = mpirank;
				}
			

Kirill Terekhov's avatar
Kirill Terekhov committed
1090
				buffer_type exch_data;
Kirill Terekhov's avatar
Kirill Terekhov committed
1091 1092
				std::vector< INMOST_DATA_REAL_TYPE > unpack_real;
				std::vector< INMOST_DATA_REAL_TYPE > pack_real;
Kirill Terekhov's avatar
Kirill Terekhov committed
1093
				element_set sorted_nodes;
Kirill Terekhov's avatar
Kirill Terekhov committed
1094 1095 1096 1097 1098 1099 1100 1101 1102
				
				sorted_nodes.reserve(NumberOfNodes());
				
				
				time = Timer();
				
				
				for(iteratorNode n = BeginNode(); n != EndNode(); n++)
				{
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
1103 1104
					real_array c = n->Coords();
					for(real_array::size_type k = 0; k < procs.size(); k++)
1105
						if( point_in_bbox(c.data(),bboxs.data()+procs[k]*dim*2,dim,GetEpsilon()) )
Kirill Terekhov's avatar
Kirill Terekhov committed
1106
						{
Kirill Terekhov's avatar
Kirill Terekhov committed
1107
							sorted_nodes.push_back(*n);
Kirill Terekhov's avatar
Kirill Terekhov committed
1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121
							break;
						}
				}
				
				time = Timer() - time;
				REPORT_STR("Prepare array of nodes");
				REPORT_VAL("time",time);
				REPORT_VAL("share nodes", sorted_nodes.size());
				REPORT_VAL("total nodes", NumberOfNodes());
				
				
				
				time = Timer();
				if( !sorted_nodes.empty() )
Kirill Terekhov's avatar
Kirill Terekhov committed
1122 1123
					std::sort(sorted_nodes.begin(),sorted_nodes.end(),CentroidComparator(this));
					//qsort(&sorted_nodes[0],sorted_nodes.size(),sizeof(Element *),CompareElementsCCentroid);
Kirill Terekhov's avatar
Kirill Terekhov committed
1124 1125 1126 1127 1128 1129 1130 1131 1132
				time = Timer() - time;
				REPORT_STR("Sort nodes");
				REPORT_VAL("time",time);
				
				
				pack_real.reserve(sorted_nodes.size()*dim);
				
				
				time = Timer();
Kirill Terekhov's avatar
Kirill Terekhov committed
1133
				for(element_set::iterator it = sorted_nodes.begin(); it != sorted_nodes.end(); it++)
Kirill Terekhov's avatar
Kirill Terekhov committed
1134
				{
Kirill Terekhov's avatar
Kirill Terekhov committed
1135
					Storage::real_array arr = RealArrayDF(*it,CoordsTag());
Kirill Terekhov's avatar
Kirill Terekhov committed
1136 1137 1138 1139 1140 1141 1142 1143 1144
					pack_real.insert(pack_real.end(),arr.begin(),arr.end());
				}
				time = Timer() - time;
				REPORT_STR("Gather coordinates");
				REPORT_VAL("time",time);
				
				
				time = Timer();
			
Kirill Terekhov's avatar
Kirill Terekhov committed
1145
				MPI_Pack_size(static_cast<int>(pack_real.size()),INMOST_MPI_DATA_REAL_TYPE,comm,&sendsize);
Kirill Terekhov's avatar
Kirill Terekhov committed
1146 1147
				exch_data.resize(sendsize);
				int position = 0;
Kirill Terekhov's avatar
Kirill Terekhov committed
1148
				if( sendsize > 0 ) MPI_Pack(&pack_real[0],static_cast<INMOST_MPI_SIZE>(pack_real.size()),INMOST_MPI_DATA_REAL_TYPE,&exch_data[0],static_cast<INMOST_MPI_SIZE>(exch_data.size()),&position,comm);
Kirill Terekhov's avatar
Kirill Terekhov committed
1149 1150 1151 1152 1153 1154 1155
				
				
				time = Timer() - time;
				REPORT_STR("Pack coordinates");
				REPORT_VAL("time",time);

				
1156
				
Kirill Terekhov's avatar
Kirill Terekhov committed
1157 1158 1159 1160
				{
					std::vector< MPI_Request > send_reqs, recv_reqs;
					exch_buffer_type send_buffs(procs.size()), recv_buffs(procs.size());
					std::vector<int> done;
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
1161
//~ #if defined(USE_MPI_P2P)
Kirill Terekhov's avatar
Kirill Terekhov committed
1162 1163
					//~ unsigned * sendsizeall = shared_space;
					//~ unsigned usend[2] = {sendsize,pack_real.size()};
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
1164
					//~ REPORT_MPI(MPI_Win_fence(0,window)); //start exchange session
Kirill Terekhov's avatar
Kirill Terekhov committed
1165 1166
					//~ for(unsigned k = 0; k < procs.size(); k++)
						//~ REPORT_MPI(MPI_Put(usend,2,MPI_UNSIGNED,procs[k],mpirank*2,2,MPI_UNSIGNED,window));
Kirill Terekhov's avatar
Fixes  
Kirill Terekhov committed
1167
					//~ REPORT_MPI(MPI_Win_fence(MPI_MODE_NOSTORE | MPI_MODE_NOSUCCEED,window)); //end exchange session
Kirill Terekhov's avatar
Kirill Terekhov committed
1168 1169 1170
//~ #else
					std::vector<unsigned> sendsizeall(mpisize*2);
					int pack_size2 = 0;
Kirill Terekhov's avatar
Kirill Terekhov committed
1171
					unsigned usend[2] = {static_cast<unsigned>(sendsize),static_cast<unsigned>(pack_real.size())};
Kirill Terekhov's avatar
Kirill Terekhov committed
1172
					MPI_Pack_size(2,MPI_UNSIGNED,comm,&pack_size2);
Kirill Terekhov's avatar
Kirill Terekhov committed
1173
					for(dynarray<integer,64>::size_type k = 0; k < procs.size(); k++)
Kirill Terekhov's avatar
Kirill Terekhov committed
1174 1175 1176 1177
					{
						send_buffs[k].first = procs[k];
						send_buffs[k].second.resize(pack_size2);
						position = 0;
Kirill Terekhov's avatar
Kirill Terekhov committed
1178
						MPI_Pack(usend,2,MPI_UNSIGNED,&send_buffs[k].second[0],static_cast<INMOST_MPI_SIZE>(send_buffs[k].second.size()),&position,comm);
Kirill Terekhov's avatar
Kirill Terekhov committed
1179 1180 1181 1182 1183 1184 1185 1186 1187
						recv_buffs[k].first = procs[k];
						recv_buffs[k].second.resize(pack_size2);
					}
					ExchangeBuffersInner(send_buffs,recv_buffs,send_reqs,recv_reqs);
					while( !(done = FinishRequests(recv_reqs)).empty() )
					{
						for(std::vector<int>::iterator qt = done.begin(); qt != done.end(); qt++)
						{
							position = 0;
Kirill Terekhov's avatar